reelkit-cli 0.6.0 → 0.10.0

package/src/pipeline/schema.ts

@@ -7,7 +7,10 @@ export type Aspect = z.infer<typeof AspectSchema>;
 
 export const SceneSchema = z.object({
   id: z.string().regex(/^[a-z0-9-]+$/).describe("short unique id, lowercase letters, digits, dashes"),
-  narration: z.string().min(1).describe("the words spoken in this scene"),
+  // The minimum of one character is enforced for a narrated plan in ScenePlanSchema below, so that a video without a voice may leave it empty.
+  narration: z.string().describe("the words spoken in this scene; an empty string when the plan has voice \"none\""),
+  // How long the scene lasts. Used only when the plan has voice "none"; with a voice the recording sets the length.
+  seconds: z.number().min(0.3).max(15).optional().describe("the scene's length in seconds, from 0.3 to 15; required when the plan has voice \"none\""),
   // clip: a generated or reused video clip is the scene's picture.
   treatment: z.enum(["motion-graphic", "illustration", "footage-overlay", "clip"]),
   onScreenText: z.array(z.string()),
@@ -18,9 +21,14 @@ export const SceneSchema = z.object({
   imageTags: z.array(z.string()).describe("3 to 6 short tags describing the illustration; empty when imagePrompt is null"),
   userAssetIds: z.array(z.string()),
   notes: z.string().describe("visual direction for whoever writes the composition"),
+  // Says that this scene begins on a deliberate hard cut, so `preview` does not count the missing carry-over against the film.
+  cutIn: z.boolean().optional().describe("true when this scene begins on a deliberate hard cut (nothing is meant to carry over into it)"),
 });
 export type Scene = z.infer<typeof SceneSchema>;
 
+export const MAX_NARRATED_SCENES = 8;
+export const MAX_VOICELESS_SCENES = 16;
+
 export const ScenePlanSchema = z.object({
   title: z.string(),
   aspect: AspectSchema,
@@ -28,18 +36,45 @@ export const ScenePlanSchema = z.object({
   // Optional only so plans stored before voices existed still load; a new plan must choose one.
   voiceId: z.string().optional().describe("id of the narration voice, one of the ids from `reelkit assets voices`, chosen to suit the idea, audience and language"),
   pace: z.enum(["slow", "normal", "fast"]).optional().describe("speaking pace: slow for calm or emotional, normal by default, fast for high-energy"),
+  // The silence between one scene's last word and the next scene's first word: tight 0.2 s, normal 0.3 s (the default), relaxed 0.5 s.
+  // (`pace` is the speed of the voice itself, so the gap has its own setting.)
+  gap: z.enum(["tight", "normal", "relaxed"]).optional().describe("silence between one sentence and the next: tight 0.2 s, normal 0.3 s (the default), relaxed 0.5 s; a tight promo is tight"),
+  // Absent means narrated. "none" is a video carried by music and sound effects alone: scenes get their length from `seconds`.
+  voice: z.enum(["narrated", "none"]).optional().describe("narrated (the default), or none for a video with no voice"),
   // Whether the video has captions and of which kind: none, one word at a time, or a phrase at a time. Absent means "phrase".
-  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a phrase at a time (phrase, the default), as the user chose"),
+  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a few words at a time (phrase, the default; a few words at a time is phrase, not word), as the user chose"),
   // A video the new one is made "like": what is taken from it is how it feels (structure, pacing, motion), never its footage, music or words.
   // Optional, so plans stored before references existed still load.
   reference: z.object({
     id: z.string().describe("the id of a reference in this project, from `reelkit ref list`"),
     take: z.array(z.string()).min(1).max(6).describe("short notes on what is taken from it, e.g. \"fast cuts every ~1.2s\", \"big type on colour fields\""),
   }).optional(),
-  scenes: z.array(SceneSchema).min(3).max(8),
+  scenes: z.array(SceneSchema).min(3).max(MAX_VOICELESS_SCENES),
+}).superRefine((plan, ctx) => {
+  if (plan.voice === "none") {
+    plan.scenes.forEach((s, i) => {
+      if (s.seconds === undefined) ctx.addIssue({ code: "custom", path: ["scenes", i, "seconds"], message: `Scene ${s.id} needs seconds (a number from 0.3 to 15) because the video has no voice. Add it, or remove voice from the plan.` });
+    });
+    return;
+  }
+  // The narrated rules, as they were before voice existed.
+  if (plan.scenes.length > MAX_NARRATED_SCENES) ctx.addIssue({ code: "too_big", origin: "array", maximum: MAX_NARRATED_SCENES, inclusive: true, input: plan.scenes, path: ["scenes"] });
+  plan.scenes.forEach((s, i) => {
+    if (s.narration.length < 1) ctx.addIssue({ code: "too_small", origin: "string", minimum: 1, inclusive: true, input: s.narration, path: ["scenes", i, "narration"] });
+  });
 });
 
+export const isVoiceless = (plan: { voice?: "narrated" | "none" }): boolean => plan.voice === "none";
+
 export const PACE_SPEED = { slow: 0.92, normal: 1, fast: 1.1 } as const;
+// The silence, in seconds, between one scene's last word and the next scene's first word, by the plan's `gap`.
+export const GAP_SEC = { tight: 0.2, normal: 0.3, relaxed: 0.5 } as const;
+export const gapSec = (plan: { gap?: keyof typeof GAP_SEC }): number => GAP_SEC[plan.gap ?? "normal"];
+// The last scene keeps this long after its last word, so that the film does not end on the last syllable.
+export const LAST_TAIL_SEC = 0.7;
+// The voice speaks about this many words a second at `pace: "normal"` (measured on a recorded film: 2.0 to 2.4, 2.25 over the whole of it, the
+// pauses inside a sentence included). `plan check` estimates the length from it.
+export const VOICE_WORDS_PER_SEC = 2.25;
 export type ScenePlan = z.infer<typeof ScenePlanSchema>;
 
 export const AssetRecordSchema = z.object({
@@ -76,6 +111,16 @@ export const ManifestSceneSchema = z.object({
   clipKey: z.string().optional(),
   clipKeyedKey: z.string().optional(),
   userAssetKeys: z.array(z.string()),
+  // The scene's picture in layers, from `reelkit assets layers`: the picture itself, the subject cut out of it when it has a clear one (a PNG with alpha beside it) and its box,
+  // and where words can sit (`region`, how light and how busy). Pass it to <ImageLayers layers={...}> and <TextOnImage layers={...}>.
+  imageLayers: z.object({
+    back: z.string(),
+    subject: z.string().optional(),
+    subjectBox: z.object({ x: z.number(), y: z.number(), w: z.number(), h: z.number() }).optional(),
+    textZone: z.object({ region: z.enum(["top", "middle", "bottom", "left", "right"]), luminance: z.number(), busy: z.number() }),
+  }).optional(),
+  // The picture chosen as this scene's ground, with `reelkit assets pull <id> --background --scene <id>`.
+  background: z.object({ key: z.string() }).optional(),
 });
 export type ManifestScene = z.infer<typeof ManifestSceneSchema>;
 
@@ -91,6 +136,8 @@ export const AssetManifestSchema = z.object({
   captions: z.enum(["none", "word", "phrase"]).optional(),
   // Linear gain per sound file path that brings every pulled sound to a common level; the kit's Sfx and Music apply it.
   soundGain: z.record(z.string(), z.number()).optional(),
+  // The film's ground (a video or a picture), chosen with `--background`; durationSec is set for a video.
+  background: z.object({ key: z.string(), durationSec: z.number().optional() }).optional(),
   scenes: z.array(ManifestSceneSchema),
 });
 export type AssetManifest = z.infer<typeof AssetManifestSchema>;
@@ -102,7 +149,7 @@ export function validatePlan(
   ctx: { aspect: Aspect; footage?: AssetRecord; assets: AssetRecord[]; voiceIds?: string[] },
 ): string[] {
   const errors: string[] = [];
-  if (ctx.voiceIds) {
+  if (ctx.voiceIds && !isVoiceless(plan)) {
     if (!plan.voiceId) errors.push("Choose a narration voice: set voiceId to one of the ids from `reelkit assets voices`.");
     else if (!ctx.voiceIds.includes(plan.voiceId)) errors.push(`voiceId ${plan.voiceId} is not one of the available voices.`);
   }

package/src/pipeline/timing.ts

@@ -1,4 +1,4 @@
-import type { Aspect, AssetRecord } from "./schema";
+import { LAST_TAIL_SEC, type Aspect, type AssetRecord, type WordTiming } from "./schema";
 
 export const FPS = 30;
 export const SCENE_PADDING_SEC = 0.4;
@@ -28,3 +28,29 @@ export function dimensionsFor(aspect: Aspect, footage?: AssetRecord) {
   if (aspect === "1:1") return { width: 1080, height: 1080 };
   return { width: 1080, height: 1920 };
 }
+
+// A narrated scene's audio: when its last word ends (seconds from its start) and when its first word starts. A recording with no word times
+// counts as one stretch of speech as long as the file.
+export type SpokenScene = { durationSec: number; words: WordTiming[] };
+export const lastWordEndSec = (v: SpokenScene): number => (v.words.length ? Math.max(...v.words.map((w) => w.endSec)) : v.durationSec);
+export const firstWordStartSec = (v: SpokenScene): number => (v.words.length ? Math.min(...v.words.map((w) => w.startSec)) : 0);
+
+// The boundary never comes closer than this after a scene's last word, however tight the gap is.
+export const MIN_AFTER_WORD_SEC = 0.1;
+
+// Lays narrated scenes end to end so that the silence between one scene's last word and the next scene's first word is `gapSec`. Each scene's
+// audio starts on its first frame, so a scene lasts until its last word has ended plus the gap, less the silence its successor opens with. The
+// last scene keeps `tailSec` after its last word. Returns the natural lengths, before any snapping to the beat.
+export function layoutNarrated(scenes: SpokenScene[], gapSec: number, fps = FPS, tailSec = LAST_TAIL_SEC) {
+  let cursor = 0;
+  const out = scenes.map((v, i) => {
+    const end = lastWordEndSec(v);
+    const next = scenes[i + 1];
+    const seconds = next ? Math.max(end + gapSec - firstWordStartSec(next), end + MIN_AFTER_WORD_SEC) : end + tailSec;
+    const durationFrames = secondsToFrames(seconds, fps);
+    const scene = { startFrame: cursor, durationFrames };
+    cursor += durationFrames;
+    return scene;
+  });
+  return { scenes: out, totalFrames: cursor };
+}

package/src/project/background.ts

@@ -0,0 +1,33 @@
+import { FILES, type Project } from "./project";
+
+// The film's ground, when one was chosen: a video or a picture for the whole film, and pictures for single scenes (a video is only ever for the film).
+// Stored in assets/background.json and shown to the composition as manifest.background and manifest.scenes[i].background.
+export type BackgroundEntry = { key: string; id?: string; title?: string; durationSec?: number };
+export type BackgroundFile = { film?: BackgroundEntry; scenes: Record<string, BackgroundEntry> };
+
+export const readBackground = (project: Project): BackgroundFile => {
+  const raw = project.readJsonOr<Partial<BackgroundFile>>(FILES.background, {});
+  return { ...(raw.film ? { film: raw.film } : {}), scenes: raw.scenes ?? {} };
+};
+
+// One ground for the film: setting another replaces it. A scene's picture is set the same way, by scene id.
+export function setBackground(project: Project, entry: BackgroundEntry, sceneId?: string): BackgroundFile {
+  const current = readBackground(project);
+  const next: BackgroundFile = sceneId ? { ...current, scenes: { ...current.scenes, [sceneId]: entry } } : { ...current, film: entry };
+  project.writeJson(FILES.background, next);
+  return next;
+}
+
+export const VIDEO_EXT = /\.(mp4|mov|webm|m4v)$/i;
+export const isVideoPath = (path: string): boolean => VIDEO_EXT.test(path);
+
+// What is added to a prompt for a video or picture meant to sit behind everything, so that it stays calm and leaves room for type.
+export const BACKGROUND_CLIP_SENTENCE = "An abstract, slow, seamless ambient background with no people, no objects in focus, no text and no logos, soft and low in contrast so text stays readable on top.";
+export const BACKGROUND_IMAGE_SENTENCE = "An abstract, soft, low-contrast background with no people, no text and no logos, with empty space for type on top.";
+export const DEFAULT_BACKGROUND_LOOK = "soft, slowly shifting colour fields in a muted dark palette";
+
+// A look for a background that was asked for with no prompt: the colours the first scene's notes name, when they name any, else a neutral default.
+export function lookFromNotes(notes: string | undefined): string {
+  const named = [...new Set((notes ?? "").toLowerCase().match(/\b(?:black|white|navy|blue|teal|cyan|green|lime|yellow|amber|orange|red|crimson|pink|magenta|purple|violet|indigo|brown|beige|cream|grey|gray|gold|silver)\b|#[0-9a-f]{6}\b/g) ?? [])].slice(0, 4);
+  return named.length ? `soft, slowly shifting colour fields in ${named.join(", ")}` : DEFAULT_BACKGROUND_LOOK;
+}

package/src/project/chromakey.ts

@@ -41,7 +41,7 @@ async function cornerGreen(input: string): Promise<{ colour: string; saturated:
   return { colour: `0x${[r, g, b].map((n) => n.toString(16).padStart(2, "0")).join("")}`, saturated: g - Math.max(r, b) >= 60 };
 }
 
-// Turns the green of a green-screen clip into transparency: VP9 with an alpha channel, which Remotion plays with `transparent`.
+// Turns the green of a green-screen clip into transparency: VP9 with an alpha channel, which HyperFrames plays with `transparent`.
 // The key is on the green found in the frame's corners, with some give for noise and uneven light; the despill pulls the green glow
 // off the subject's edges. The original is left alone.
 export async function keyGreen(input: string, output: string, opts: KeyOptions = {}): Promise<void> {

package/src/project/layers.ts

@@ -0,0 +1,60 @@
+import { execFile } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { coverageOf, subjectBoxOf, textZoneOf, type ImageLayersRecord, type Rect, type TextZone } from "../hyperframes/kit/image-layers-math";
+
+const run = promisify(execFile);
+const GRID = 48;
+
+// The picture's size is capped, so that the 1-second video a cutout is made from stays small.
+const MAX_SIDE = 1920;
+
+async function raw(args: string[]): Promise<Buffer> {
+  try {
+    const { stdout } = await run("ffmpeg", ["-v", "error", ...args], { encoding: "buffer", maxBuffer: 1 << 24 });
+    return stdout;
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException)?.code === "ENOENT") throw new Error("ffmpeg was not found. Install ffmpeg (macOS: `brew install ffmpeg`) and run the command again.");
+    throw new Error(`ffmpeg could not read the picture: ${e instanceof Error ? e.message.split("\n")[0] : String(e)}`);
+  }
+}
+
+// A picture as a GRID by GRID grid of grey values.
+export async function greyGrid(image: string): Promise<Uint8Array> {
+  return new Uint8Array(await raw(["-i", image, "-frames:v", "1", "-vf", `scale=${GRID}:${GRID}:flags=area,format=gray`, "-f", "rawvideo", "-"]));
+}
+// The alpha channel of a picture as a GRID by GRID grid (255 opaque), 255 everywhere when it has none.
+export async function alphaGrid(image: string): Promise<Uint8Array> {
+  return new Uint8Array(await raw(["-i", image, "-frames:v", "1", "-vf", `format=rgba,alphaextract,scale=${GRID}:${GRID}:flags=area,format=gray`, "-f", "rawvideo", "-"]));
+}
+
+// Where words can sit in this picture: the calmest region, how light it is and how busy. `subject` is the subject's box when there is one.
+export async function measureTextZone(image: string, subject?: Rect): Promise<TextZone> {
+  return textZoneOf(await greyGrid(image), GRID, GRID, subject);
+}
+
+// The one-second video a still is sent to the cutout service as: 2 frames a second, even sides, never over 1920 on the long side.
+export async function stillToVideo(image: string, out: string): Promise<void> {
+  const vf = `scale='min(iw,${MAX_SIDE})':'min(ih,${MAX_SIDE})':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p`;
+  await raw(["-y", "-loop", "1", "-framerate", "2", "-i", image, "-t", "1", "-vf", vf, "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-an", out]);
+}
+
+// One frame of the transparent video the service returns, as a PNG with alpha: VP9 with alpha needs the libvpx decoder, so it is asked for by name.
+// Throws when the PNG has no transparent pixel at all (the alpha did not survive), and returns what the subject covers and the box that holds it.
+export async function frameWithAlpha(webm: string, png: string): Promise<{ coverage: number; box?: Rect }> {
+  await raw(["-y", "-c:v", "libvpx-vp9", "-i", webm, "-frames:v", "1", "-pix_fmt", "rgba", png]);
+  const a = await alphaGrid(png);
+  if (!a.some((v) => v < 250)) throw new Error("The cut-out came back without any transparent pixel, so there is no subject layer to make. The picture stays flat.");
+  return { coverage: coverageOf(a), box: subjectBoxOf(a, GRID, GRID) };
+}
+
+export const tempDir = (): { dir: string; done: () => void } => {
+  const dir = mkdtempSync(join(tmpdir(), "rk-layers-"));
+  return { dir, done: () => rmSync(dir, { recursive: true, force: true }) };
+};
+
+export type LayerEntry = { subject?: string; subjectBox?: Rect; textZone: TextZone; noSubject?: boolean; coverage?: number; cutoutId?: string };
+// The record the manifest carries for a picture: its key as `back`, the subject's file and box when there is one, and where text can sit.
+export const toImageLayers = (key: string, e: LayerEntry): ImageLayersRecord => ({ back: key, ...(e.subject && !e.noSubject ? { subject: e.subject, ...(e.subjectBox ? { subjectBox: e.subjectBox } : {}) } : {}), textZone: e.textZone });

package/src/project/manifest.ts

@@ -1,6 +1,8 @@
-import { PACE_SPEED, type AssetManifest, type ScenePlan, type WordTiming } from "../pipeline/schema";
-import { extendBeats, snapToBeats, toBeatFrames } from "../pipeline/beatsnap";
-import { dimensionsFor, FPS, layoutScenes } from "../pipeline/timing";
+import { gapSec, isVoiceless, PACE_SPEED, type AssetManifest, type ScenePlan, type WordTiming } from "../pipeline/schema";
+import { extendBeats, MIN_TAIL_SEC, snapNarrated, snapToBeats, toBeatFrames } from "../pipeline/beatsnap";
+import { dimensionsFor, FPS, lastWordEndSec, layoutNarrated, layoutScenes, secondsToFrames } from "../pipeline/timing";
+import { readBackground } from "./background";
+import { toImageLayers, type LayerEntry } from "./layers";
 import { dbToLinear } from "./loudness";
 import type { MusicRecord } from "./music";
 import { FILES, type Project } from "./project";
@@ -17,26 +19,48 @@ export function voiceoverStale(vo: Voiceover, scene: ScenePlan["scenes"][number]
   return vo.text !== scene.narration || vo.voiceId !== plan.voiceId || vo.speed !== PACE_SPEED[plan.pace ?? "normal"];
 }
 
+// What the length of a film with no voice comes to once its cuts are on the beat of the pulled track, in seconds. Undefined without a track
+// whose tempo can be trusted. Used by `plan check`, which has no manifest yet.
+export function snappedSeconds(project: Project, plan: ScenePlan): number | undefined {
+  if (!isVoiceless(plan)) return undefined;
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  if (!music?.bpm || !music.beats.length) return undefined;
+  const layout = layoutScenes(plan.scenes.map((s) => s.seconds ?? 0), FPS, 0);
+  const beatFrames = toBeatFrames(extendBeats(music.beats, music.bpm, layout.totalFrames / FPS + 10), FPS);
+  const lengths = snapToBeats({ naturalFrames: layout.scenes.map((s) => s.durationFrames), lastWordEndFrame: 0, beatFrames, fps: FPS, grid: "nearest" });
+  return lengths.reduce((a, b) => a + b, 0) / FPS;
+}
+
 // Lays the scenes out on the timeline from the real voiceover lengths. Paths are relative to the project folder.
 // Returns undefined, and writes nothing, until every scene has its voiceover.
 export function buildManifest(project: Project, plan: ScenePlan): AssetManifest | undefined {
-  const voiceovers = project.readJsonOr<Record<string, Voiceover>>(FILES.voiceovers, {});
-  if (plan.scenes.some((s) => !voiceovers[s.id])) return undefined;
+  const voiceless = isVoiceless(plan);
+  const voiceovers = voiceless ? {} : project.readJsonOr<Record<string, Voiceover>>(FILES.voiceovers, {});
+  if (!voiceless && plan.scenes.some((s) => !voiceovers[s.id])) return undefined;
   const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
   const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
   const footage = project.footage();
   const byId = new Map(project.assets().map((a) => [a.id, a]));
 
-  let layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  // With no voice the plan's own seconds are the lengths, with no padding after them: the plan already says how long each scene holds.
+  // With a voice each scene lasts until its last word has ended plus the plan's gap (the silence before the next sentence); the last keeps a longer tail.
+  let layout = voiceless ? layoutScenes(plan.scenes.map((s) => s.seconds ?? 0), FPS, 0) : layoutNarrated(plan.scenes.map((s) => voiceovers[s.id]!), gapSec(plan), FPS);
   const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
   let beatFrames: number[] = [];
   if (music) {
     beatFrames = toBeatFrames(extendBeats(music.beats, music.bpm ?? 120, layout.totalFrames / FPS + 10), FPS);
     // Scene changes land on beats only when the track has a trusted tempo. Footage has a fixed length that a longer scene must not overrun.
     if (music.bpm && music.beats.length) {
-      const last = voiceovers[plan.scenes[plan.scenes.length - 1].id];
-      const lastWords = last.words.length ? Math.max(...last.words.map((w) => w.endSec)) : last.durationSec;
-      const lengths = snapToBeats({ naturalFrames: layout.scenes.map((s) => s.durationFrames), lastWordEndFrame: Math.ceil(lastWords * FPS), beatFrames, fps: FPS });
+      const naturalFrames = layout.scenes.map((s) => s.durationFrames);
+      let lengths: number[];
+      if (voiceless) {
+        // No word to leave a tail after: the last scene is as long as the plan says, so the tail rule must not stretch it.
+        const lastWordEndFrame = Math.max(0, naturalFrames[naturalFrames.length - 1]! - Math.ceil(MIN_TAIL_SEC * FPS));
+        lengths = snapToBeats({ naturalFrames, lastWordEndFrame, beatFrames, fps: FPS, grid: "nearest" });
+      } else {
+        // The voice is the clock: a change moves to a beat only when that is a few frames away and cuts into no word.
+        lengths = snapNarrated({ naturalFrames, lastWordEndFrames: plan.scenes.map((s) => Math.ceil(lastWordEndSec(voiceovers[s.id]!) * FPS - 1e-9)), beatFrames }).frames;
+      }
       let cursor = 0;
       const snapped = { scenes: lengths.map((durationFrames) => { const r = { startFrame: cursor, durationFrames }; cursor += durationFrames; return r; }), totalFrames: 0 };
       snapped.totalFrames = cursor;
@@ -47,7 +71,9 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
   if (footage) {
     const footageFrames = Math.floor((footage.durationSec ?? 0) * FPS);
     if (layout.totalFrames > footageFrames) {
-      throw new Error(`The voiceover (${(layout.totalFrames / FPS).toFixed(1)}s) is longer than the footage (${(footageFrames / FPS).toFixed(1)}s). Use a longer clip or shorten the script.`);
+      throw new Error(voiceless
+        ? `The scenes (${(layout.totalFrames / FPS).toFixed(1)}s) are longer than the footage (${(footageFrames / FPS).toFixed(1)}s). Use a longer clip or shorten the scenes' seconds.`
+        : `The voiceover (${(layout.totalFrames / FPS).toFixed(1)}s) is longer than the footage (${(footageFrames / FPS).toFixed(1)}s). Use a longer clip or shorten the script.`);
     }
     totalFrames = footageFrames;
   }
@@ -59,22 +85,27 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
   }
   if (music) soundGain[music.key] = dbToLinear(music.gainDb);
 
+  const background = readBackground(project);
+  const layerEntries = project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {});
   const manifest: AssetManifest = {
     fps: FPS,
     ...dimensionsFor(plan.aspect, footage),
     totalFrames,
-    captions: plan.captions ?? "phrase",
+    captions: plan.captions ?? (voiceless ? "none" : "phrase"),
     ...(Object.keys(soundGain).length ? { soundGain } : {}),
     ...(footage ? { footageKey: footage.key } : {}),
+    ...(background.film ? { background: { key: background.film.key, ...(background.film.durationSec ? { durationSec: background.film.durationSec } : {}) } } : {}),
     ...(music ? { music: { key: music.key, ...(music.bpm ? { bpm: music.bpm } : {}), beatFrames: music.bpm ? beatFrames.filter((f) => f < totalFrames) : [] } } : {}),
     scenes: plan.scenes.map((scene, i) => ({
       id: scene.id,
       startFrame: layout.scenes[i].startFrame,
       durationFrames: layout.scenes[i].durationFrames,
-      voiceoverKey: voiceovers[scene.id].key,
-      words: voiceovers[scene.id].words,
+      ...(voiceless ? {} : { voiceoverKey: voiceovers[scene.id]!.key }),
+      words: voiceless ? [] : voiceovers[scene.id]!.words,
       ...(scene.treatment === "illustration" && images[scene.id] ? { imageKey: images[scene.id] } : {}),
+      ...(scene.treatment === "illustration" && images[scene.id] && layerEntries[images[scene.id]!] ? { imageLayers: toImageLayers(images[scene.id]!, layerEntries[images[scene.id]!]!) } : {}),
       ...(scene.treatment === "clip" && clips[scene.id] ? { clipKey: clips[scene.id].key, ...(clips[scene.id].keyedKey ? { clipKeyedKey: clips[scene.id].keyedKey } : {}) } : {}),
+      ...(background.scenes[scene.id] ? { background: { key: background.scenes[scene.id]!.key } } : {}),
       userAssetKeys: scene.userAssetIds.map((id) => {
         const a = byId.get(id);
         if (!a) throw new Error(`Scene ${scene.id} references unknown asset ${id}. Add it with \`reelkit assets upload\`.`);
@@ -92,10 +123,12 @@ export function missingAssets(project: Project, plan: ScenePlan): string[] {
   const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
   const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
   const problems: string[] = [];
+  const voiceless = isVoiceless(plan);
   const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
   if (music && !project.exists(music.key)) problems.push(`Missing file: ${music.key}. Pull the track again with \`reelkit assets pull ${music.id} --music\`.`);
   for (const s of plan.scenes) {
-    if (!voiceovers[s.id]) problems.push(`Scene ${s.id} has no voiceover.`);
+    if (voiceless) { /* nothing is recorded for a video with no voice */ }
+    else if (!voiceovers[s.id]) problems.push(`Scene ${s.id} has no voiceover.`);
     else if (voiceoverStale(voiceovers[s.id], s, plan)) problems.push(`Scene ${s.id}'s voiceover is out of date. Run \`reelkit assets voiceover --all\`.`);
     else if (!project.exists(voiceovers[s.id].key)) problems.push(`Missing file: ${voiceovers[s.id].key}`);
     // Only an illustration scene uses an image; one left over from an earlier plan is not needed.
@@ -111,3 +144,15 @@ export function missingAssets(project: Project, plan: ScenePlan): string[] {
   }
   return problems;
 }
+
+// The silence between each scene's last word and the next scene's first word, in seconds, as the manifest lays them out. This is what a listener hears
+// between two sentences, and it is what the plan's `gap` sets (the beat can move a change by a few frames).
+export function sentenceGaps(manifest: AssetManifest): number[] {
+  return manifest.scenes.slice(0, -1).map((s, i) => {
+    const next = manifest.scenes[i + 1]!;
+    if (!s.words.length || !next.words.length) return Math.max(0, (next.startFrame - s.startFrame) / manifest.fps);
+    const end = s.startFrame / manifest.fps + Math.max(...s.words.map((w) => w.endSec));
+    const start = next.startFrame / manifest.fps + Math.min(...next.words.map((w) => w.startSec));
+    return Math.round((start - end) * 100) / 100;
+  });
+}

package/src/project/music.ts

@@ -1,4 +1,5 @@
 import type { AssetManifest } from "../pipeline/schema";
+import { gridPoints } from "../pipeline/beatsnap";
 import { analyzeBeats } from "./beats";
 import { audioDuration, decodeMono } from "./refmeasure";
 
@@ -14,12 +15,25 @@ export async function measureMusic(path: string): Promise<Pick<MusicRecord, "dur
   return { durationSec, beatConfidence: r.confidence, beats: r.tempoBpm !== undefined && r.beats ? r.beats : [], ...(r.tempoBpm !== undefined ? { bpm: r.tempoBpm } : {}) };
 }
 
-// How many scene changes fall on a beat, for the preview.
-export function beatReport(manifest: AssetManifest): { bpm: number; boundariesOnBeat: number; boundaries: number; line: string } | undefined {
+// How many scene changes fall on a beat, for the preview. A film with no voice puts its cuts on a finer grid, so there a change on a half or
+// a quarter beat counts too, and the line says which grid was needed.
+export function beatReport(manifest: AssetManifest, opts: { finer?: boolean } = {}): { bpm: number; boundariesOnBeat: number; boundaries: number; line: string } | undefined {
   const m = manifest.music;
   if (!m?.bpm || manifest.scenes.length < 2) return undefined;
-  const beats = new Set(m.beatFrames);
   const starts = manifest.scenes.slice(1).map((s) => s.startFrame);
-  const on = starts.filter((f) => beats.has(f)).length;
-  return { bpm: m.bpm, boundariesOnBeat: on, boundaries: starts.length, line: `${on} of ${starts.length} scene changes land on the beat at ${Math.round(m.bpm)} BPM` };
+  const beats = new Set(m.beatFrames);
+  const bpm = Math.round(m.bpm);
+  if (!opts.finer) {
+    const on = starts.filter((f) => beats.has(f)).length;
+    // With a narrator the voice sets where a scene ends: a change is moved to a beat only when that is a few frames away, so say honestly how many are.
+    const rest = starts.length - on;
+    return { bpm: m.bpm, boundariesOnBeat: on, boundaries: starts.length, line: `${on} of ${starts.length} scene changes land on the beat at ${bpm} BPM${rest ? `; the other${rest === 1 ? "" : "s"} follow${rest === 1 ? "s" : ""} the voice` : ""}` };
+  }
+  const halves = new Set(gridPoints(m.beatFrames, 2)), quarters = new Set(gridPoints(m.beatFrames, 4));
+  const onBeat = starts.filter((f) => beats.has(f)).length;
+  const onHalf = starts.filter((f) => !beats.has(f) && halves.has(f)).length;
+  const onQuarter = starts.filter((f) => !beats.has(f) && !halves.has(f) && quarters.has(f)).length;
+  const on = onBeat + onHalf + onQuarter;
+  const grid = onQuarter ? "the beat, a half beat or a quarter beat" : onHalf ? "the beat or a half beat" : "the beat";
+  return { bpm: m.bpm, boundariesOnBeat: on, boundaries: starts.length, line: `${on} of ${starts.length} scene changes land on ${grid} at ${bpm} BPM` };
 }

package/src/project/project.ts

@@ -5,7 +5,7 @@ import { AspectSchema, type AssetRecord } from "../pipeline/schema";
 
 export const FILES = {
   config: "reelkit.json", plan: "plan.json", manifest: "manifest.json",
-  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json", music: "assets/music.json", searches: "assets/searches.json", shared: "assets/shared.json",
+  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json", music: "assets/music.json", background: "assets/background.json", layers: "assets/layers.json", searches: "assets/searches.json", shared: "assets/shared.json",
 } as const;
 
 // "path: message", or just the message for a problem at the root of the file.
@@ -17,6 +17,9 @@ export const ProjectConfigSchema = z.object({ aspect: AspectSchema, name: z.stri
   shareComponents: z.boolean().optional() });
 export type ProjectConfig = z.infer<typeof ProjectConfigSchema>;
 
+// True for a project made with `reelkit init --private` (shareComponents: false): nothing in it is sent to the library, whatever the plan or a flag says.
+export const isPrivateProject = (project: { config(): ProjectConfig }): boolean => project.config().shareComponents === false;
+
 // One video's working folder. Every command reads and writes its state here.
 export class Project {
   constructor(readonly dir: string) {}

package/src/project/serve.ts

@@ -12,9 +12,9 @@ const TYPES: Record<string, string> = {
 
 export type MediaServer = { url(rel: string): string; close(): Promise<void> };
 
-// Remotion loads media over HTTP, so the project's media is served on a local port while a build runs.
+// HyperFrames loads media over HTTP, so the project's media is served on a local port while a build runs.
 // Only files under <project>/assets/ are served, never .json, and only under a random per-run path prefix: access-control-allow-origin
-// has to be * for Remotion's renderer, so the unguessable prefix is what keeps other web pages from reading the files.
+// has to be * for HyperFrames's renderer, so the unguessable prefix is what keeps other web pages from reading the files.
 export async function serveDir(root: string): Promise<MediaServer> {
   const assets = join(resolve(root), "assets");
   const prefix = randomBytes(16).toString("hex");