reelkit-cli 0.5.0 → 0.8.0

package/src/commands/plan.ts

@@ -1,7 +1,8 @@
 import { client, type Ctx, type Result } from "../context";
 import { loadCredentials } from "../credentials";
 import { estimateLength, lengthVariation, reviewPlan, sceneSeconds } from "../pipeline/review";
-import { ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
+import { isVoiceless, ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
+import { snappedSeconds } from "../project/manifest";
 import { FILES, issueLines, openProject, type Project } from "../project/project";
 
 export function loadPlan(project: Project): ScenePlan {
@@ -52,7 +53,8 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
   }
   const footage = project.footage();
   const assets = project.assets().filter((a) => a.id !== footage?.id);
-  const voiceIds = loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
+  const voiceless = isVoiceless(parsed.data);
+  const voiceIds = !voiceless && loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
   const ref = referenceChecks(project, parsed.data);
   const mustFix = [...validatePlan(parsed.data, { aspect: project.config().aspect, footage, assets, voiceIds }), ...ref.mustFix];
   const shouldImprove = mustFix.length ? [] : [...reviewPlan(parsed.data, { footage }), ...ref.improve];
@@ -60,11 +62,12 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
   const estimatedSeconds = Math.round(length.seconds);
   const perScene = sceneSeconds(parsed.data).map((s) => ({ id: s.id, seconds: Math.round(s.seconds * 10) / 10 }));
   const variation = Math.round(lengthVariation(sceneSeconds(parsed.data).map((s) => s.seconds)) * 100) / 100;
+  const snapped = voiceless ? snappedSeconds(project, parsed.data) : undefined;
   const lines = [
     mustFix.length ? `Fix these:\n- ${mustFix.join("\n- ")}` : "The plan is valid.",
-    `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
+    voiceless ? `Length: ${(Math.round(length.seconds * 10) / 10).toFixed(1)}s over ${parsed.data.scenes.length} scenes${snapped !== undefined ? `; about ${snapped.toFixed(1)}s once the cuts are on the beat` : ""}.` : `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
     ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : []),
-    ...(voiceIds ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
+    ...(voiceIds || voiceless ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
   ];
-  return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation }, summary: lines.join("\n") };
+  return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation, ...(snapped !== undefined ? { snappedSeconds: Math.round(snapped * 10) / 10 } : {}) }, summary: lines.join("\n") };
 }

package/src/commands/ref.ts

@@ -60,11 +60,14 @@ const realYtdlp: YtdlpRunner = {
 
 const YTDLP_MISSING = "yt-dlp is not installed, and links need it. Install it (macOS: `brew install yt-dlp`, or `pipx install yt-dlp`) and run the command again; a video file on this machine works without it.";
 
+// The duration filter uses "<=?": some sites (Instagram) give no duration before the download, and without the "?" such a video is
+// skipped as if it were too long. A video that turns out longer is refused after it is probed.
+// No "--max-downloads 1": yt-dlp exits with an error code after reaching it even when the one video was saved.
 // One video, at most 1080p, as mp4, and nothing longer than the limit: the flags that keep a link from fetching a playlist, a channel or a huge file.
 export function ytdlpArgs(url: string, dest: string): string[] {
   return [
-    "--ignore-config", "--no-playlist", "--max-downloads", "1", "--no-progress", "--no-warnings",
-    "--match-filter", `duration<=${MAX_REF_SECONDS}`,
+    "--ignore-config", "--no-playlist", "--no-progress", "--no-warnings",
+    "--match-filter", `duration<=?${MAX_REF_SECONDS}`,
     "-f", "bv*[height<=1080][ext=mp4]+ba[ext=m4a]/b[height<=1080][ext=mp4]/bv*[height<=1080]+ba/b[height<=1080]",
     "--merge-output-format", "mp4", "-o", dest, "--", url,
   ];

package/src/contract/index.ts

@@ -13,7 +13,12 @@
 // - A voiceover `text` must contain at least one non-space character: whitespace alone is 400 invalid_request.
 // - A library item `id` (pull, commit) matches `^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`; a `deviceCode` is 1 to 200 characters of plain text.
 // - A `voiceId` is 1 to 64 letters and digits and must be one of the voices the server lists (`voices`), otherwise 400 invalid_request.
-// - Components are added to the library only by the owner's scripts, never through this API (UploadKindSchema has no `component`).
+// - A user's component is uploaded for review through the same two calls as any other file (`kind: "component"`). The server checks its source
+//   at commit with the same static check a pulled component gets (only react, remotion and reelkit/kit imports, no network, no globals that reach
+//   outside the video) and refuses a source that fails it with 400 `invalid_request` and the first problem. It stays in `review`, is never found
+//   by search or the public listing, and is published only by the owner. A component upload is: `contentType` `text/plain`, `bytes` up to 65,536,
+//   `shareable` true, `filename` matching `^[A-Z][A-Za-z0-9]*\.tsx$`, a `description` of at least 20 characters, and `meta.example` a string of at most
+//   2,000 characters that starts with `<` and the file's name (`<StatCard value="42" />` for StatCard.tsx). Anything else is 400 `invalid_request`.
 // - Upload is a single `PUT` to `uploadUrl` with exactly the declared content type and byte length, and no auth header.
 //   Anything else is refused with a non-2xx status and stores nothing. That refusal comes from the storage service, so its body
 //   is not an API error and a client must not parse it.
@@ -105,6 +110,11 @@ const Meta = z.record(z.string(), z.unknown()).refine((m) => new TextEncoder().e
 
 // The most a single upload may be: 200 MB.
 export const MAX_UPLOAD_BYTES = 209_715_200;
+// A component's source is at most 64 KB, its file name is PascalCase, and its description and example have the sizes below.
+export const MAX_COMPONENT_BYTES = 65_536;
+export const COMPONENT_FILE = /^[A-Z][A-Za-z0-9]*\.tsx$/;
+export const MIN_COMPONENT_DESCRIPTION = 20;
+export const MAX_COMPONENT_EXAMPLE = 2000;
 // The video types a cutout takes.
 export const CutoutTypeSchema = z.enum(["video/mp4", "video/quicktime", "video/webm"]);
 export type CutoutType = z.infer<typeof CutoutTypeSchema>;
@@ -124,8 +134,8 @@ export const ApiErrorSchema = z.object({ error: z.object({ code: z.string(), mes
 
 export const LibraryKindSchema = z.enum(["image", "overlay", "sfx", "music", "component", "clip"]);
 export type LibraryKind = z.infer<typeof LibraryKindSchema>;
-// What a user may upload: everything except components, which are code and come only from Reelkit.
-export const UploadKindSchema = LibraryKindSchema.exclude(["component"]);
+// What a user may upload: every kind. A component is code and is held for review (see the rules at the top).
+export const UploadKindSchema = LibraryKindSchema;
 export type UploadKind = z.infer<typeof UploadKindSchema>;
 
 export const LibraryItemSchema = z.object({
@@ -183,6 +193,16 @@ export const routes = {
       kind: UploadKindSchema, title: text(200).min(1), description: text(2000), tags: z.array(text(40)).max(20),
       meta: Meta, filename: text(200).min(1), contentType: text(100), bytes: z.number().int().positive().max(MAX_UPLOAD_BYTES),
       shareable: z.boolean(),
+    }).superRefine((r, ctx) => {
+      if (r.kind !== "component") return;
+      const bad = (path: string, message: string) => ctx.addIssue({ code: "custom", path: [path], message });
+      if (r.contentType.toLowerCase() !== "text/plain") bad("contentType", "a component must be text/plain");
+      if (r.bytes > MAX_COMPONENT_BYTES) bad("bytes", `a component is at most ${MAX_COMPONENT_BYTES} bytes`);
+      if (!COMPONENT_FILE.test(r.filename)) bad("filename", "a component file is named like StatCard.tsx");
+      if (!r.shareable) bad("shareable", "a component is only uploaded to be shared for review");
+      if (r.description.trim().length < MIN_COMPONENT_DESCRIPTION) bad("description", `a component needs a description of at least ${MIN_COMPONENT_DESCRIPTION} characters`);
+      const example = r.meta.example;
+      if (typeof example !== "string" || example.length > MAX_COMPONENT_EXAMPLE || !example.startsWith(`<${r.filename.replace(/\.tsx$/, "")}`)) bad("meta.example", `a component needs meta.example, at most ${MAX_COMPONENT_EXAMPLE} characters, that starts with <Name`);
     }),
     z.object({ id: z.string(), uploadUrl: z.string() })),
   libraryCommit: route("POST", "/library/upload/commit", true, z.object({ id: ItemId }), z.object({ item: LibraryItemSchema })),
@@ -190,10 +210,12 @@ export const routes = {
   voiceover: route("POST", "/voiceover", true,
     z.object({ text: text(5000).refine((t) => /\S/.test(t), "must contain a non-space character"), voiceId: z.string().regex(/^[A-Za-z0-9]{1,64}$/).optional(), speed: z.number().min(0.7).max(1.3).optional() }),
     z.object({ url: z.string(), ext: z.enum(["mp3", "wav", "m4a"]), contentType: z.string(), durationSec: z.number().positive(), words: z.array(WordTimingSchema), chars: z.number() })),
+  // `format` is "png" (a raster image, the default) or "svg" (a vector graphic). An SVG counts as one image against the image quota, takes about 45 s, and the
+  // server returns only a cleaned SVG: no scripts, no event attributes, no outside references. Its `ext` is "svg" and its contentType `image/svg+xml`.
   images: route("POST", "/images", true,
-    z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12) }),
+    z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12), format: z.enum(["png", "svg"]).default("png") }),
     // libraryId is set when the image was also added to the shared library.
-    z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp"]), contentType: z.string(), libraryId: z.string().optional() })),
+    z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp", "svg"]), contentType: z.string(), libraryId: z.string().optional() })),
   clipStart: route("POST", "/clips", true,
     z.object({
       prompt: text(2000).min(1), aspect: AspectSchema, greenScreen: z.boolean().default(false), durationSec: z.union([z.literal(5), z.literal(10)]).default(5),

package/src/pipeline/beatsnap.ts

@@ -12,6 +12,9 @@ export type SnapInput = {
   // Beats as frame numbers, ascending. They should reach past the end of the video.
   beatFrames: number[];
   fps: number;
+  // "next" holds a scene to the next whole beat (narration cannot be cut). "nearest" moves each end to the closest point on a finer grid
+  // (a film with no voice can be shortened as well as lengthened, so its total stays near the plan).
+  grid?: "next" | "nearest";
 };
 
 const firstAtOrAfter = (beats: number[], frame: number): number | undefined => beats.find((b) => b >= frame);
@@ -21,6 +24,7 @@ const firstAtOrAfter = (beats: number[], frame: number): number | undefined => b
 export function snapToBeats(input: SnapInput): number[] {
   const { naturalFrames, beatFrames, fps } = input;
   if (beatFrames.length < 2) return [...naturalFrames];
+  if (input.grid === "nearest") return snapToNearest(naturalFrames, beatFrames);
   const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
   const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
   let cursor = 0;
@@ -40,6 +44,74 @@ export function snapToBeats(input: SnapInput): number[] {
   });
 }
 
+// How far a scene change of a narrated film may move to reach a beat, in frames, either way.
+export const NARRATED_SNAP_FRAMES = 4;
+// A boundary never comes closer than this to the last word of the scene it ends, in frames.
+export const NARRATED_WORD_CLEARANCE = 2;
+
+// Scene changes for a narrated film. The voice sets where a scene ends; a boundary is moved to the nearest beat only when that is at most
+// `maxMove` frames away either way and leaves at least `clearance` frames after the scene's last word (a boundary never cuts into a word). The
+// next scene's audio starts on its first frame, so it moves with the boundary. Every other boundary stays where the narration puts it, and the
+// last scene is not touched: the film's end is not a scene change. Each boundary is judged from where its scene actually starts, so
+// nothing adds up. `lastWordEndFrames[i]` is when scene i's last word ends, in frames from its own start.
+export function snapNarrated(input: { naturalFrames: number[]; lastWordEndFrames: number[]; beatFrames: number[]; maxMove?: number; clearance?: number }): { frames: number[]; onBeat: boolean[] } {
+  const { naturalFrames, lastWordEndFrames, beatFrames } = input;
+  const maxMove = input.maxMove ?? NARRATED_SNAP_FRAMES, clearance = input.clearance ?? NARRATED_WORD_CLEARANCE;
+  let cursor = 0;
+  const onBeat: boolean[] = [];
+  const frames = naturalFrames.map((natural, i) => {
+    let end = cursor + natural;
+    if (i < naturalFrames.length - 1) {
+      const floor = cursor + (lastWordEndFrames[i] ?? 0) + clearance;
+      let best: number | undefined;
+      for (const b of beatFrames) {
+        if (Math.abs(b - end) > maxMove || b < floor) continue;
+        if (best === undefined || Math.abs(b - end) < Math.abs(best - end) || (Math.abs(b - end) === Math.abs(best - end) && b > best)) best = b;
+      }
+      if (best !== undefined) end = best;
+      onBeat.push(beatFrames.includes(end));
+    }
+    const length = end - cursor;
+    cursor = end;
+    return length;
+  });
+  return { frames, onBeat };
+}
+
+// The beat frames with `divisions - 1` evenly spaced points between each pair, rounded to frames. The last beat is followed by one more
+// at the median spacing, so a point just after it exists too.
+export function gridPoints(beatFrames: number[], divisions: number): number[] {
+  if (beatFrames.length < 2) return [...beatFrames];
+  const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
+  const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
+  const all = [...beatFrames, beatFrames[beatFrames.length - 1]! + period];
+  const out = new Set<number>();
+  for (let i = 0; i + 1 < all.length; i++) for (let k = 0; k < divisions; k++) out.add(Math.round(all[i]! + ((all[i + 1]! - all[i]!) * k) / divisions));
+  out.add(all[all.length - 1]!);
+  return [...out].sort((a, b) => a - b);
+}
+
+// A scene of at least 1.5 beats ends on the nearest whole beat; a shorter one on the nearest half beat, and under half a beat the nearest
+// quarter. The target is the planned end (the sum of the natural lengths), so rounding errors do not add up; a scene is never made empty.
+function snapToNearest(naturalFrames: number[], beatFrames: number[]): number[] {
+  const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
+  const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
+  const grids = new Map<number, number[]>();
+  const grid = (d: number) => grids.get(d) ?? (grids.set(d, gridPoints(beatFrames, d)), grids.get(d)!);
+  let cursor = 0, planned = 0;
+  return naturalFrames.map((natural) => {
+    planned += natural;
+    const beats = natural / period;
+    const points = grid(beats >= 1.5 ? 1 : beats >= 0.5 ? 2 : 4).filter((f) => f > cursor);
+    let target = planned;
+    if (points.length) target = points.reduce((best, f) => (Math.abs(f - planned) < Math.abs(best - planned) ? f : best));
+    else target = Math.max(planned, cursor + 1);
+    const frames = target - cursor;
+    cursor = target;
+    return frames;
+  });
+}
+
 // Beats in seconds, continued at the tempo's own spacing until `untilSec`, for a video longer than the track (the track loops).
 export function extendBeats(beats: number[], bpm: number, untilSec: number): number[] {
   if (!beats.length) return [];

package/src/pipeline/review.ts

@@ -1,16 +1,23 @@
-import { WORDS_PER_SEC, type AssetRecord, type ScenePlan } from "./schema";
+import { gapSec, isVoiceless, LAST_TAIL_SEC, PACE_SPEED, VOICE_WORDS_PER_SEC, type AssetRecord, type ScenePlan } from "./schema";
 
 const words = (s: string) => s.trim().split(/\s+/).filter(Boolean).length;
 
-// The same estimate the length advice below is based on: narration words at the voice's usual pace.
+// What the voice speaks at, in words a second, for this plan's pace.
+const speechRate = (plan: ScenePlan) => VOICE_WORDS_PER_SEC * PACE_SPEED[plan.pace ?? "normal"];
+
+// The estimate the length advice below is based on: each scene's words at the voice's real speed (2.25 words a second at the normal pace, the
+// pauses within a sentence included), the plan's silence between sentences after every scene but the last, and the last scene's tail after its
+// last word. It matches the manifest `reelkit assets voiceover` builds. A video with no voice has no words to count: its length is the sum of the
+// scenes' own seconds.
 export function estimateLength(plan: ScenePlan): { words: number; seconds: number } {
+  if (isVoiceless(plan)) return { words: 0, seconds: plan.scenes.reduce((n, s) => n + (s.seconds ?? 0), 0) };
   const total = plan.scenes.reduce((n, s) => n + words(s.narration), 0);
-  return { words: total, seconds: total / WORDS_PER_SEC };
+  return { words: total, seconds: total / speechRate(plan) + (plan.scenes.length - 1) * gapSec(plan) + LAST_TAIL_SEC };
 }
 
-// Each scene's estimated length in seconds, from its narration at the same pace as the whole-video estimate above.
+// Each scene's estimated length in seconds: its speech and what follows it (the gap, or the tail for the last scene), so they add up to the whole estimate.
 export function sceneSeconds(plan: ScenePlan): { id: string; seconds: number }[] {
-  return plan.scenes.map((s) => ({ id: s.id, seconds: words(s.narration) / WORDS_PER_SEC }));
+  return plan.scenes.map((s, i) => ({ id: s.id, seconds: isVoiceless(plan) ? (s.seconds ?? 0) : words(s.narration) / speechRate(plan) + (i === plan.scenes.length - 1 ? LAST_TAIL_SEC : gapSec(plan)) }));
 }
 
 // How uneven the scene lengths are: the standard deviation over the mean (0 when they are all the same).
@@ -29,12 +36,14 @@ export function rhythmNote(plan: ScenePlan): string | undefined {
   const cv = lengthVariation(values);
   if (cv >= 0.25 && values.some((v) => v < mean / 2)) return undefined;
   const shortest = lengths.reduce((a, b) => (b.seconds < a.seconds ? b : a)), longest = lengths.reduce((a, b) => (b.seconds > a.seconds ? b : a));
-  return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). Make one or two scenes much shorter (a hit of a few words) and let one run long.`;
+  const fix = isVoiceless(plan) ? "Make one or two scenes much shorter (under a second) and let one run long." : "Make one or two scenes much shorter (a hit of a few words) and let one run long.";
+  return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). ${fix}`;
 }
 
 // Soft quality checks on a plan that already passes the hard rules in validatePlan.
 // These are things worth improving in the script; they never fail a video on their own.
 export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
+  if (isVoiceless(plan)) return reviewVoicelessPlan(plan, ctx);
   const issues: string[] = [];
   const { words: total, seconds } = estimateLength(plan);
 
@@ -53,10 +62,61 @@ export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): str
     // A hit of a few words is wanted in the rhythm of a video, so only a scene with almost nothing to say is flagged.
     if (!ctx.footage && n < 3) issues.push(`Scene ${s.id} has only ${n} word${n === 1 ? "" : "s"} of narration; give it at least a short phrase.`);
     if (n > 45) issues.push(`Scene ${s.id} has ${n} words of narration; split it or cut it to under 45.`);
-    if (s.onScreenText.length > 3) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 3.`);
+    if (s.onScreenText.length > 5) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 5 (the items of one list on screen count as one text, but the plan cannot say which they are).`);
+    for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
+  }
+  const rhythm = rhythmNote(plan);
+  if (rhythm) issues.push(rhythm);
+  issues.push(...structureNotes(plan, ctx));
+  return issues;
+}
+
+// The same kind of advice for a video with no voice: no words to count, so length and rhythm come from `seconds`, and every word the
+// viewer reads is on screen. A video of this kind needs at least one quick hit, so a plan with no scene under a second is flagged.
+function reviewVoicelessPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
+  const issues: string[] = [];
+  const { seconds } = estimateLength(plan);
+  if (seconds < 10) issues.push(`The video runs ${seconds.toFixed(1)}s. A video with no voice needs at least 10 seconds to get across; aim for 15 to 45.`);
+  if (seconds > 60) issues.push(`The video runs ${seconds.toFixed(1)}s. Cut it to 60 seconds or less; aim for 15 to 45.`);
+  if (!plan.scenes.some((s) => (s.seconds ?? 0) < 1)) issues.push("No scene is under 1 second. A film like this needs at least one quick hit; shorten one or two scenes to under a second.");
+  if (!ctx.footage) {
+    const illustrations = plan.scenes.filter((s) => s.treatment === "illustration").length;
+    if (illustrations > Math.ceil(plan.scenes.length / 2)) issues.push(`${illustrations} of ${plan.scenes.length} scenes are illustrations. Use illustrations for at most half the scenes.`);
+  }
+  for (const s of plan.scenes) {
+    if (!s.onScreenText.length && !s.notes.trim()) issues.push(`Scene ${s.id} has neither onScreenText nor notes. Say what is on screen, since no voice explains it.`);
+    if (s.onScreenText.length > 5) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 5 (the items of one list on screen count as one text, but the plan cannot say which they are).`);
     for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
   }
   const rhythm = rhythmNote(plan);
   if (rhythm) issues.push(rhythm);
+  issues.push(...structureNotes(plan, ctx));
   return issues;
 }
+
+const hasPicture = (s: ScenePlan["scenes"][number]) => s.treatment === "illustration" || s.treatment === "clip" || s.userAssetIds.length > 0;
+
+// The first sentence of some narration, up to . ! ? … or their Hebrew and Arabic forms.
+export function firstSentence(text: string): string {
+  const m = /^[\s\S]*?[.!?…؟׃。！？]+(?=\s|$)/.exec(text.trim());
+  return (m ? m[0] : text).trim();
+}
+
+// What the plan alone says about whether the video will look like something: pictures, an opening that shows, an opening that gets to the point.
+// Used by `plan check` and, because the composition is built from the plan, by `check` too.
+export function structureNotes(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
+  const notes: string[] = [];
+  if (ctx.footage) return notes;
+  // A film with no voice is made of type and interface pieces on purpose (reference/launch-film.md), so it is not asked for a picture.
+  const voiceless = isVoiceless(plan);
+  if (!voiceless && plan.scenes.length >= 4 && !plan.scenes.some(hasPicture)) {
+    notes.push("Every scene is type and shapes: no illustration, no clip and none of the user's own files. Give at least one scene a picture so the video has something to look at; see reference/scene-treatments.md.");
+  }
+  const first = plan.scenes[0]!;
+  if (!voiceless && !hasPicture(first)) {
+    notes.push(`The opening has no picture, clip or user asset (scene ${first.id}). The first second decides whether the video is watched, so open on something to look at; see reference/scriptwriting.md.`);
+  }
+  const n = isVoiceless(plan) ? 0 : words(firstSentence(first.narration));
+  if (n > 12) notes.push(`The first sentence of the opening is ${n} words. Open with a sentence of 12 words or fewer; see reference/scriptwriting.md.`);
+  return notes;
+}

package/src/pipeline/schema.ts

@@ -7,7 +7,10 @@ export type Aspect = z.infer<typeof AspectSchema>;
 
 export const SceneSchema = z.object({
   id: z.string().regex(/^[a-z0-9-]+$/).describe("short unique id, lowercase letters, digits, dashes"),
-  narration: z.string().min(1).describe("the words spoken in this scene"),
+  // The minimum of one character is enforced for a narrated plan in ScenePlanSchema below, so that a video without a voice may leave it empty.
+  narration: z.string().describe("the words spoken in this scene; an empty string when the plan has voice \"none\""),
+  // How long the scene lasts. Used only when the plan has voice "none"; with a voice the recording sets the length.
+  seconds: z.number().min(0.3).max(15).optional().describe("the scene's length in seconds, from 0.3 to 15; required when the plan has voice \"none\""),
   // clip: a generated or reused video clip is the scene's picture.
   treatment: z.enum(["motion-graphic", "illustration", "footage-overlay", "clip"]),
   onScreenText: z.array(z.string()),
@@ -18,9 +21,14 @@ export const SceneSchema = z.object({
   imageTags: z.array(z.string()).describe("3 to 6 short tags describing the illustration; empty when imagePrompt is null"),
   userAssetIds: z.array(z.string()),
   notes: z.string().describe("visual direction for whoever writes the composition"),
+  // Says that this scene begins on a deliberate hard cut, so `preview` does not count the missing carry-over against the film.
+  cutIn: z.boolean().optional().describe("true when this scene begins on a deliberate hard cut (nothing is meant to carry over into it)"),
 });
 export type Scene = z.infer<typeof SceneSchema>;
 
+export const MAX_NARRATED_SCENES = 8;
+export const MAX_VOICELESS_SCENES = 16;
+
 export const ScenePlanSchema = z.object({
   title: z.string(),
   aspect: AspectSchema,
@@ -28,18 +36,45 @@ export const ScenePlanSchema = z.object({
   // Optional only so plans stored before voices existed still load; a new plan must choose one.
   voiceId: z.string().optional().describe("id of the narration voice, one of the ids from `reelkit assets voices`, chosen to suit the idea, audience and language"),
   pace: z.enum(["slow", "normal", "fast"]).optional().describe("speaking pace: slow for calm or emotional, normal by default, fast for high-energy"),
+  // The silence between one scene's last word and the next scene's first word: tight 0.2 s, normal 0.3 s (the default), relaxed 0.5 s.
+  // (`pace` is the speed of the voice itself, so the gap has its own setting.)
+  gap: z.enum(["tight", "normal", "relaxed"]).optional().describe("silence between one sentence and the next: tight 0.2 s, normal 0.3 s (the default), relaxed 0.5 s; a tight promo is tight"),
+  // Absent means narrated. "none" is a video carried by music and sound effects alone: scenes get their length from `seconds`.
+  voice: z.enum(["narrated", "none"]).optional().describe("narrated (the default), or none for a video with no voice"),
   // Whether the video has captions and of which kind: none, one word at a time, or a phrase at a time. Absent means "phrase".
-  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a phrase at a time (phrase, the default), as the user chose"),
+  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a few words at a time (phrase, the default; a few words at a time is phrase, not word), as the user chose"),
   // A video the new one is made "like": what is taken from it is how it feels (structure, pacing, motion), never its footage, music or words.
   // Optional, so plans stored before references existed still load.
   reference: z.object({
     id: z.string().describe("the id of a reference in this project, from `reelkit ref list`"),
     take: z.array(z.string()).min(1).max(6).describe("short notes on what is taken from it, e.g. \"fast cuts every ~1.2s\", \"big type on colour fields\""),
   }).optional(),
-  scenes: z.array(SceneSchema).min(3).max(8),
+  scenes: z.array(SceneSchema).min(3).max(MAX_VOICELESS_SCENES),
+}).superRefine((plan, ctx) => {
+  if (plan.voice === "none") {
+    plan.scenes.forEach((s, i) => {
+      if (s.seconds === undefined) ctx.addIssue({ code: "custom", path: ["scenes", i, "seconds"], message: `Scene ${s.id} needs seconds (a number from 0.3 to 15) because the video has no voice. Add it, or remove voice from the plan.` });
+    });
+    return;
+  }
+  // The narrated rules, as they were before voice existed.
+  if (plan.scenes.length > MAX_NARRATED_SCENES) ctx.addIssue({ code: "too_big", origin: "array", maximum: MAX_NARRATED_SCENES, inclusive: true, input: plan.scenes, path: ["scenes"] });
+  plan.scenes.forEach((s, i) => {
+    if (s.narration.length < 1) ctx.addIssue({ code: "too_small", origin: "string", minimum: 1, inclusive: true, input: s.narration, path: ["scenes", i, "narration"] });
+  });
 });
 
+export const isVoiceless = (plan: { voice?: "narrated" | "none" }): boolean => plan.voice === "none";
+
 export const PACE_SPEED = { slow: 0.92, normal: 1, fast: 1.1 } as const;
+// The silence, in seconds, between one scene's last word and the next scene's first word, by the plan's `gap`.
+export const GAP_SEC = { tight: 0.2, normal: 0.3, relaxed: 0.5 } as const;
+export const gapSec = (plan: { gap?: keyof typeof GAP_SEC }): number => GAP_SEC[plan.gap ?? "normal"];
+// The last scene keeps this long after its last word, so that the film does not end on the last syllable.
+export const LAST_TAIL_SEC = 0.7;
+// The voice speaks about this many words a second at `pace: "normal"` (measured on a recorded film: 2.0 to 2.4, 2.25 over the whole of it, the
+// pauses inside a sentence included). `plan check` estimates the length from it.
+export const VOICE_WORDS_PER_SEC = 2.25;
 export type ScenePlan = z.infer<typeof ScenePlanSchema>;
 
 export const AssetRecordSchema = z.object({
@@ -76,6 +111,16 @@ export const ManifestSceneSchema = z.object({
   clipKey: z.string().optional(),
   clipKeyedKey: z.string().optional(),
   userAssetKeys: z.array(z.string()),
+  // The scene's picture in layers, from `reelkit assets layers`: the picture itself, the subject cut out of it when it has a clear one (a PNG with alpha beside it) and its box,
+  // and where words can sit (`region`, how light and how busy). Pass it to <ImageLayers layers={...}> and <TextOnImage layers={...}>.
+  imageLayers: z.object({
+    back: z.string(),
+    subject: z.string().optional(),
+    subjectBox: z.object({ x: z.number(), y: z.number(), w: z.number(), h: z.number() }).optional(),
+    textZone: z.object({ region: z.enum(["top", "middle", "bottom", "left", "right"]), luminance: z.number(), busy: z.number() }),
+  }).optional(),
+  // The picture chosen as this scene's ground, with `reelkit assets pull <id> --background --scene <id>`.
+  background: z.object({ key: z.string() }).optional(),
 });
 export type ManifestScene = z.infer<typeof ManifestSceneSchema>;
 
@@ -91,6 +136,8 @@ export const AssetManifestSchema = z.object({
   captions: z.enum(["none", "word", "phrase"]).optional(),
   // Linear gain per sound file path that brings every pulled sound to a common level; the kit's Sfx and Music apply it.
   soundGain: z.record(z.string(), z.number()).optional(),
+  // The film's ground (a video or a picture), chosen with `--background`; durationSec is set for a video.
+  background: z.object({ key: z.string(), durationSec: z.number().optional() }).optional(),
   scenes: z.array(ManifestSceneSchema),
 });
 export type AssetManifest = z.infer<typeof AssetManifestSchema>;
@@ -102,7 +149,7 @@ export function validatePlan(
   ctx: { aspect: Aspect; footage?: AssetRecord; assets: AssetRecord[]; voiceIds?: string[] },
 ): string[] {
   const errors: string[] = [];
-  if (ctx.voiceIds) {
+  if (ctx.voiceIds && !isVoiceless(plan)) {
     if (!plan.voiceId) errors.push("Choose a narration voice: set voiceId to one of the ids from `reelkit assets voices`.");
     else if (!ctx.voiceIds.includes(plan.voiceId)) errors.push(`voiceId ${plan.voiceId} is not one of the available voices.`);
   }

package/src/pipeline/timing.ts

@@ -1,4 +1,4 @@
-import type { Aspect, AssetRecord } from "./schema";
+import { LAST_TAIL_SEC, type Aspect, type AssetRecord, type WordTiming } from "./schema";
 
 export const FPS = 30;
 export const SCENE_PADDING_SEC = 0.4;
@@ -28,3 +28,29 @@ export function dimensionsFor(aspect: Aspect, footage?: AssetRecord) {
   if (aspect === "1:1") return { width: 1080, height: 1080 };
   return { width: 1080, height: 1920 };
 }
+
+// A narrated scene's audio: when its last word ends (seconds from its start) and when its first word starts. A recording with no word times
+// counts as one stretch of speech as long as the file.
+export type SpokenScene = { durationSec: number; words: WordTiming[] };
+export const lastWordEndSec = (v: SpokenScene): number => (v.words.length ? Math.max(...v.words.map((w) => w.endSec)) : v.durationSec);
+export const firstWordStartSec = (v: SpokenScene): number => (v.words.length ? Math.min(...v.words.map((w) => w.startSec)) : 0);
+
+// The boundary never comes closer than this after a scene's last word, however tight the gap is.
+export const MIN_AFTER_WORD_SEC = 0.1;
+
+// Lays narrated scenes end to end so that the silence between one scene's last word and the next scene's first word is `gapSec`. Each scene's
+// audio starts on its first frame, so a scene lasts until its last word has ended plus the gap, less the silence its successor opens with. The
+// last scene keeps `tailSec` after its last word. Returns the natural lengths, before any snapping to the beat.
+export function layoutNarrated(scenes: SpokenScene[], gapSec: number, fps = FPS, tailSec = LAST_TAIL_SEC) {
+  let cursor = 0;
+  const out = scenes.map((v, i) => {
+    const end = lastWordEndSec(v);
+    const next = scenes[i + 1];
+    const seconds = next ? Math.max(end + gapSec - firstWordStartSec(next), end + MIN_AFTER_WORD_SEC) : end + tailSec;
+    const durationFrames = secondsToFrames(seconds, fps);
+    const scene = { startFrame: cursor, durationFrames };
+    cursor += durationFrames;
+    return scene;
+  });
+  return { scenes: out, totalFrames: cursor };
+}

package/src/project/background.ts

@@ -0,0 +1,33 @@
+import { FILES, type Project } from "./project";
+
+// The film's ground, when one was chosen: a video or a picture for the whole film, and pictures for single scenes (a video is only ever for the film).
+// Stored in assets/background.json and shown to the composition as manifest.background and manifest.scenes[i].background.
+export type BackgroundEntry = { key: string; id?: string; title?: string; durationSec?: number };
+export type BackgroundFile = { film?: BackgroundEntry; scenes: Record<string, BackgroundEntry> };
+
+export const readBackground = (project: Project): BackgroundFile => {
+  const raw = project.readJsonOr<Partial<BackgroundFile>>(FILES.background, {});
+  return { ...(raw.film ? { film: raw.film } : {}), scenes: raw.scenes ?? {} };
+};
+
+// One ground for the film: setting another replaces it. A scene's picture is set the same way, by scene id.
+export function setBackground(project: Project, entry: BackgroundEntry, sceneId?: string): BackgroundFile {
+  const current = readBackground(project);
+  const next: BackgroundFile = sceneId ? { ...current, scenes: { ...current.scenes, [sceneId]: entry } } : { ...current, film: entry };
+  project.writeJson(FILES.background, next);
+  return next;
+}
+
+export const VIDEO_EXT = /\.(mp4|mov|webm|m4v)$/i;
+export const isVideoPath = (path: string): boolean => VIDEO_EXT.test(path);
+
+// What is added to a prompt for a video or picture meant to sit behind everything, so that it stays calm and leaves room for type.
+export const BACKGROUND_CLIP_SENTENCE = "An abstract, slow, seamless ambient background with no people, no objects in focus, no text and no logos, soft and low in contrast so text stays readable on top.";
+export const BACKGROUND_IMAGE_SENTENCE = "An abstract, soft, low-contrast background with no people, no text and no logos, with empty space for type on top.";
+export const DEFAULT_BACKGROUND_LOOK = "soft, slowly shifting colour fields in a muted dark palette";
+
+// A look for a background that was asked for with no prompt: the colours the first scene's notes name, when they name any, else a neutral default.
+export function lookFromNotes(notes: string | undefined): string {
+  const named = [...new Set((notes ?? "").toLowerCase().match(/\b(?:black|white|navy|blue|teal|cyan|green|lime|yellow|amber|orange|red|crimson|pink|magenta|purple|violet|indigo|brown|beige|cream|grey|gray|gold|silver)\b|#[0-9a-f]{6}\b/g) ?? [])].slice(0, 4);
+  return named.length ? `soft, slowly shifting colour fields in ${named.join(", ")}` : DEFAULT_BACKGROUND_LOOK;
+}

package/src/project/layers.ts

@@ -0,0 +1,60 @@
+import { execFile } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { coverageOf, subjectBoxOf, textZoneOf, type ImageLayersRecord, type Rect, type TextZone } from "../remotion/kit/image-layers-math";
+
+const run = promisify(execFile);
+const GRID = 48;
+
+// The picture's size is capped, so that the 1-second video a cutout is made from stays small.
+const MAX_SIDE = 1920;
+
+async function raw(args: string[]): Promise<Buffer> {
+  try {
+    const { stdout } = await run("ffmpeg", ["-v", "error", ...args], { encoding: "buffer", maxBuffer: 1 << 24 });
+    return stdout;
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException)?.code === "ENOENT") throw new Error("ffmpeg was not found. Install ffmpeg (macOS: `brew install ffmpeg`) and run the command again.");
+    throw new Error(`ffmpeg could not read the picture: ${e instanceof Error ? e.message.split("\n")[0] : String(e)}`);
+  }
+}
+
+// A picture as a GRID by GRID grid of grey values.
+export async function greyGrid(image: string): Promise<Uint8Array> {
+  return new Uint8Array(await raw(["-i", image, "-frames:v", "1", "-vf", `scale=${GRID}:${GRID}:flags=area,format=gray`, "-f", "rawvideo", "-"]));
+}
+// The alpha channel of a picture as a GRID by GRID grid (255 opaque), 255 everywhere when it has none.
+export async function alphaGrid(image: string): Promise<Uint8Array> {
+  return new Uint8Array(await raw(["-i", image, "-frames:v", "1", "-vf", `format=rgba,alphaextract,scale=${GRID}:${GRID}:flags=area,format=gray`, "-f", "rawvideo", "-"]));
+}
+
+// Where words can sit in this picture: the calmest region, how light it is and how busy. `subject` is the subject's box when there is one.
+export async function measureTextZone(image: string, subject?: Rect): Promise<TextZone> {
+  return textZoneOf(await greyGrid(image), GRID, GRID, subject);
+}
+
+// The one-second video a still is sent to the cutout service as: 2 frames a second, even sides, never over 1920 on the long side.
+export async function stillToVideo(image: string, out: string): Promise<void> {
+  const vf = `scale='min(iw,${MAX_SIDE})':'min(ih,${MAX_SIDE})':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p`;
+  await raw(["-y", "-loop", "1", "-framerate", "2", "-i", image, "-t", "1", "-vf", vf, "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-an", out]);
+}
+
+// One frame of the transparent video the service returns, as a PNG with alpha: VP9 with alpha needs the libvpx decoder, so it is asked for by name.
+// Throws when the PNG has no transparent pixel at all (the alpha did not survive), and returns what the subject covers and the box that holds it.
+export async function frameWithAlpha(webm: string, png: string): Promise<{ coverage: number; box?: Rect }> {
+  await raw(["-y", "-c:v", "libvpx-vp9", "-i", webm, "-frames:v", "1", "-pix_fmt", "rgba", png]);
+  const a = await alphaGrid(png);
+  if (!a.some((v) => v < 250)) throw new Error("The cut-out came back without any transparent pixel, so there is no subject layer to make. The picture stays flat.");
+  return { coverage: coverageOf(a), box: subjectBoxOf(a, GRID, GRID) };
+}
+
+export const tempDir = (): { dir: string; done: () => void } => {
+  const dir = mkdtempSync(join(tmpdir(), "rk-layers-"));
+  return { dir, done: () => rmSync(dir, { recursive: true, force: true }) };
+};
+
+export type LayerEntry = { subject?: string; subjectBox?: Rect; textZone: TextZone; noSubject?: boolean; coverage?: number; cutoutId?: string };
+// The record the manifest carries for a picture: its key as `back`, the subject's file and box when there is one, and where text can sit.
+export const toImageLayers = (key: string, e: LayerEntry): ImageLayersRecord => ({ back: key, ...(e.subject && !e.noSubject ? { subject: e.subject, ...(e.subjectBox ? { subjectBox: e.subjectBox } : {}) } : {}), textZone: e.textZone });