reelkit-cli 0.4.0 → 0.6.0

package/src/contract/index.ts

@@ -13,7 +13,12 @@
 // - A voiceover `text` must contain at least one non-space character: whitespace alone is 400 invalid_request.
 // - A library item `id` (pull, commit) matches `^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`; a `deviceCode` is 1 to 200 characters of plain text.
 // - A `voiceId` is 1 to 64 letters and digits and must be one of the voices the server lists (`voices`), otherwise 400 invalid_request.
-// - Components are added to the library only by the owner's scripts, never through this API (UploadKindSchema has no `component`).
+// - A user's component is uploaded for review through the same two calls as any other file (`kind: "component"`). The server checks its source
+//   at commit with the same static check a pulled component gets (only react, remotion and reelkit/kit imports, no network, no globals that reach
+//   outside the video) and refuses a source that fails it with 400 `invalid_request` and the first problem. It stays in `review`, is never found
+//   by search or the public listing, and is published only by the owner. A component upload is: `contentType` `text/plain`, `bytes` up to 65,536,
+//   `shareable` true, `filename` matching `^[A-Z][A-Za-z0-9]*\.tsx$`, a `description` of at least 20 characters, and `meta.example` a string of at most
+//   2,000 characters that starts with `<` and the file's name (`<StatCard value="42" />` for StatCard.tsx). Anything else is 400 `invalid_request`.
 // - Upload is a single `PUT` to `uploadUrl` with exactly the declared content type and byte length, and no auth header.
 //   Anything else is refused with a non-2xx status and stores nothing. That refusal comes from the storage service, so its body
 //   is not an API error and a client must not parse it.
@@ -105,6 +110,11 @@ const Meta = z.record(z.string(), z.unknown()).refine((m) => new TextEncoder().e
 
 // The most a single upload may be: 200 MB.
 export const MAX_UPLOAD_BYTES = 209_715_200;
+// A component's source is at most 64 KB, its file name is PascalCase, and its description and example have the sizes below.
+export const MAX_COMPONENT_BYTES = 65_536;
+export const COMPONENT_FILE = /^[A-Z][A-Za-z0-9]*\.tsx$/;
+export const MIN_COMPONENT_DESCRIPTION = 20;
+export const MAX_COMPONENT_EXAMPLE = 2000;
 // The video types a cutout takes.
 export const CutoutTypeSchema = z.enum(["video/mp4", "video/quicktime", "video/webm"]);
 export type CutoutType = z.infer<typeof CutoutTypeSchema>;
@@ -124,8 +134,8 @@ export const ApiErrorSchema = z.object({ error: z.object({ code: z.string(), mes
 
 export const LibraryKindSchema = z.enum(["image", "overlay", "sfx", "music", "component", "clip"]);
 export type LibraryKind = z.infer<typeof LibraryKindSchema>;
-// What a user may upload: everything except components, which are code and come only from Reelkit.
-export const UploadKindSchema = LibraryKindSchema.exclude(["component"]);
+// What a user may upload: every kind. A component is code and is held for review (see the rules at the top).
+export const UploadKindSchema = LibraryKindSchema;
 export type UploadKind = z.infer<typeof UploadKindSchema>;
 
 export const LibraryItemSchema = z.object({
@@ -183,6 +193,16 @@ export const routes = {
       kind: UploadKindSchema, title: text(200).min(1), description: text(2000), tags: z.array(text(40)).max(20),
       meta: Meta, filename: text(200).min(1), contentType: text(100), bytes: z.number().int().positive().max(MAX_UPLOAD_BYTES),
       shareable: z.boolean(),
+    }).superRefine((r, ctx) => {
+      if (r.kind !== "component") return;
+      const bad = (path: string, message: string) => ctx.addIssue({ code: "custom", path: [path], message });
+      if (r.contentType.toLowerCase() !== "text/plain") bad("contentType", "a component must be text/plain");
+      if (r.bytes > MAX_COMPONENT_BYTES) bad("bytes", `a component is at most ${MAX_COMPONENT_BYTES} bytes`);
+      if (!COMPONENT_FILE.test(r.filename)) bad("filename", "a component file is named like StatCard.tsx");
+      if (!r.shareable) bad("shareable", "a component is only uploaded to be shared for review");
+      if (r.description.trim().length < MIN_COMPONENT_DESCRIPTION) bad("description", `a component needs a description of at least ${MIN_COMPONENT_DESCRIPTION} characters`);
+      const example = r.meta.example;
+      if (typeof example !== "string" || example.length > MAX_COMPONENT_EXAMPLE || !example.startsWith(`<${r.filename.replace(/\.tsx$/, "")}`)) bad("meta.example", `a component needs meta.example, at most ${MAX_COMPONENT_EXAMPLE} characters, that starts with <Name`);
     }),
     z.object({ id: z.string(), uploadUrl: z.string() })),
   libraryCommit: route("POST", "/library/upload/commit", true, z.object({ id: ItemId }), z.object({ item: LibraryItemSchema })),

package/src/pipeline/beatsnap.ts

@@ -0,0 +1,52 @@
+// Puts scene changes on the beat. A scene is held a little longer at its end so that the next one starts exactly on a beat.
+// The voiceover and its word times never move: only a scene's length, and so where the later scenes start, changes.
+
+// The last scene must keep at least this long after its last word, as the natural padding already does (0.4 s) plus a little.
+export const MIN_TAIL_SEC = 0.5;
+
+export type SnapInput = {
+  // Each scene's natural length in frames.
+  naturalFrames: number[];
+  // When the last word of the last scene ends, in frames from the start of that scene.
+  lastWordEndFrame: number;
+  // Beats as frame numbers, ascending. They should reach past the end of the video.
+  beatFrames: number[];
+  fps: number;
+};
+
+const firstAtOrAfter = (beats: number[], frame: number): number | undefined => beats.find((b) => b >= frame);
+
+// The new length of every scene. A scene is only ever made longer, by less than one beat (the last by a few frames more, to keep its tail).
+// A boundary that has no beat close enough keeps its natural place, and the scenes after it follow from there.
+export function snapToBeats(input: SnapInput): number[] {
+  const { naturalFrames, beatFrames, fps } = input;
+  if (beatFrames.length < 2) return [...naturalFrames];
+  const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
+  const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
+  let cursor = 0;
+  return naturalFrames.map((natural, i) => {
+    const last = i === naturalFrames.length - 1;
+    let end = cursor + natural;
+    let limit = period;
+    if (last) {
+      const tail = cursor + input.lastWordEndFrame + Math.ceil(MIN_TAIL_SEC * fps);
+      if (tail > end) { limit += tail - end; end = tail; }
+    }
+    const beat = firstAtOrAfter(beatFrames, end);
+    const target = beat !== undefined && beat - (cursor + natural) <= limit ? beat : end;
+    const frames = target - cursor;
+    cursor = target;
+    return frames;
+  });
+}
+
+// Beats in seconds, continued at the tempo's own spacing until `untilSec`, for a video longer than the track (the track loops).
+export function extendBeats(beats: number[], bpm: number, untilSec: number): number[] {
+  if (!beats.length) return [];
+  const period = 60 / bpm;
+  const out = [...beats];
+  while (out[out.length - 1]! < untilSec) out.push(Math.round((out[out.length - 1]! + period) * 1000) / 1000);
+  return out;
+}
+
+export const toBeatFrames = (beatsSec: number[], fps: number): number[] => [...new Set(beatsSec.map((t) => Math.round(t * fps)))].sort((a, b) => a - b);

package/src/pipeline/review.ts

@@ -8,6 +8,30 @@ export function estimateLength(plan: ScenePlan): { words: number; seconds: numbe
   return { words: total, seconds: total / WORDS_PER_SEC };
 }
 
+// Each scene's estimated length in seconds, from its narration at the same pace as the whole-video estimate above.
+export function sceneSeconds(plan: ScenePlan): { id: string; seconds: number }[] {
+  return plan.scenes.map((s) => ({ id: s.id, seconds: words(s.narration) / WORDS_PER_SEC }));
+}
+
+// How uneven the scene lengths are: the standard deviation over the mean (0 when they are all the same).
+export function lengthVariation(seconds: number[]): number {
+  const mean = seconds.reduce((a, b) => a + b, 0) / (seconds.length || 1);
+  if (!(mean > 0)) return 0;
+  return Math.sqrt(seconds.reduce((a, b) => a + (b - mean) ** 2, 0) / seconds.length) / mean;
+}
+
+// A video whose scenes all last about as long feels like a slideshow. With four scenes or more, the lengths should differ a lot.
+export function rhythmNote(plan: ScenePlan): string | undefined {
+  if (plan.scenes.length < 4) return undefined;
+  const lengths = sceneSeconds(plan);
+  const values = lengths.map((l) => l.seconds);
+  const mean = values.reduce((a, b) => a + b, 0) / values.length;
+  const cv = lengthVariation(values);
+  if (cv >= 0.25 && values.some((v) => v < mean / 2)) return undefined;
+  const shortest = lengths.reduce((a, b) => (b.seconds < a.seconds ? b : a)), longest = lengths.reduce((a, b) => (b.seconds > a.seconds ? b : a));
+  return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). Make one or two scenes much shorter (a hit of a few words) and let one run long.`;
+}
+
 // Soft quality checks on a plan that already passes the hard rules in validatePlan.
 // These are things worth improving in the script; they never fail a video on their own.
 export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
@@ -26,10 +50,39 @@ export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): str
 
   for (const s of plan.scenes) {
     const n = words(s.narration);
-    if (!ctx.footage && n < 5) issues.push(`Scene ${s.id} has only ${n} words of narration; give it one full sentence.`);
+    // A hit of a few words is wanted in the rhythm of a video, so only a scene with almost nothing to say is flagged.
+    if (!ctx.footage && n < 3) issues.push(`Scene ${s.id} has only ${n} word${n === 1 ? "" : "s"} of narration; give it at least a short phrase.`);
     if (n > 45) issues.push(`Scene ${s.id} has ${n} words of narration; split it or cut it to under 45.`);
     if (s.onScreenText.length > 3) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 3.`);
     for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
   }
+  const rhythm = rhythmNote(plan);
+  if (rhythm) issues.push(rhythm);
+  issues.push(...structureNotes(plan, ctx));
   return issues;
 }
+
+const hasPicture = (s: ScenePlan["scenes"][number]) => s.treatment === "illustration" || s.treatment === "clip" || s.userAssetIds.length > 0;
+
+// The first sentence of some narration, up to . ! ? … or their Hebrew and Arabic forms.
+export function firstSentence(text: string): string {
+  const m = /^[\s\S]*?[.!?…؟׃。！？]+(?=\s|$)/.exec(text.trim());
+  return (m ? m[0] : text).trim();
+}
+
+// What the plan alone says about whether the video will look like something: pictures, an opening that shows, an opening that gets to the point.
+// Used by `plan check` and, because the composition is built from the plan, by `check` too.
+export function structureNotes(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
+  const notes: string[] = [];
+  if (ctx.footage) return notes;
+  if (plan.scenes.length >= 4 && !plan.scenes.some(hasPicture)) {
+    notes.push("Every scene is type and shapes: no illustration, no clip and none of the user's own files. Give at least one scene a picture so the video has something to look at; see reference/scene-treatments.md.");
+  }
+  const first = plan.scenes[0]!;
+  if (!hasPicture(first)) {
+    notes.push(`The opening has no picture, clip or user asset (scene ${first.id}). The first second decides whether the video is watched, so open on something to look at; see reference/scriptwriting.md.`);
+  }
+  const n = words(firstSentence(first.narration));
+  if (n > 12) notes.push(`The first sentence of the opening is ${n} words. Open with a sentence of 12 words or fewer; see reference/scriptwriting.md.`);
+  return notes;
+}

package/src/pipeline/schema.ts

@@ -28,6 +28,8 @@ export const ScenePlanSchema = z.object({
   // Optional only so plans stored before voices existed still load; a new plan must choose one.
   voiceId: z.string().optional().describe("id of the narration voice, one of the ids from `reelkit assets voices`, chosen to suit the idea, audience and language"),
   pace: z.enum(["slow", "normal", "fast"]).optional().describe("speaking pace: slow for calm or emotional, normal by default, fast for high-energy"),
+  // Whether the video has captions and of which kind: none, one word at a time, or a phrase at a time. Absent means "phrase".
+  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a phrase at a time (phrase, the default), as the user chose"),
   // A video the new one is made "like": what is taken from it is how it feels (structure, pacing, motion), never its footage, music or words.
   // Optional, so plans stored before references existed still load.
   reference: z.object({
@@ -83,6 +85,12 @@ export const AssetManifestSchema = z.object({
   height: z.number(),
   totalFrames: z.number(),
   footageKey: z.string().optional(),
+  // The video's music track, when one was pulled with `reelkit assets pull <id> --music`. beatFrames are the beats that fall inside the video.
+  music: z.object({ key: z.string(), bpm: z.number().optional(), beatFrames: z.array(z.number()) }).optional(),
+  // The plan's caption choice, with the default filled in, so the composition can pass it to <Captions group={...}>.
+  captions: z.enum(["none", "word", "phrase"]).optional(),
+  // Linear gain per sound file path that brings every pulled sound to a common level; the kit's Sfx and Music apply it.
+  soundGain: z.record(z.string(), z.number()).optional(),
   scenes: z.array(ManifestSceneSchema),
 });
 export type AssetManifest = z.infer<typeof AssetManifestSchema>;

package/src/project/loudness.ts

@@ -0,0 +1,68 @@
+import { audioDuration, tool } from "./refmeasure";
+
+// How loud a sound is, measured on this machine, and how much to turn it up or down so that the same `volume` number sounds equally loud
+// for every file.
+//   A one-shot (a sound effect under 3 s) is levelled by its loudest moment: its peak is brought to -3 dBFS.
+//   A longer sound (music, a long effect) is levelled by how loud it is overall: about -18 LUFS, integrated.
+// The gain is limited to 18 dB either way, so a nearly silent or a broken file is not blown up into noise.
+export const ONE_SHOT_SECONDS = 3;
+export const ONE_SHOT_PEAK_DB = -3;
+export const LONG_TARGET_LUFS = -18;
+export const MAX_GAIN_DB = 18;
+
+export type SoundLevel = { durationSec: number; peakDb?: number; lufs?: number };
+
+const round = (n: number, digits: number) => Math.round(n * 10 ** digits) / 10 ** digits;
+
+// The gain in dB that brings a measured sound to its target, or 0 when it could not be measured (silence).
+export function gainDbFor(level: SoundLevel): number {
+  const oneShot = level.durationSec < ONE_SHOT_SECONDS;
+  const measured = oneShot ? level.peakDb : level.lufs ?? level.peakDb;
+  if (measured === undefined || !Number.isFinite(measured)) return 0;
+  const target = oneShot || level.lufs === undefined ? ONE_SHOT_PEAK_DB : LONG_TARGET_LUFS;
+  return round(Math.max(-MAX_GAIN_DB, Math.min(MAX_GAIN_DB, target - measured)), 1);
+}
+
+export const dbToLinear = (db: number) => round(10 ** (db / 20), 4);
+
+// ffmpeg's volumedetect: the loudest sample, in dBFS. Undefined for digital silence.
+export async function peakOf(path: string): Promise<number | undefined> {
+  const { stderr } = await tool("ffmpeg", ["-hide_banner", "-nostats", "-i", path, "-vn", "-af", "volumedetect", "-f", "null", "-"]);
+  const peak = Number(/max_volume:\s*(-?[0-9.]+)\s*dB/.exec(stderr)?.[1]);
+  return Number.isFinite(peak) ? peak : undefined;
+}
+
+// What loudnorm's first pass reports; its numbers are text in the JSON it prints, and "-inf" for silence.
+export type LoudnormReport = { input_i: string; input_tp: string; input_lra: string; input_thresh: string; target_offset: string };
+
+export function parseLoudnorm(stderr: string): LoudnormReport | undefined {
+  const end = stderr.lastIndexOf("}");
+  const start = stderr.lastIndexOf("{", end);
+  if (start < 0 || end < start) return undefined;
+  try {
+    const r = JSON.parse(stderr.slice(start, end + 1)) as LoudnormReport;
+    return typeof r.input_i === "string" ? r : undefined;
+  } catch { return undefined; }
+}
+
+export async function measureLoudnorm(path: string, target: { i: number; tp: number }): Promise<LoudnormReport | undefined> {
+  const { stderr } = await tool("ffmpeg", ["-hide_banner", "-nostats", "-i", path, "-vn", "-af", `loudnorm=I=${target.i}:TP=${target.tp}:LRA=11:print_format=json`, "-f", "null", "-"]);
+  return parseLoudnorm(stderr);
+}
+
+// Measures a sound for levelling. A failure to measure is not the caller's failure: it returns undefined and the sound is used as it is.
+export async function levelOf(path: string): Promise<(SoundLevel & { gainDb: number }) | undefined> {
+  try {
+    const durationSec = await audioDuration(path);
+    if (!(durationSec > 0)) return undefined;
+    const peakDb = await peakOf(path);
+    let lufs: number | undefined;
+    if (durationSec >= ONE_SHOT_SECONDS) {
+      const r = await measureLoudnorm(path, { i: LONG_TARGET_LUFS, tp: -1.5 });
+      const v = Number(r?.input_i);
+      if (Number.isFinite(v)) lufs = round(v, 1);
+    }
+    const level: SoundLevel = { durationSec, ...(peakDb !== undefined ? { peakDb } : {}), ...(lufs !== undefined ? { lufs } : {}) };
+    return { ...level, gainDb: gainDbFor(level) };
+  } catch { return undefined; }
+}

package/src/project/manifest.ts

@@ -1,5 +1,8 @@
 import { PACE_SPEED, type AssetManifest, type ScenePlan, type WordTiming } from "../pipeline/schema";
+import { extendBeats, snapToBeats, toBeatFrames } from "../pipeline/beatsnap";
 import { dimensionsFor, FPS, layoutScenes } from "../pipeline/timing";
+import { dbToLinear } from "./loudness";
+import type { MusicRecord } from "./music";
 import { FILES, type Project } from "./project";
 
 // What a scene was recorded from is stored with it, so a changed script or voice is noticed.
@@ -24,7 +27,22 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
   const footage = project.footage();
   const byId = new Map(project.assets().map((a) => [a.id, a]));
 
-  const layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  let layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  let beatFrames: number[] = [];
+  if (music) {
+    beatFrames = toBeatFrames(extendBeats(music.beats, music.bpm ?? 120, layout.totalFrames / FPS + 10), FPS);
+    // Scene changes land on beats only when the track has a trusted tempo. Footage has a fixed length that a longer scene must not overrun.
+    if (music.bpm && music.beats.length) {
+      const last = voiceovers[plan.scenes[plan.scenes.length - 1].id];
+      const lastWords = last.words.length ? Math.max(...last.words.map((w) => w.endSec)) : last.durationSec;
+      const lengths = snapToBeats({ naturalFrames: layout.scenes.map((s) => s.durationFrames), lastWordEndFrame: Math.ceil(lastWords * FPS), beatFrames, fps: FPS });
+      let cursor = 0;
+      const snapped = { scenes: lengths.map((durationFrames) => { const r = { startFrame: cursor, durationFrames }; cursor += durationFrames; return r; }), totalFrames: 0 };
+      snapped.totalFrames = cursor;
+      if (!footage || cursor <= Math.floor((footage.durationSec ?? 0) * FPS)) layout = snapped;
+    }
+  }
   let totalFrames = layout.totalFrames;
   if (footage) {
     const footageFrames = Math.floor((footage.durationSec ?? 0) * FPS);
@@ -34,11 +52,21 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
     totalFrames = footageFrames;
   }
 
+  // The levelling of every sound that was measured when it was pulled, as linear gains by path.
+  const soundGain: Record<string, number> = {};
+  for (const e of Object.values(project.readJsonOr<Record<string, { path: string; kind: string; gainDb?: number }>>(FILES.library, {}))) {
+    if ((e.kind === "sfx" || e.kind === "music") && typeof e.gainDb === "number") soundGain[e.path] = dbToLinear(e.gainDb);
+  }
+  if (music) soundGain[music.key] = dbToLinear(music.gainDb);
+
   const manifest: AssetManifest = {
     fps: FPS,
     ...dimensionsFor(plan.aspect, footage),
     totalFrames,
+    captions: plan.captions ?? "phrase",
+    ...(Object.keys(soundGain).length ? { soundGain } : {}),
     ...(footage ? { footageKey: footage.key } : {}),
+    ...(music ? { music: { key: music.key, ...(music.bpm ? { bpm: music.bpm } : {}), beatFrames: music.bpm ? beatFrames.filter((f) => f < totalFrames) : [] } } : {}),
     scenes: plan.scenes.map((scene, i) => ({
       id: scene.id,
       startFrame: layout.scenes[i].startFrame,
@@ -64,6 +92,8 @@ export function missingAssets(project: Project, plan: ScenePlan): string[] {
   const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
   const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
   const problems: string[] = [];
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  if (music && !project.exists(music.key)) problems.push(`Missing file: ${music.key}. Pull the track again with \`reelkit assets pull ${music.id} --music\`.`);
   for (const s of plan.scenes) {
     if (!voiceovers[s.id]) problems.push(`Scene ${s.id} has no voiceover.`);
     else if (voiceoverStale(voiceovers[s.id], s, plan)) problems.push(`Scene ${s.id}'s voiceover is out of date. Run \`reelkit assets voiceover --all\`.`);

package/src/project/music.ts

@@ -0,0 +1,25 @@
+import type { AssetManifest } from "../pipeline/schema";
+import { analyzeBeats } from "./beats";
+import { audioDuration, decodeMono } from "./refmeasure";
+
+// The video's one music track, stored in assets/music.json. `bpm` and `beats` are present only when the pulse was clear enough to trust.
+export type MusicRecord = { key: string; id: string; title: string; durationSec: number; bpm?: number; beatConfidence: number; beats: number[]; gainDb: number };
+
+const RATE = 11025;
+
+// Measures a pulled track on this machine: its length, its tempo and where its beats fall.
+export async function measureMusic(path: string): Promise<Pick<MusicRecord, "durationSec" | "bpm" | "beatConfidence" | "beats">> {
+  const durationSec = Math.round((await audioDuration(path)) * 1000) / 1000;
+  const r = analyzeBeats(await decodeMono(path, RATE), RATE);
+  return { durationSec, beatConfidence: r.confidence, beats: r.tempoBpm !== undefined && r.beats ? r.beats : [], ...(r.tempoBpm !== undefined ? { bpm: r.tempoBpm } : {}) };
+}
+
+// How many scene changes fall on a beat, for the preview.
+export function beatReport(manifest: AssetManifest): { bpm: number; boundariesOnBeat: number; boundaries: number; line: string } | undefined {
+  const m = manifest.music;
+  if (!m?.bpm || manifest.scenes.length < 2) return undefined;
+  const beats = new Set(m.beatFrames);
+  const starts = manifest.scenes.slice(1).map((s) => s.startFrame);
+  const on = starts.filter((f) => beats.has(f)).length;
+  return { bpm: m.bpm, boundariesOnBeat: on, boundaries: starts.length, line: `${on} of ${starts.length} scene changes land on the beat at ${Math.round(m.bpm)} BPM` };
+}

package/src/project/project.ts

@@ -5,14 +5,16 @@ import { AspectSchema, type AssetRecord } from "../pipeline/schema";
 
 export const FILES = {
   config: "reelkit.json", plan: "plan.json", manifest: "manifest.json",
-  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json",
+  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json", music: "assets/music.json", searches: "assets/searches.json", shared: "assets/shared.json",
 } as const;
 
 // "path: message", or just the message for a problem at the root of the file.
 export const issueLines = (e: { issues: { path: PropertyKey[]; message: string }[] }) =>
   e.issues.map((i) => (i.path.length ? `${i.path.map(String).join(".")}: ${i.message}` : i.message));
 
-export const ProjectConfigSchema = z.object({ aspect: AspectSchema, name: z.string().optional(), footage: z.string().optional() });
+export const ProjectConfigSchema = z.object({ aspect: AspectSchema, name: z.string().optional(), footage: z.string().optional(),
+  // false when the project was made with --private: nothing is shared with the library automatically.
+  shareComponents: z.boolean().optional() });
 export type ProjectConfig = z.infer<typeof ProjectConfigSchema>;
 
 // One video's working folder. Every command reads and writes its state here.

package/src/project/refmeasure.ts

@@ -9,7 +9,7 @@ const exec = promisify(execFile);
 export const REF_ASPECTS = ["9:16", "16:9", "1:1", "4:5"] as const;
 export type RefAspect = (typeof REF_ASPECTS)[number];
 
-async function tool(cmd: "ffmpeg" | "ffprobe", args: string[]): Promise<{ stdout: Buffer; stderr: string }> {
+export async function tool(cmd: "ffmpeg" | "ffprobe", args: string[]): Promise<{ stdout: Buffer; stderr: string }> {
   try {
     const r = await exec(cmd, args, { maxBuffer: 256 * 1024 * 1024, encoding: "buffer" });
     return { stdout: r.stdout, stderr: r.stderr.toString("utf8") };
@@ -146,3 +146,47 @@ export async function audioDuration(path: string): Promise<number> {
   const { stdout } = await tool("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path]);
   return Number(stdout.toString("utf8").trim());
 }
+
+// The mean absolute difference, from 0 to 255, between each frame of a run of grey frames and the next. frameSize is the pixels in one frame.
+export function meanAbsDiffs(frames: Uint8Array, frameSize: number): number[] {
+  const count = Math.floor(frames.length / frameSize);
+  const out: number[] = [];
+  for (let f = 0; f + 1 < count; f++) {
+    let sum = 0;
+    for (let i = 0, a = f * frameSize, b = a + frameSize; i < frameSize; i++) sum += Math.abs(frames[a + i]! - frames[b + i]!);
+    out.push(sum / frameSize);
+  }
+  return out;
+}
+
+// Two frames whose pictures differ by less than this, on average over 255 grey levels, are the same picture for a viewer: video
+// compression noise on a held shot stays well under it, and a slow push still stays above it.
+const STILL_BELOW = 1;
+
+// The share of the video's time in which the picture barely changes from one sampled frame to the next, and the longest unbroken stretch
+// of it. diffs has one entry for each pair of consecutive samples, taken `rate` times a second.
+export function stillness(diffs: number[], rate: number): { share: number; longestSec: number } {
+  if (!diffs.length) return { share: 0, longestSec: 0 };
+  let still = 0, run = 0, longest = 0;
+  for (const d of diffs) {
+    if (d < STILL_BELOW) { still++; longest = Math.max(longest, ++run); } else run = 0;
+  }
+  const round = (n: number) => Math.round(n * 1000) / 1000;
+  return { share: round(still / diffs.length), longestSec: round(longest / rate) };
+}
+
+// Sampled at 10 frames a second and 64 pixels wide, a ten-minute video is about 44 MB of grey pixels and is measured in a second or two.
+export const STILL_SAMPLE = { rate: 10, width: 64 };
+
+export async function stillnessOfVideo(video: string, width: number, height: number): Promise<{ share: number; longestSec: number }> {
+  const w = STILL_SAMPLE.width, h = Math.max(2, Math.round((w * height) / width / 2) * 2);
+  const { stdout } = await tool("ffmpeg", ["-v", "error", "-i", video, "-an", "-vf", `fps=${STILL_SAMPLE.rate},scale=${w}:${h}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"]);
+  return stillness(meanAbsDiffs(new Uint8Array(stdout.buffer, stdout.byteOffset, stdout.length), w * h), STILL_SAMPLE.rate);
+}
+
+// The standard deviation of the values over their mean: 0 when they are all equal.
+export function coefficientOfVariation(values: number[]): number {
+  const mean = values.reduce((a, b) => a + b, 0) / (values.length || 1);
+  if (!(mean > 0)) return 0;
+  return Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / values.length) / mean;
+}

package/src/remotion/Root.tsx

@@ -1,6 +1,6 @@
 import React from "react";
 import { Composition } from "remotion";
-import { UrlsContext } from "./kit/media";
+import { ManifestContext, UrlsContext } from "./kit/media";
 import type { VideoProps } from "./types";
 
 const EMPTY: VideoProps = { manifest: { fps: 30, width: 1080, height: 1920, totalFrames: 1, scenes: [] }, urls: {} };
@@ -8,7 +8,9 @@ const EMPTY: VideoProps = { manifest: { fps: 30, width: 1080, height: 1920, tota
 export const makeRoot = (Video: React.FC<VideoProps>): React.FC => {
   const WithMedia: React.FC<VideoProps> = (props) => (
     <UrlsContext.Provider value={props.urls}>
-      <Video {...props} />
+      <ManifestContext.Provider value={props.manifest}>
+        <Video {...props} />
+      </ManifestContext.Provider>
     </UrlsContext.Provider>
   );
   return () => (

package/src/remotion/kit/Camera.tsx

@@ -0,0 +1,22 @@
+import React from "react";
+import { AbsoluteFill, useCurrentFrame, useVideoConfig } from "remotion";
+import { cameraAt, cameraTransform, type CameraKey } from "./motion-math";
+
+// Moves the whole picture. Wrap all the scenes in it for one camera that never cuts, or wrap one scene's content.
+// Each key gives the point of the content, as fractions of the frame, that sits at the middle of the screen (x, y), the zoom (1 shows
+// the content as it fits) and a turn in degrees; what a key leaves out stays as it was. The move toward a key starts `lead` frames
+// before its frame (12 unless set) on a spring and lands on the key's frame. drift is a very slow push on top, as a fraction of the
+// zoom per second (0.02 is two percent a second), so a held shot is never perfectly still.
+export const Camera: React.FC<{
+  keys: CameraKey[];
+  drift?: number;
+  lead?: number;
+  stiffness?: number;
+  damping?: number;
+  children: React.ReactNode;
+}> = ({ keys, drift, lead, stiffness, damping, children }) => {
+  const frame = useCurrentFrame();
+  const { fps, width, height } = useVideoConfig();
+  const { origin, transform } = cameraTransform(cameraAt(keys, frame, fps, { lead, stiffness, damping, drift }), width, height);
+  return <AbsoluteFill style={{ transformOrigin: origin, transform }}>{children}</AbsoluteFill>;
+};

package/src/remotion/kit/Captions.tsx

@@ -1,6 +1,7 @@
 import React from "react";
 import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
 import type { WordTiming } from "../../pipeline/schema";
+import { captionGroups, groupAt, type CaptionGroup } from "./caption-groups";
 import { fonts, springs } from "./theme";
 
 export type CaptionMode = "highlight" | "pop" | "karaoke";
@@ -9,34 +10,50 @@ export type CaptionMode = "highlight" | "pop" | "karaoke";
 //   highlight: a line of words, the spoken one in the highlight colour (calm, educational)
 //   pop:       one to three words at a time, each popping in as it is spoken (high energy)
 //   karaoke:   a line that fills with the highlight colour as it is spoken (voiceover, music)
+// group, when given, decides which words share the screen and replaces perLine:
+//   "word":   exactly one word at a time
+//   "phrase": words grouped as they are spoken: a group ends at a sentence end, at a comma or dash after three words, or at a pause of
+//             0.35 s, and never holds more than 6 words or about 32 characters
+//   "none":   nothing is drawn (the plan's `captions: "none"`), so the manifest's value can be passed straight through
 export const Captions: React.FC<{
   words: WordTiming[];
   mode?: CaptionMode;
   highlight?: string;
   color?: string;
   perLine?: number;
+  group?: CaptionGroup;
   uppercase?: boolean;
   // Distance of the caption baseline from the bottom, as a fraction of the height. Default keeps it clear of platform UI.
   bottom?: number;
   // Font family. Set to font("heebo") or font("rubik") for Hebrew; rtl also reverses the word order on the line.
   face?: string;
   rtl?: boolean;
-}> = ({ words, mode = "highlight", highlight = "#ffe14d", color = "#ffffff", perLine, uppercase = false, bottom = 0.16, face, rtl = false }) => {
+}> = ({ words, mode = "highlight", highlight = "#ffe14d", color = "#ffffff", perLine, group, uppercase = false, bottom = 0.16, face, rtl = false }) => {
   const frame = useCurrentFrame();
   const { fps, width, height } = useVideoConfig();
   const t = frame / fps;
-  if (words.length === 0) return null;
+  if (words.length === 0 || group === "none") return null;
 
-  const size = perLine ?? (mode === "pop" ? 2 : 4);
-  let active = words.findIndex((w) => t < w.endSec);
-  if (active === -1) active = words.length - 1;
-  const lineStart = Math.floor(active / size) * size;
-  const line = words.slice(lineStart, lineStart + size);
+  if (group === "word" && t < words[0]!.startSec) return null;
+  let active: number, lineStart: number, line: WordTiming[];
+  if (group) {
+    const groups = captionGroups(words, group);
+    const at = groupAt(words, groups, t);
+    active = at.active;
+    [lineStart] = groups[at.index]!;
+    line = words.slice(lineStart, groups[at.index]![1]);
+  } else {
+    const size = perLine ?? (mode === "pop" ? 2 : 4);
+    active = words.findIndex((w) => t < w.endSec);
+    if (active === -1) active = words.length - 1;
+    lineStart = Math.floor(active / size) * size;
+    line = words.slice(lineStart, lineStart + size);
+  }
   const show = (w: WordTiming) => (uppercase ? w.word.toUpperCase() : w.word);
 
   const base: React.CSSProperties = {
     fontFamily: face ?? fonts.body, fontWeight: 800, textAlign: "center", direction: rtl ? "rtl" : "ltr", padding: `0 ${width * 0.06}px`,
-    fontSize: width * (mode === "pop" ? 0.075 : 0.055), lineHeight: 1.15, color,
+    fontSize: width * (mode === "pop" || group === "word" ? 0.075 : 0.055), lineHeight: 1.15, color,
     // A dark outline plus shadow keeps contrast above 4.5:1 on any background.
     WebkitTextStroke: `${width * 0.006}px rgba(0,0,0,0.85)`, paintOrder: "stroke fill",
     textShadow: "0 4px 18px rgba(0,0,0,0.7)",

package/src/remotion/kit/Carry.tsx

@@ -0,0 +1,38 @@
+import React from "react";
+import { useCurrentFrame, useVideoConfig } from "remotion";
+import { boxAt, type Box, type BoxKey } from "./motion-math";
+
+// What a function child is given: the box now, as fractions of the frame, and its size in pixels.
+export type CarryBox = Box & { widthPx: number; heightPx: number };
+
+// One element that lives outside the scenes and is held through several of them, so that it is still on screen when a scene changes
+// and moves, grows or turns into what the next scene needs. Place it once, beside the SceneFrames and above them.
+// Each key says where the box must have arrived by the given absolute frame: x and y are its centre, width and height are
+// fractions of the frame, radius is the corner radius as a fraction of the frame width, rotate is in degrees. The move toward a key
+// starts `lead` frames before its frame (12 unless set) on a spring and lands on the key's frame exactly. Before the first key nothing is
+// drawn unless that key sets an opacity; after the last the box holds.
+export const Carry: React.FC<{
+  keys: BoxKey[];
+  lead?: number;
+  stiffness?: number;
+  damping?: number;
+  children: React.ReactNode | ((box: CarryBox) => React.ReactNode);
+}> = ({ keys, lead, stiffness, damping, children }) => {
+  const frame = useCurrentFrame();
+  const { fps, width, height } = useVideoConfig();
+  const box = boxAt(keys, frame, fps, { lead, stiffness, damping });
+  if (!box || box.opacity <= 0) return null;
+  const widthPx = box.width * width, heightPx = box.height * height;
+  return (
+    <div
+      style={{
+        position: "absolute", left: (box.x - box.width / 2) * width, top: (box.y - box.height / 2) * height, width: widthPx, height: heightPx,
+        opacity: box.opacity, transform: box.rotate ? `rotate(${box.rotate}deg)` : undefined, borderRadius: box.radius * width, overflow: box.radius > 0 ? "hidden" : undefined,
+        // The layer sits above the scenes; it must not be the thing a pointer-like overlay thinks it hits.
+        pointerEvents: "none",
+      }}
+    >
+      {typeof children === "function" ? children({ ...box, widthPx, heightPx }) : children}
+    </div>
+  );
+};

package/src/remotion/kit/Music.tsx

@@ -0,0 +1,19 @@
+import React from "react";
+import { Audio, useVideoConfig } from "remotion";
+import { musicVolume, speechSpans, type SpeechScene } from "./music-math";
+import { useGain, useManifest, useMedia } from "./media";
+
+// The video's music track. Place it once, outside the scenes. It sits at `duckTo` while the voiceover is heard and comes up to `volume`
+// in the gaps, loops when the video is longer than the track, and fades out over the last second. `scenes` defaults to the manifest's.
+export const Music: React.FC<{ src: string; volume?: number; duckTo?: number; scenes?: SpeechScene[] }> = ({ src, volume = 0.5, duckTo = 0.12, scenes }) => {
+  const { fps, durationInFrames } = useVideoConfig();
+  const manifest = useManifest();
+  const gain = useGain(src);
+  const spans = React.useMemo(() => speechSpans(scenes ?? manifest?.scenes ?? [], fps), [scenes, manifest, fps]);
+  return (
+    <Audio
+      src={useMedia(src)} loop loopVolumeCurveBehavior="extend"
+      volume={(f) => musicVolume(f, { fps, totalFrames: durationInFrames, volume, duckTo, spans, gain })}
+    />
+  );
+};