reelkit-cli 0.3.1 → 0.5.0

package/src/contract/index.ts

@@ -67,6 +67,15 @@
 //   20 seconds is refused by the request schema.
 // - When the cutout quota is used up, `cutoutRun` is 429 `quota_exceeded` with the reset date. When the server has no GPU service configured,
 //   `cutoutStart` and `cutoutRun` are 400 `invalid_request` with the message "Background removal is not available on this server yet."
+// - A transcript of a reference video's audio is a job with two calls. `transcribeStart` checks the request and answers with an id and an
+//   `uploadUrl` (charging nothing; the client sends the audio there exactly as for a library upload); `transcribeRun` checks that the audio
+//   arrived (else 400 `invalid_request`), reserves `ceil(durationSec)` seconds from the caller's monthly transcription quota, transcribes,
+//   deletes the audio and answers with the transcript. Only the audio is sent, never the video. The audio is deleted from the server when the
+//   call ends, whatever the result. The transcript is kept with the job, so calling `transcribeRun` again for the same id returns the same
+//   transcript and does not charge again. A failed run is not charged: the seconds are given back. It is charged in whole seconds of audio
+//   (`ceil(durationSec)`), and audio longer than 600 seconds is refused by the request schema. Segment times are in seconds from the start
+//   of the audio. A transcribe id belongs to its caller: another user's id, or an unknown one, is `not_found`.
+// - When the transcription quota is used up, `transcribeRun` is 429 `quota_exceeded` with the reset date.
 // - `durationSec` is the decoded audio length; `words` are in seconds from the start of that audio.
 // - Search returns published items only, and leaves out items that are not matches at all; the default `limit` is 8.
 import { z } from "zod";
@@ -101,6 +110,12 @@ export const CutoutTypeSchema = z.enum(["video/mp4", "video/quicktime", "video/w
 export type CutoutType = z.infer<typeof CutoutTypeSchema>;
 // The longest video a cutout takes, in seconds.
 export const MAX_CUTOUT_SECONDS = 20;
+// The audio types a transcription takes, the most bytes it takes (25 MB) and the longest audio in seconds.
+export const TranscribeTypeSchema = z.enum(["audio/mpeg", "audio/mp4", "audio/wav"]);
+export type TranscribeType = z.infer<typeof TranscribeTypeSchema>;
+export const MAX_TRANSCRIBE_BYTES = 26_214_400;
+export const MAX_TRANSCRIBE_SECONDS = 600;
+export const TranscriptSegmentSchema = z.object({ text: z.string(), startSec: z.number(), endSec: z.number() });
 
 export const ErrorCodeSchema = z.enum(["unauthenticated", "quota_exceeded", "not_found", "invalid_request", "server_error"]);
 export type ErrorCode = z.infer<typeof ErrorCodeSchema>;
@@ -136,7 +151,7 @@ export type Voice = z.infer<typeof VoiceSchema>;
 const Meter = z.object({ used: z.number(), limit: z.number() });
 export const MeSchema = z.object({
   userId: z.string(), handle: z.string(),
-  quota: z.object({ voiceoverChars: Meter, images: Meter, clips: Meter, cutoutSeconds: Meter, resetsAt: z.string() }),
+  quota: z.object({ voiceoverChars: Meter, images: Meter, clips: Meter, cutoutSeconds: Meter, transcribeSeconds: Meter, resetsAt: z.string() }),
   // How many of the user's own items are published in the shared library. An item still in review, or sent back, is not counted.
   contributions: z.number(),
 });
@@ -207,6 +222,15 @@ export const routes = {
       url: z.string().optional(), ext: z.enum(["webm"]).optional(), contentType: z.literal("video/webm").optional(),
     }).refine((r) => (r.status === "done") === (r.url !== undefined && r.ext !== undefined && r.contentType !== undefined) && (r.status === "failed") === (r.message !== undefined),
       "url, ext and contentType belong to done and message to failed")),
+  transcribeStart: route("POST", "/transcripts", true,
+    z.object({
+      filename: text(200), contentType: TranscribeTypeSchema,
+      bytes: z.number().int().min(1).max(MAX_TRANSCRIBE_BYTES), durationSec: z.number().positive().max(MAX_TRANSCRIBE_SECONDS),
+    }),
+    z.object({ id: z.string(), uploadUrl: z.string() })),
+  transcribeRun: route("POST", "/transcripts/run", true,
+    z.object({ id: ItemId }),
+    z.object({ id: z.string(), text: z.string(), language: z.string().optional(), segments: z.array(TranscriptSegmentSchema), durationSec: z.number().positive() })),
   publicLibrary: route("GET", "/public/library", false,
     z.object({ q: text(200).optional(), kind: LibraryKindSchema.optional(), page: z.coerce.number().int().min(1).max(10000).optional() }),
     // With `q` the items are the best matches by meaning, best first, each with `match` (0..1), in one page (`hasMore` false). Under heavy

package/src/pipeline/beatsnap.ts

@@ -0,0 +1,52 @@
+// Puts scene changes on the beat. A scene is held a little longer at its end so that the next one starts exactly on a beat.
+// The voiceover and its word times never move: only a scene's length, and so where the later scenes start, changes.
+
+// The last scene must keep at least this long after its last word, as the natural padding already does (0.4 s) plus a little.
+export const MIN_TAIL_SEC = 0.5;
+
+export type SnapInput = {
+  // Each scene's natural length in frames.
+  naturalFrames: number[];
+  // When the last word of the last scene ends, in frames from the start of that scene.
+  lastWordEndFrame: number;
+  // Beats as frame numbers, ascending. They should reach past the end of the video.
+  beatFrames: number[];
+  fps: number;
+};
+
+const firstAtOrAfter = (beats: number[], frame: number): number | undefined => beats.find((b) => b >= frame);
+
+// The new length of every scene. A scene is only ever made longer, by less than one beat (the last by a few frames more, to keep its tail).
+// A boundary that has no beat close enough keeps its natural place, and the scenes after it follow from there.
+export function snapToBeats(input: SnapInput): number[] {
+  const { naturalFrames, beatFrames, fps } = input;
+  if (beatFrames.length < 2) return [...naturalFrames];
+  const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
+  const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
+  let cursor = 0;
+  return naturalFrames.map((natural, i) => {
+    const last = i === naturalFrames.length - 1;
+    let end = cursor + natural;
+    let limit = period;
+    if (last) {
+      const tail = cursor + input.lastWordEndFrame + Math.ceil(MIN_TAIL_SEC * fps);
+      if (tail > end) { limit += tail - end; end = tail; }
+    }
+    const beat = firstAtOrAfter(beatFrames, end);
+    const target = beat !== undefined && beat - (cursor + natural) <= limit ? beat : end;
+    const frames = target - cursor;
+    cursor = target;
+    return frames;
+  });
+}
+
+// Beats in seconds, continued at the tempo's own spacing until `untilSec`, for a video longer than the track (the track loops).
+export function extendBeats(beats: number[], bpm: number, untilSec: number): number[] {
+  if (!beats.length) return [];
+  const period = 60 / bpm;
+  const out = [...beats];
+  while (out[out.length - 1]! < untilSec) out.push(Math.round((out[out.length - 1]! + period) * 1000) / 1000);
+  return out;
+}
+
+export const toBeatFrames = (beatsSec: number[], fps: number): number[] => [...new Set(beatsSec.map((t) => Math.round(t * fps)))].sort((a, b) => a - b);

package/src/pipeline/review.ts

@@ -8,6 +8,30 @@ export function estimateLength(plan: ScenePlan): { words: number; seconds: numbe
   return { words: total, seconds: total / WORDS_PER_SEC };
 }
 
+// Each scene's estimated length in seconds, from its narration at the same pace as the whole-video estimate above.
+export function sceneSeconds(plan: ScenePlan): { id: string; seconds: number }[] {
+  return plan.scenes.map((s) => ({ id: s.id, seconds: words(s.narration) / WORDS_PER_SEC }));
+}
+
+// How uneven the scene lengths are: the standard deviation over the mean (0 when they are all the same).
+export function lengthVariation(seconds: number[]): number {
+  const mean = seconds.reduce((a, b) => a + b, 0) / (seconds.length || 1);
+  if (!(mean > 0)) return 0;
+  return Math.sqrt(seconds.reduce((a, b) => a + (b - mean) ** 2, 0) / seconds.length) / mean;
+}
+
+// A video whose scenes all last about as long feels like a slideshow. With four scenes or more, the lengths should differ a lot.
+export function rhythmNote(plan: ScenePlan): string | undefined {
+  if (plan.scenes.length < 4) return undefined;
+  const lengths = sceneSeconds(plan);
+  const values = lengths.map((l) => l.seconds);
+  const mean = values.reduce((a, b) => a + b, 0) / values.length;
+  const cv = lengthVariation(values);
+  if (cv >= 0.25 && values.some((v) => v < mean / 2)) return undefined;
+  const shortest = lengths.reduce((a, b) => (b.seconds < a.seconds ? b : a)), longest = lengths.reduce((a, b) => (b.seconds > a.seconds ? b : a));
+  return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). Make one or two scenes much shorter (a hit of a few words) and let one run long.`;
+}
+
 // Soft quality checks on a plan that already passes the hard rules in validatePlan.
 // These are things worth improving in the script; they never fail a video on their own.
 export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
@@ -26,10 +50,13 @@ export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): str
 
   for (const s of plan.scenes) {
     const n = words(s.narration);
-    if (!ctx.footage && n < 5) issues.push(`Scene ${s.id} has only ${n} words of narration; give it one full sentence.`);
+    // A hit of a few words is wanted in the rhythm of a video, so only a scene with almost nothing to say is flagged.
+    if (!ctx.footage && n < 3) issues.push(`Scene ${s.id} has only ${n} word${n === 1 ? "" : "s"} of narration; give it at least a short phrase.`);
     if (n > 45) issues.push(`Scene ${s.id} has ${n} words of narration; split it or cut it to under 45.`);
     if (s.onScreenText.length > 3) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 3.`);
     for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
   }
+  const rhythm = rhythmNote(plan);
+  if (rhythm) issues.push(rhythm);
   return issues;
 }

package/src/pipeline/schema.ts

@@ -28,6 +28,14 @@ export const ScenePlanSchema = z.object({
   // Optional only so plans stored before voices existed still load; a new plan must choose one.
   voiceId: z.string().optional().describe("id of the narration voice, one of the ids from `reelkit assets voices`, chosen to suit the idea, audience and language"),
   pace: z.enum(["slow", "normal", "fast"]).optional().describe("speaking pace: slow for calm or emotional, normal by default, fast for high-energy"),
+  // Whether the video has captions and of which kind: none, one word at a time, or a phrase at a time. Absent means "phrase".
+  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a phrase at a time (phrase, the default), as the user chose"),
+  // A video the new one is made "like": what is taken from it is how it feels (structure, pacing, motion), never its footage, music or words.
+  // Optional, so plans stored before references existed still load.
+  reference: z.object({
+    id: z.string().describe("the id of a reference in this project, from `reelkit ref list`"),
+    take: z.array(z.string()).min(1).max(6).describe("short notes on what is taken from it, e.g. \"fast cuts every ~1.2s\", \"big type on colour fields\""),
+  }).optional(),
   scenes: z.array(SceneSchema).min(3).max(8),
 });
 
@@ -77,6 +85,12 @@ export const AssetManifestSchema = z.object({
   height: z.number(),
   totalFrames: z.number(),
   footageKey: z.string().optional(),
+  // The video's music track, when one was pulled with `reelkit assets pull <id> --music`. beatFrames are the beats that fall inside the video.
+  music: z.object({ key: z.string(), bpm: z.number().optional(), beatFrames: z.array(z.number()) }).optional(),
+  // The plan's caption choice, with the default filled in, so the composition can pass it to <Captions group={...}>.
+  captions: z.enum(["none", "word", "phrase"]).optional(),
+  // Linear gain per sound file path that brings every pulled sound to a common level; the kit's Sfx and Music apply it.
+  soundGain: z.record(z.string(), z.number()).optional(),
   scenes: z.array(ManifestSceneSchema),
 });
 export type AssetManifest = z.infer<typeof AssetManifestSchema>;

package/src/project/beats.ts

@@ -0,0 +1,130 @@
+// Finds the tempo and the beat times of a piece of audio, on this machine, with no libraries. The audio is given as mono samples.
+//
+// The idea: loud moments (a drum hit, a click, a word) make the energy of the sound jump. The list of those jumps over time is the onset curve.
+// If the music has a pulse, the curve repeats itself after one beat, so the tempo is the delay at which the curve best matches a copy of itself.
+
+export type BeatAnalysis = {
+  // How sure the tempo is, from 0 (nothing repeats) to 1 (a perfect pulse). Real music is usually 0.3 to 0.7.
+  confidence: number;
+  // Present only when the pulse is clear enough to trust; a made-up tempo is worse than none.
+  tempoBpm?: number;
+  // Beat times in seconds from the start of the samples, on the grid of `tempoBpm`. Present with `tempoBpm`.
+  beats?: number[];
+};
+
+// Below this the autocorrelation peak is what noise produces by chance, so no tempo is reported.
+export const MIN_BEAT_CONFIDENCE = 0.2;
+
+const WINDOW = 512;
+const HOP = 256;
+const MIN_BPM = 60;
+const MAX_BPM = 200;
+// The range people actually move to; when a tempo outside it scores nearly as well, it is the half or double of one inside it.
+const PREFERRED_LOW = 80;
+const PREFERRED_HIGH = 160;
+const BPM_STEP = 0.5;
+// How many multiples of the beat length are compared: a real pulse matches at two, three and four beats too, noise does not.
+const HARMONICS = 4;
+
+// Energy per window, as a curve that jumps up where the sound gets louder. Quiet onsets count as much as loud ones (log scale), and the
+// curve is smoothed a little so a hit that falls between two windows still shows up in both.
+function onsetCurve(samples: Float32Array): Float64Array {
+  const frames = Math.max(0, Math.floor((samples.length - WINDOW) / HOP) + 1);
+  const energy = new Float64Array(frames);
+  for (let f = 0; f < frames; f++) {
+    let sum = 0;
+    const start = f * HOP;
+    for (let i = 0; i < WINDOW; i++) { const s = samples[start + i]!; sum += s * s; }
+    energy[f] = Math.log1p(100 * Math.sqrt(sum / WINDOW));
+  }
+  const rise = new Float64Array(frames);
+  for (let f = 1; f < frames; f++) rise[f] = Math.max(0, energy[f]! - energy[f - 1]!);
+  const smooth = new Float64Array(frames);
+  for (let f = 0; f < frames; f++) smooth[f] = 0.25 * (rise[f - 1] ?? 0) + 0.5 * rise[f]! + 0.25 * (rise[f + 1] ?? 0);
+  return smooth;
+}
+
+// The value of the curve at a position between two windows.
+function at(curve: Float64Array, pos: number): number {
+  if (pos < 0 || pos > curve.length - 1) return 0;
+  const i = Math.floor(pos), frac = pos - i;
+  return curve[i]! * (1 - frac) + (curve[i + 1] ?? curve[i]!) * frac;
+}
+
+// How much the curve looks like itself `lag` windows later, from 0 to about 1.
+function selfSimilarity(curve: Float64Array, lag: number): number {
+  const n = Math.floor(curve.length - lag);
+  if (n < 8) return 0;
+  let cross = 0, own = 0, shifted = 0;
+  for (let i = 0; i < n; i++) {
+    const a = curve[i]!, b = at(curve, i + lag);
+    cross += a * b; own += a * a; shifted += b * b;
+  }
+  return own > 0 && shifted > 0 ? cross / Math.sqrt(own * shifted) : 0;
+}
+
+export function analyzeBeats(samples: Float32Array, sampleRate: number): BeatAnalysis {
+  const curve = onsetCurve(samples);
+  if (curve.length < 16) return { confidence: 0 };
+  const mean = curve.reduce((s, v) => s + v, 0) / curve.length;
+  // The mean is taken out so that a steady loudness (which has no beat) does not look like a match.
+  for (let i = 0; i < curve.length; i++) curve[i] = curve[i]! - mean;
+  let energy = 0;
+  for (const v of curve) energy += v * v;
+  if (energy < 1e-9) return { confidence: 0 };
+
+  const framesPerSec = sampleRate / HOP;
+  const candidates: { bpm: number; score: number }[] = [];
+  for (let bpm = MIN_BPM; bpm <= MAX_BPM + 1e-9; bpm += BPM_STEP) {
+    const period = (60 / bpm) * framesPerSec;
+    let total = 0, used = 0;
+    for (let m = 1; m <= HARMONICS; m++) {
+      if (m * period > curve.length / 2) break;
+      total += selfSimilarity(curve, m * period);
+      used++;
+    }
+    candidates.push({ bpm, score: used ? total / used : 0 });
+  }
+  const isPeak = (i: number) => candidates[i]!.score >= (candidates[i - 1]?.score ?? -Infinity) && candidates[i]!.score >= (candidates[i + 1]?.score ?? -Infinity);
+  let best = 0;
+  for (let i = 1; i < candidates.length; i++) if (candidates[i]!.score > candidates[best]!.score) best = i;
+  const inPreferred = (i: number) => candidates[i]!.bpm >= PREFERRED_LOW && candidates[i]!.bpm <= PREFERRED_HIGH;
+  if (!inPreferred(best)) {
+    let alt = -1;
+    for (let i = 0; i < candidates.length; i++) if (inPreferred(i) && isPeak(i) && (alt < 0 || candidates[i]!.score > candidates[alt]!.score)) alt = i;
+    if (alt >= 0 && candidates[alt]!.score >= 0.8 * candidates[best]!.score) best = alt;
+  }
+
+  const confidence = Math.max(0, Math.min(1, candidates[best]!.score));
+  if (confidence < MIN_BEAT_CONFIDENCE) return { confidence: round(confidence, 2) };
+
+  // The peak lies between two of the tested tempos: a parabola through the three scores finds where it really is.
+  let bpm = candidates[best]!.bpm;
+  const l = candidates[best - 1]?.score, c = candidates[best]!.score, r = candidates[best + 1]?.score;
+  if (l !== undefined && r !== undefined && l - 2 * c + r < 0) bpm += (BPM_STEP * 0.5 * (l - r)) / (l - 2 * c + r);
+
+  // Where the beats fall: the shift of the grid that lands on the most onset.
+  const periodSec = 60 / bpm;
+  // Frame f describes the window that starts at f * HOP; a rise there means the sound arrived in the later part of that window.
+  const posOf = (t: number) => (t * sampleRate - 0.75 * WINDOW) / HOP;
+  const duration = samples.length / sampleRate;
+  let bestPhase = 0, bestHit = -Infinity;
+  for (let phase = 0; phase < periodSec; phase += HOP / 4 / sampleRate) {
+    let hit = 0;
+    for (let t = phase; t < duration; t += periodSec) hit += at(curve, posOf(t));
+    if (hit > bestHit) { bestHit = hit; bestPhase = phase; }
+  }
+  const beats: number[] = [];
+  // A beat that lands just before the first sample (a track that starts on a hit) is the first beat, at 0.
+  for (let t = bestPhase - periodSec; t < duration; t += periodSec) if (t >= -0.03) beats.push(round(Math.max(0, t), 3));
+  return { confidence: round(confidence, 2), tempoBpm: round(bpm, 1), beats };
+}
+
+// The share of cuts (0..1) that fall within `toleranceSec` of a beat.
+export function cutsOnBeat(cuts: number[], beats: number[], toleranceSec = 0.08): number {
+  if (!cuts.length || !beats.length) return 0;
+  const near = cuts.filter((c) => beats.some((b) => Math.abs(b - c) <= toleranceSec)).length;
+  return near / cuts.length;
+}
+
+const round = (n: number, digits: number) => Math.round(n * 10 ** digits) / 10 ** digits;

package/src/project/chromakey.ts

@@ -12,12 +12,19 @@ export type KeyOptions = {
   requireGreen?: boolean;
 };
 
+// Thrown when the background is green but not a saturated chroma green, or (for a clip that was asked to be on green) not green at
+// all. Video models rarely produce a true chroma green: they give a soft sage or olive that sits too close to skin and grey to be
+// keyed without taking the subject with it. The caller should cut the subject out on the server instead.
+export class GreenTooDullError extends Error {
+  constructor() { super("The green background is too dull to key cleanly: it is not a saturated chroma green. Use --cutout instead, which removes any background."); this.name = "GreenTooDullError"; }
+}
+
 const NOT_GREEN = "This video does not have a green background in its corners, so there is nothing to key out. Film the subject in front of an evenly lit green background that fills the frame.";
 
 // The colour to key on, read from the first frame's four corners. A real green screen, filmed or generated, is never exactly
 // #00FF00: it is whatever green the light and the encoder made of it, and keying on the wrong green leaves a haze. Undefined when
 // the corners are not green.
-async function cornerGreen(input: string): Promise<string | undefined> {
+async function cornerGreen(input: string): Promise<{ colour: string; saturated: boolean } | undefined> {
   const corner = (x: string, y: string) => `crop=24:24:${x}:${y},scale=1:1:flags=area`;
   const picks = ["0:0", "iw-24:0", "0:ih-24", "iw-24:ih-24"].map((c) => { const [x, y] = c.split(":") as [string, string]; return corner(x, y); });
   const colours: [number, number, number][] = [];
@@ -29,7 +36,9 @@ async function cornerGreen(input: string): Promise<string | undefined> {
   // Most corners must agree: a subject may reach one corner, but not three.
   if (green.length < 3) return undefined;
   const avg = (i: 0 | 1 | 2) => Math.round(green.reduce((sum, c) => sum + c[i], 0) / green.length);
-  return `0x${[avg(0), avg(1), avg(2)].map((n) => n.toString(16).padStart(2, "0")).join("")}`;
+  const [r, g, b] = [avg(0), avg(1), avg(2)];
+  // How far the green stands above the other two channels. Studio and pure greens are far above 60; a sage green is around 30.
+  return { colour: `0x${[r, g, b].map((n) => n.toString(16).padStart(2, "0")).join("")}`, saturated: g - Math.max(r, b) >= 60 };
 }
 
 // Turns the green of a green-screen clip into transparency: VP9 with an alpha channel, which Remotion plays with `transparent`.
@@ -39,12 +48,13 @@ export async function keyGreen(input: string, output: string, opts: KeyOptions =
   try {
     const found = await cornerGreen(input);
     if (!found && opts.requireGreen) throw new Error(NOT_GREEN);
-    const filter = `chromakey=color=${found ?? PURE_GREEN}:similarity=0.16:blend=0.08,despill=type=green:mix=0.5:expand=0,format=yuva420p`;
+    if (!found || !found.saturated) throw new GreenTooDullError();
+    const filter = `chromakey=color=${found.colour ?? PURE_GREEN}:similarity=0.16:blend=0.08,despill=type=green:mix=0.5:expand=0,format=yuva420p`;
     const audio = opts.keepAudio ? ["-c:a", "libopus", "-b:a", "96k"] : ["-an"];
     await run("ffmpeg", ["-v", "error", "-y", "-i", input, "-vf", filter, "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-b:v", "0", "-crf", "30", ...audio, output]);
   } catch (e) {
     rmSync(output, { force: true });
-    if (e instanceof Error && e.message === NOT_GREEN) throw e;
+    if (e instanceof GreenTooDullError || (e instanceof Error && e.message === NOT_GREEN)) throw e;
     if ((e as NodeJS.ErrnoException)?.code === "ENOENT") throw new Error("ffmpeg was not found. Install ffmpeg (macOS: `brew install ffmpeg`) and run the command again.");
     throw new Error("ffmpeg could not key the green out of the clip. The original clip is saved; run the command again to retry.");
   }

package/src/project/manifest.ts

@@ -1,5 +1,7 @@
 import { PACE_SPEED, type AssetManifest, type ScenePlan, type WordTiming } from "../pipeline/schema";
+import { extendBeats, snapToBeats, toBeatFrames } from "../pipeline/beatsnap";
 import { dimensionsFor, FPS, layoutScenes } from "../pipeline/timing";
+import type { MusicRecord } from "./music";
 import { FILES, type Project } from "./project";
 
 // What a scene was recorded from is stored with it, so a changed script or voice is noticed.
@@ -24,7 +26,22 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
   const footage = project.footage();
   const byId = new Map(project.assets().map((a) => [a.id, a]));
 
-  const layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  let layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  let beatFrames: number[] = [];
+  if (music) {
+    beatFrames = toBeatFrames(extendBeats(music.beats, music.bpm ?? 120, layout.totalFrames / FPS + 10), FPS);
+    // Scene changes land on beats only when the track has a trusted tempo. Footage has a fixed length that a longer scene must not overrun.
+    if (music.bpm && music.beats.length) {
+      const last = voiceovers[plan.scenes[plan.scenes.length - 1].id];
+      const lastWords = last.words.length ? Math.max(...last.words.map((w) => w.endSec)) : last.durationSec;
+      const lengths = snapToBeats({ naturalFrames: layout.scenes.map((s) => s.durationFrames), lastWordEndFrame: Math.ceil(lastWords * FPS), beatFrames, fps: FPS });
+      let cursor = 0;
+      const snapped = { scenes: lengths.map((durationFrames) => { const r = { startFrame: cursor, durationFrames }; cursor += durationFrames; return r; }), totalFrames: 0 };
+      snapped.totalFrames = cursor;
+      if (!footage || cursor <= Math.floor((footage.durationSec ?? 0) * FPS)) layout = snapped;
+    }
+  }
   let totalFrames = layout.totalFrames;
   if (footage) {
     const footageFrames = Math.floor((footage.durationSec ?? 0) * FPS);
@@ -38,7 +55,9 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
     fps: FPS,
     ...dimensionsFor(plan.aspect, footage),
     totalFrames,
+    captions: plan.captions ?? "phrase",
     ...(footage ? { footageKey: footage.key } : {}),
+    ...(music ? { music: { key: music.key, ...(music.bpm ? { bpm: music.bpm } : {}), beatFrames: music.bpm ? beatFrames.filter((f) => f < totalFrames) : [] } } : {}),
     scenes: plan.scenes.map((scene, i) => ({
       id: scene.id,
       startFrame: layout.scenes[i].startFrame,
@@ -64,6 +83,8 @@ export function missingAssets(project: Project, plan: ScenePlan): string[] {
   const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
   const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
   const problems: string[] = [];
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  if (music && !project.exists(music.key)) problems.push(`Missing file: ${music.key}. Pull the track again with \`reelkit assets pull ${music.id} --music\`.`);
   for (const s of plan.scenes) {
     if (!voiceovers[s.id]) problems.push(`Scene ${s.id} has no voiceover.`);
     else if (voiceoverStale(voiceovers[s.id], s, plan)) problems.push(`Scene ${s.id}'s voiceover is out of date. Run \`reelkit assets voiceover --all\`.`);

package/src/project/music.ts

@@ -0,0 +1,25 @@
+import type { AssetManifest } from "../pipeline/schema";
+import { analyzeBeats } from "./beats";
+import { audioDuration, decodeMono } from "./refmeasure";
+
+// The video's one music track, stored in assets/music.json. `bpm` and `beats` are present only when the pulse was clear enough to trust.
+export type MusicRecord = { key: string; id: string; title: string; durationSec: number; bpm?: number; beatConfidence: number; beats: number[]; gainDb: number };
+
+const RATE = 11025;
+
+// Measures a pulled track on this machine: its length, its tempo and where its beats fall.
+export async function measureMusic(path: string): Promise<Pick<MusicRecord, "durationSec" | "bpm" | "beatConfidence" | "beats">> {
+  const durationSec = Math.round((await audioDuration(path)) * 1000) / 1000;
+  const r = analyzeBeats(await decodeMono(path, RATE), RATE);
+  return { durationSec, beatConfidence: r.confidence, beats: r.tempoBpm !== undefined && r.beats ? r.beats : [], ...(r.tempoBpm !== undefined ? { bpm: r.tempoBpm } : {}) };
+}
+
+// How many scene changes fall on a beat, for the preview.
+export function beatReport(manifest: AssetManifest): { bpm: number; boundariesOnBeat: number; boundaries: number; line: string } | undefined {
+  const m = manifest.music;
+  if (!m?.bpm || manifest.scenes.length < 2) return undefined;
+  const beats = new Set(m.beatFrames);
+  const starts = manifest.scenes.slice(1).map((s) => s.startFrame);
+  const on = starts.filter((f) => beats.has(f)).length;
+  return { bpm: m.bpm, boundariesOnBeat: on, boundaries: starts.length, line: `${on} of ${starts.length} scene changes land on the beat at ${Math.round(m.bpm)} BPM` };
+}

package/src/project/project.ts

@@ -5,7 +5,7 @@ import { AspectSchema, type AssetRecord } from "../pipeline/schema";
 
 export const FILES = {
   config: "reelkit.json", plan: "plan.json", manifest: "manifest.json",
-  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json",
+  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json", music: "assets/music.json",
 } as const;
 
 // "path: message", or just the message for a problem at the root of the file.

package/src/project/refmeasure.ts

@@ -0,0 +1,192 @@
+// Measuring a reference video with ffmpeg and ffprobe, all on this machine. Every failure is a one-line Error for the person who gave the video.
+import { execFile } from "node:child_process";
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { promisify } from "node:util";
+
+const exec = promisify(execFile);
+
+export const REF_ASPECTS = ["9:16", "16:9", "1:1", "4:5"] as const;
+export type RefAspect = (typeof REF_ASPECTS)[number];
+
+async function tool(cmd: "ffmpeg" | "ffprobe", args: string[]): Promise<{ stdout: Buffer; stderr: string }> {
+  try {
+    const r = await exec(cmd, args, { maxBuffer: 256 * 1024 * 1024, encoding: "buffer" });
+    return { stdout: r.stdout, stderr: r.stderr.toString("utf8") };
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException)?.code === "ENOENT") throw new Error(`${cmd} was not found. Install ffmpeg (macOS: \`brew install ffmpeg\`) and try again.`);
+    throw e;
+  }
+}
+
+export type VideoInfo = { durationSec: number; width: number; height: number; fps: number; hasAudio: boolean };
+
+const rate = (r: string | undefined): number => {
+  const [a, b] = (r ?? "").split("/").map(Number);
+  return a && b ? a / b : a || 0;
+};
+
+export async function probeVideo(path: string, name: string): Promise<VideoInfo> {
+  const unreadable = new Error(`${name} could not be read as a video. It may be damaged or not a video: try another file or link.`);
+  let info: { streams?: { codec_type?: string; width?: number; height?: number; avg_frame_rate?: string; r_frame_rate?: string; duration?: string }[]; format?: { duration?: string } };
+  try {
+    info = JSON.parse((await tool("ffprobe", ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", path])).stdout.toString("utf8"));
+  } catch (e) {
+    if (e instanceof Error && e.message.startsWith("ffprobe was not found")) throw e;
+    throw unreadable;
+  }
+  const v = info.streams?.find((s) => s.codec_type === "video");
+  const duration = Number(info.format?.duration ?? v?.duration);
+  if (!v || !(v.width! > 0) || !(v.height! > 0) || !(duration > 0)) throw unreadable;
+  const fps = rate(v.avg_frame_rate) || rate(v.r_frame_rate);
+  return { durationSec: Math.round(duration * 100) / 100, width: v.width!, height: v.height!, fps: Math.round(fps * 100) / 100, hasAudio: Boolean(info.streams?.some((s) => s.codec_type === "audio")) };
+}
+
+// The nearest of the four shapes, by how far the ratios are apart on a log scale (so 2:1 is as far from 1:1 as 1:2).
+export function nearestAspect(width: number, height: number): RefAspect {
+  const ratio = width / height;
+  const value: Record<RefAspect, number> = { "9:16": 9 / 16, "16:9": 16 / 9, "1:1": 1, "4:5": 4 / 5 };
+  return REF_ASPECTS.reduce((best, a) => (Math.abs(Math.log(ratio / value[a])) < Math.abs(Math.log(ratio / value[best])) ? a : best));
+}
+
+const MIN_SHOT_SEC = 0.25;
+
+// The times where the picture changes completely. Cuts closer than a quarter of a second to each other, to the start or to the end are one
+// flash or a fade, not a shot of their own.
+export async function detectCuts(video: string, durationSec: number, threshold = 0.3): Promise<number[]> {
+  const { stderr } = await tool("ffmpeg", ["-hide_banner", "-nostats", "-i", video, "-an", "-vf", `select='gt(scene,${threshold})',showinfo`, "-f", "null", "-"]);
+  const times = [...stderr.matchAll(/pts_time:\s*([0-9.]+)/g)].map((m) => Number(m[1])).sort((a, b) => a - b);
+  const cuts: number[] = [];
+  for (const t of times) {
+    if (t < MIN_SHOT_SEC || durationSec - t < MIN_SHOT_SEC) continue;
+    if (cuts.length && t - cuts[cuts.length - 1]! < MIN_SHOT_SEC) continue;
+    cuts.push(Math.round(t * 1000) / 1000);
+  }
+  return cuts;
+}
+
+// One JPEG of the video at `atSec`, at most 540 pixels on the long side.
+export async function extractFrame(video: string, atSec: number, dest: string): Promise<void> {
+  mkdirSync(dirname(dest), { recursive: true });
+  await tool("ffmpeg", ["-v", "error", "-y", "-ss", atSec.toFixed(3), "-i", video, "-frames:v", "1",
+    "-vf", "scale=w='if(gt(iw,ih),min(540,iw),-2)':h='if(gt(iw,ih),-2,min(540,ih))'", "-q:v", "4", dest]);
+}
+
+type Pixel = [number, number, number];
+
+// Averages of the main colours: each pixel goes in one of 64 buckets (four levels per channel), and the fullest buckets, skipping one whose
+// colour is close to a colour already taken, are named by the average of their pixels.
+export function dominantColors(pixels: Pixel[], max: number): string[] {
+  const buckets = new Map<number, { n: number; sum: Pixel }>();
+  for (const [r, g, b] of pixels) {
+    const key = (r >> 6) * 16 + (g >> 6) * 4 + (b >> 6);
+    const e = buckets.get(key) ?? { n: 0, sum: [0, 0, 0] as Pixel };
+    e.n++; e.sum[0] += r; e.sum[1] += g; e.sum[2] += b;
+    buckets.set(key, e);
+  }
+  const chosen: Pixel[] = [];
+  for (const e of [...buckets.values()].sort((a, b) => b.n - a.n)) {
+    const c = e.sum.map((v) => Math.round(v / e.n)) as Pixel;
+    if (chosen.some((o) => Math.hypot(o[0] - c[0], o[1] - c[1], o[2] - c[2]) < 48)) continue;
+    chosen.push(c);
+    if (chosen.length === max) break;
+  }
+  return chosen.map(([r, g, b]) => `#${[r, g, b].map((v) => v.toString(16).padStart(2, "0")).join("")}`);
+}
+
+// Every picture is shrunk to 4 by 4 pixels, which averages away detail and leaves the colour of each part of the frame.
+async function tinyPixels(inputArgs: string[], filter: string): Promise<Pixel[]> {
+  const { stdout } = await tool("ffmpeg", ["-v", "error", ...inputArgs, "-an", "-vf", `${filter}scale=4:4:flags=area`, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"]);
+  const out: Pixel[] = [];
+  for (let i = 0; i + 2 < stdout.length; i += 3) out.push([stdout[i]!, stdout[i + 1]!, stdout[i + 2]!]);
+  return out;
+}
+
+export async function paletteOfVideo(video: string, durationSec: number, max = 5): Promise<string[]> {
+  // At most about 120 pictures however long the video is.
+  const fps = Math.max(0.15, Math.min(2, 120 / durationSec));
+  return dominantColors(await tinyPixels(["-i", video], `fps=${fps.toFixed(3)},`), max);
+}
+
+export async function paletteOfFrames(frames: string[], max = 3): Promise<string[]> {
+  const pixels: Pixel[] = [];
+  for (const f of frames) pixels.push(...(await tinyPixels(["-i", f], "")));
+  return dominantColors(pixels, max);
+}
+
+// The integrated loudness in LUFS, or undefined when there is none (silence is minus infinity).
+export async function loudnessLufs(video: string): Promise<number | undefined> {
+  const { stderr } = await tool("ffmpeg", ["-hide_banner", "-nostats", "-i", video, "-vn", "-af", "ebur128=framelog=quiet", "-f", "null", "-"]);
+  const all = [...stderr.matchAll(/\bI:\s+(-?[0-9.]+)\s+LUFS/g)];
+  const value = Number(all.at(-1)?.[1]);
+  return Number.isFinite(value) ? Math.round(value * 10) / 10 : undefined;
+}
+
+// The audio as mono samples between -1 and 1, for the beat analysis.
+export async function decodeMono(video: string, sampleRate: number): Promise<Float32Array> {
+  const { stdout } = await tool("ffmpeg", ["-v", "error", "-i", video, "-vn", "-ac", "1", "-ar", String(sampleRate), "-f", "s16le", "-"]);
+  const n = Math.floor(stdout.length / 2);
+  const out = new Float32Array(n);
+  for (let i = 0; i < n; i++) out[i] = stdout.readInt16LE(i * 2) / 32768;
+  return out;
+}
+
+// Speech needs little: mono, 16 kHz and 64 kbit/s keep a ten-minute reference under 5 MB.
+export async function extractMp3(video: string, dest: string): Promise<void> {
+  mkdirSync(dirname(dest), { recursive: true });
+  await tool("ffmpeg", ["-v", "error", "-y", "-i", video, "-vn", "-ac", "1", "-ar", "16000", "-b:a", "64k", "-c:a", "libmp3lame", dest]);
+}
+
+export async function toMp4(source: string, dest: string): Promise<void> {
+  mkdirSync(dirname(dest), { recursive: true });
+  await tool("ffmpeg", ["-v", "error", "-y", "-i", source, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-movflags", "+faststart", dest]);
+}
+
+export async function audioDuration(path: string): Promise<number> {
+  const { stdout } = await tool("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path]);
+  return Number(stdout.toString("utf8").trim());
+}
+
+// The mean absolute difference, from 0 to 255, between each frame of a run of grey frames and the next. frameSize is the pixels in one frame.
+export function meanAbsDiffs(frames: Uint8Array, frameSize: number): number[] {
+  const count = Math.floor(frames.length / frameSize);
+  const out: number[] = [];
+  for (let f = 0; f + 1 < count; f++) {
+    let sum = 0;
+    for (let i = 0, a = f * frameSize, b = a + frameSize; i < frameSize; i++) sum += Math.abs(frames[a + i]! - frames[b + i]!);
+    out.push(sum / frameSize);
+  }
+  return out;
+}
+
+// Two frames whose pictures differ by less than this, on average over 255 grey levels, are the same picture for a viewer: video
+// compression noise on a held shot stays well under it, and a slow push still stays above it.
+const STILL_BELOW = 1;
+
+// The share of the video's time in which the picture barely changes from one sampled frame to the next, and the longest unbroken stretch
+// of it. diffs has one entry for each pair of consecutive samples, taken `rate` times a second.
+export function stillness(diffs: number[], rate: number): { share: number; longestSec: number } {
+  if (!diffs.length) return { share: 0, longestSec: 0 };
+  let still = 0, run = 0, longest = 0;
+  for (const d of diffs) {
+    if (d < STILL_BELOW) { still++; longest = Math.max(longest, ++run); } else run = 0;
+  }
+  const round = (n: number) => Math.round(n * 1000) / 1000;
+  return { share: round(still / diffs.length), longestSec: round(longest / rate) };
+}
+
+// Sampled at 10 frames a second and 64 pixels wide, a ten-minute video is about 44 MB of grey pixels and is measured in a second or two.
+export const STILL_SAMPLE = { rate: 10, width: 64 };
+
+export async function stillnessOfVideo(video: string, width: number, height: number): Promise<{ share: number; longestSec: number }> {
+  const w = STILL_SAMPLE.width, h = Math.max(2, Math.round((w * height) / width / 2) * 2);
+  const { stdout } = await tool("ffmpeg", ["-v", "error", "-i", video, "-an", "-vf", `fps=${STILL_SAMPLE.rate},scale=${w}:${h}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"]);
+  return stillness(meanAbsDiffs(new Uint8Array(stdout.buffer, stdout.byteOffset, stdout.length), w * h), STILL_SAMPLE.rate);
+}
+
+// The standard deviation of the values over their mean: 0 when they are all equal.
+export function coefficientOfVariation(values: number[]): number {
+  const mean = values.reduce((a, b) => a + b, 0) / (values.length || 1);
+  if (!(mean > 0)) return 0;
+  return Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / values.length) / mean;
+}