reelkit-cli 0.3.1 → 0.5.0

package/src/commands/assets.ts

@@ -5,10 +5,11 @@ import { ApiFailure, download, uploadTo } from "../api/client";
 import { client, type Ctx, type Result } from "../context";
 import { LibraryKindSchema, MAX_CUTOUT_SECONDS, MAX_UPLOAD_BYTES, UploadKindSchema, type CutoutType } from "../contract";
 import { PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
-import { keyGreen } from "../project/chromakey";
+import { GreenTooDullError, keyGreen } from "../project/chromakey";
 import { buildManifest, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
 import { loadPlan } from "./plan";
 import { FILE_NAME } from "../render/validate";
+import { measureMusic, type MusicRecord } from "../project/music";
 import { probeFile } from "../project/probe";
 import { FILES, openProject, type Project } from "../project/project";
 
@@ -228,7 +229,7 @@ function setSceneClip(project: Project, sceneId: string, record: ClipRecord) {
   project.writeJson(FILES.clips, { ...project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {}), [sceneId]: record });
 }
 
-export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean }, deps: ClipDeps = {}): Promise<Result> {
+export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean; music?: boolean }, deps: ClipDeps = {}): Promise<Result> {
   const project = openProject(ctx.cwd);
   if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id)) return { ok: false, summary: `"${id}" is not a library id. Copy the id from \`reelkit assets search\`.` };
   if (opts.scene) {
@@ -240,7 +241,9 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
       return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}${clipScenes.length ? ` The clip scenes are: ${clipScenes.join(", ")}.` : ""}` };
     }
   }
+  if (opts.music && opts.scene) return { ok: false, summary: "--music makes the item the video's track, not a scene's picture. Use --music or --scene, not both." };
   const pulled = await client(ctx)("libraryPull", { id });
+  if (opts.music && pulled.item.kind !== "music") return { ok: false, summary: `${id} is ${pulled.item.kind}, not music, so it cannot be the video's track. Search with --kind music.` };
   if (unsafeName(pulled.filename, pulled.item.kind)) return { ok: false, summary: `The library returned an unsafe file name for ${id}, so nothing was written. Report this item.` };
   if (opts.scene) {
     const treatment = loadPlan(project).scenes.find((x) => x.id === opts.scene)?.treatment;
@@ -273,16 +276,36 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
     } else setSceneImage(project, opts.scene, path);
     if (project.exists(FILES.plan)) note = tryManifest(project, loadPlan(project)).note;
   }
+  if (pulled.item.kind === "music" && opts.music) return pullMusic(project, id, pulled.item.title, path);
+  const musicHint = pulled.item.kind === "music" ? ` To make it the video's track, with scene changes on its beat, run \`reelkit assets pull ${id} --music\`.` : "";
   const refs = keyedPath ? ` The original is urls["${path}"]; the keyed copy with a transparent background is urls["${keyedPath}"].` : ` Reference it in the composition as urls["${path}"].`;
   return {
     ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}) },
     summary: [
-      opts.scene ? `Pulled ${id} as the ${sceneIsClip ? "clip" : "image"} for scene ${opts.scene}.${keyedPath ? refs : ""}` : `Pulled ${id} to ${path}.${refs}`,
+      opts.scene ? `Pulled ${id} as the ${sceneIsClip ? "clip" : "image"} for scene ${opts.scene}.${keyedPath ? refs : ""}` : `Pulled ${id} to ${path}.${refs}${musicHint}`,
       ...(note ? [note] : []),
     ].join("\n"),
   };
 }
 
+// Makes a pulled track the video's one music track: measured here, saved in assets/music.json, and the timeline is laid out again around it.
+async function pullMusic(project: Project, id: string, title: string, path: string): Promise<Result> {
+  let measured: Awaited<ReturnType<typeof measureMusic>>;
+  try { measured = await measureMusic(project.path(path)); }
+  catch (e) { return { ok: false, summary: `${id} was downloaded to ${path} but could not be read as audio (${e instanceof Error ? e.message : String(e)}). Pull a different track.` }; }
+  const previous = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  const record: MusicRecord = { key: path, id, title, ...measured, gainDb: 0 };
+  project.writeJson(FILES.music, record);
+  const note = project.exists(FILES.plan) ? tryManifest(project, loadPlan(project)).note : undefined;
+  const beat = record.bpm
+    ? `about ${Math.round(record.bpm)} BPM; scene changes will land on its beat.`
+    : "no clear tempo was found, so scene changes stay where the narration puts them.";
+  return {
+    ok: true, data: { path, kind: "music", music: record },
+    summary: [`Pulled ${id} as the video's music track${previous && previous.id !== id ? ` (it replaces ${previous.id})` : ""}: ${record.durationSec.toFixed(1)}s, ${beat} Place it once with <Music src={urls["${path}"]} />.`, ...(note ? [note] : [])].join("\n"),
+  };
+}
+
 export async function assetsVoices(ctx: Ctx): Promise<Result> {
   const { voices } = await client(ctx)("voices", {});
   return {
@@ -388,6 +411,10 @@ export async function assetsGenClip(
     if (opts.green && have.greenScreen && !have.keyedKey) {
       // The clip was paid for but not keyed (ffmpeg failed or --green came later): keying is free, so do it now.
       const keyed = await keyScene(project, scene.id, have, key);
+      if (!keyed.ok && (keyed.data as { dull?: boolean } | undefined)?.dull) {
+        const cut = await cutoutScene(ctx, project, scene.id, have, now);
+        return { ...cut, data: { path: have.key, ...(cut.ok ? { keyedPath: keyedKey } : {}) }, summary: cut.ok ? `The green in scene ${scene.id}'s clip was too dull to key on this machine, so the subject was cut out on the server instead. ${cut.summary}` : cut.summary };
+      }
       if (!keyed.ok) return keyed;
       return { ok: true, data: { path: have.key, keyedPath: keyedKey }, summary: `Keyed the green out of scene ${scene.id}'s clip into ${keyedKey}.` };
     }
@@ -421,20 +448,23 @@ export async function assetsGenClip(
         const record: ClipRecord = { key: path, greenScreen: green, durationSec: st.durationSec ?? seconds };
         setSceneClip(project, scene.id, record);
         let keyedPath: string | undefined;
+        let dullGreen = false;
         if (green || opts.green) {
           // The paid clip is already saved above; if keying fails, running the command again keys it without a new charge.
           const keyed = await keyScene(project, scene.id, { ...record, greenScreen: true }, key);
-          if (!keyed.ok) return keyed;
-          keyedPath = keyedKey;
+          // A model's green is often too dull to key. The subject is then cut out on the server, which needs no green at all.
+          if (!keyed.ok && (keyed.data as { dull?: boolean } | undefined)?.dull) dullGreen = true;
+          else if (!keyed.ok) return keyed;
+          else keyedPath = keyedKey;
         }
         let cutoutNote: string | undefined;
-        if (opts.cutout) {
+        if (opts.cutout || dullGreen) {
           // The paid clip is already saved above; if the cutout fails, running the command again does only the cutout.
           const cut = await cutoutScene(ctx, project, scene.id, record, now);
           const shared = st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : "";
           if (!cut.ok) return { ...cut, data: { id, path, durationSec: record.durationSec, libraryId: st.libraryId ?? null }, summary: `Generated the clip for scene ${scene.id}${shared}. ${cut.summary}` };
           keyedPath = keyedKey;
-          cutoutNote = cut.summary;
+          cutoutNote = dullGreen ? `The generated green was too dull to key on this machine, so the subject was cut out on the server instead. ${cut.summary}` : cut.summary;
         }
         const { note } = tryManifest(project, plan);
         return {
@@ -508,7 +538,7 @@ async function keyScene(project: Project, sceneId: string, record: ClipRecord, k
   try {
     await key(project.path(record.key), project.path(keyedKey));
   } catch (e) {
-    return { ok: false, data: { path: record.key }, summary: `The clip is saved at ${record.key}, but the green could not be keyed out: ${e instanceof Error ? e.message : String(e)}` };
+    return { ok: false, data: { path: record.key, ...(e instanceof GreenTooDullError ? { dull: true } : {}) }, summary: `The clip is saved at ${record.key}, but the green could not be keyed out: ${e instanceof Error ? e.message : String(e)}` };
   }
   setSceneClip(project, sceneId, { ...record, greenScreen: true, keyedKey });
   return { ok: true, summary: "" };

package/src/commands/auth.ts

@@ -113,6 +113,6 @@ export async function whoami(ctx: Ctx): Promise<Result> {
   const q = me.quota;
   return {
     ok: true, data: me,
-    summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nClips: ${q.clips.used}/${q.clips.limit}\nCutouts: ${q.cutoutSeconds.used}/${q.cutoutSeconds.limit} seconds\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
+    summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nClips: ${q.clips.used}/${q.clips.limit}\nCutouts: ${q.cutoutSeconds.used}/${q.cutoutSeconds.limit} seconds\nTranscription: ${q.transcribeSeconds.used}/${q.transcribeSeconds.limit} seconds\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
   };
 }

package/src/commands/build.ts

@@ -3,12 +3,14 @@ import { mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { promisify } from "node:util";
 import type { Ctx, Result } from "../context";
-import type { AssetManifest } from "../pipeline/schema";
+import type { AssetManifest, ScenePlan } from "../pipeline/schema";
+import { compareFrames, SAMPLE_WIDTH, summariseContinuity, type Boundary } from "../render/continuity";
 import { buildManifest, missingAssets } from "../project/manifest";
+import { beatReport } from "../project/music";
 import { openProject, type Project } from "../project/project";
 import { mediaUrls, serveDir } from "../project/serve";
 import { bundleProject, disposeBundle, ensureRenderBrowser, renderStills, renderVideo, writeEntry, type EnsureBrowser } from "../render/render";
-import { FILE_NAME, MAIN_FILE, staticCheck, typecheck } from "../render/validate";
+import { FILE_NAME, MAIN_FILE, jsxUses, staticCheck, typecheck } from "../render/validate";
 import { loadPlan } from "./plan";
 
 const exec = promisify(execFile);
@@ -36,11 +38,28 @@ function inspect(project: Project): { errors: string[]; manifest?: AssetManifest
   return manifest ? { errors: [], manifest } : { errors: ["The manifest could not be built. Run `reelkit assets voiceover --all`."] };
 }
 
+// Advice about a composition that passes: never an error.
+function compositionNotes(project: Project, plan: ScenePlan): string[] {
+  const notes: string[] = [];
+  const names = readdirSync(project.path("src")).filter((f) => FILE_NAME.test(f));
+  const sources = names.map((n) => [n, readFileSync(project.path(join("src", n)), "utf8")] as const);
+  const rendersCaptions = sources.some(([n, src]) => jsxUses(src, n, "Captions").length > 0);
+  if (plan.captions === "none" && rendersCaptions) notes.push('The plan says captions: "none" but the composition renders <Captions>. Remove them, or change the plan if the user wants captions.');
+  if ((plan.captions === "word" || plan.captions === "phrase") && !rendersCaptions) notes.push(`The plan asks for captions (${plan.captions}) but the composition renders no <Captions>. Add <Captions words={s.words} group={manifest.captions} /> in each scene, or set captions to "none" in the plan if the user does not want them.`);
+  return notes;
+}
+
 const failed = (errors: string[]): Result => ({ ok: false, data: { errors }, summary: `${errors.length} problem(s):\n- ${errors.join("\n- ")}` });
 
 export async function check(ctx: Ctx): Promise<Result> {
-  const { errors } = inspect(openProject(ctx.cwd));
-  return errors.length ? failed(errors) : { ok: true, data: { errors: [] }, summary: "The composition passes. Run `reelkit preview` to see it." };
+  const project = openProject(ctx.cwd);
+  const { errors } = inspect(project);
+  if (errors.length) return failed(errors);
+  const shouldImprove = compositionNotes(project, loadPlan(project));
+  return {
+    ok: true, data: { errors: [], shouldImprove },
+    summary: ["The composition passes. Run `reelkit preview` to see it.", ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : [])].join("\n"),
+  };
 }
 
 // Bundles the composition and serves the project's media for as long as fn runs.
@@ -72,7 +91,7 @@ export async function ensureBrowserWithNotice(ctx: Ctx, ensure: EnsureBrowser =
 
 const ffmpegToJpeg = (png: string, jpg: string) => exec("ffmpeg", ["-y", "-loglevel", "error", "-i", png, "-vf", "scale=540:-2", "-q:v", "5", jpg]).then(() => undefined);
 
-type PreviewPoint = { sceneId: string; point: "early" | "late" | "middle"; frame: number; name: string };
+type PreviewPoint = { sceneId: string; point: "early" | "late" | "middle" | "end" | "start"; frame: number; name: string };
 
 // Two frames per scene, at 30% and 90% of its length, so what enters late in a scene is seen as well as what opens it.
 // A scene under 4 frames has no room for two distinct frames and gets its middle one. The scene's number leads each name
@@ -86,6 +105,30 @@ export function previewPoints(scenes: { id: string; startFrame: number; duration
   });
 }
 
+type BoundaryPoint = { from: string; to: string; end: PreviewPoint; start: PreviewPoint };
+
+// SceneFrame fades each scene in and out over up to 8 frames, so the very last and very first frames of two scenes are nearly empty.
+// The frames that show what a scene really holds are the last one at full strength and the first one at full strength: those are the two
+// either side of every change. The files are named by the change, b01-end-<scene> then b01-start-<scene>, so that they sort in order.
+export function boundaryPoints(scenes: { id: string; startFrame: number; durationFrames: number }[]): BoundaryPoint[] {
+  const edge = (d: number) => Math.max(1, Math.min(8, Math.floor(d / 2)));
+  return scenes.slice(0, -1).map((a, i) => {
+    const b = scenes[i + 1]!;
+    const n = `b${String(i + 1).padStart(2, "0")}`;
+    return {
+      from: a.id, to: b.id,
+      end: { sceneId: a.id, point: "end", frame: a.startFrame + Math.max(0, a.durationFrames - edge(a.durationFrames)), name: `${n}-end-${a.id}` },
+      start: { sceneId: b.id, point: "start", frame: b.startFrame + Math.min(b.durationFrames - 1, edge(b.durationFrames)), name: `${n}-start-${b.id}` },
+    };
+  });
+}
+
+// A small grey copy of a frame as plain pixels, for the continuity measurement.
+async function decodeGray(png: string, width: number, height: number): Promise<Uint8Array> {
+  const { stdout } = await exec("ffmpeg", ["-v", "error", "-i", png, "-vf", `scale=${width}:${height}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024 });
+  return new Uint8Array(stdout);
+}
+
 export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: string) => Promise<void>; ensureBrowser?: EnsureBrowser } = {}): Promise<Result> {
   const toJpeg = deps.toJpeg ?? ffmpegToJpeg;
   const project = openProject(ctx.cwd);
@@ -94,11 +137,13 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
   const dir = project.path("out/preview");
   rmSync(dir, { recursive: true, force: true });
   mkdirSync(dir, { recursive: true });
-  const points = previewPoints(manifest.scenes);
+  const sceneFrames = previewPoints(manifest.scenes);
+  const boundaries = boundaryPoints(manifest.scenes);
+  const points = [...sceneFrames, ...boundaries.flatMap((b) => [b.end, b.start])];
   try {
     try {
       await ensureBrowserWithNotice(ctx, deps.ensureBrowser);
-      await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, points.map((p) => p.frame), dir));
+      await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, [...new Set(points.map((p) => p.frame))], dir));
     } catch (e) {
       return failed([`Render failed: ${e instanceof Error ? e.message : String(e)}`]);
     }
@@ -113,7 +158,27 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
     } catch {
       return failed(["Could not write the preview frames: ffmpeg failed. Check that ffmpeg is installed (macOS: `brew install ffmpeg`) and run `reelkit preview` again."]);
     }
-    return { ok: true, data: { frames }, summary: `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}` };
+    // Whether anything survives each scene change is advice. If it cannot be measured, the preview is still the preview.
+    let continuity: { score: number | null; boundaries: Boundary[] } | undefined;
+    let continuityLines: string[] = [];
+    try {
+      const h = Math.round((SAMPLE_WIDTH * manifest.height) / manifest.width);
+      const measured: Boundary[] = [];
+      for (const b of boundaries) {
+        const [a, c] = await Promise.all([decodeGray(join(dir, `still-${b.end.frame}.png`), SAMPLE_WIDTH, h), decodeGray(join(dir, `still-${b.start.frame}.png`), SAMPLE_WIDTH, h)]);
+        const r = compareFrames(a, c, SAMPLE_WIDTH, h);
+        measured.push({ from: b.from, to: b.to, survived: Math.round(r.survived * 100) / 100, kind: r.kind });
+      }
+      if (measured.length) {
+        const summary = summariseContinuity(measured);
+        continuity = { score: summary.score === null ? null : Math.round(summary.score * 100) / 100, boundaries: measured };
+        continuityLines = summary.lines;
+      }
+    } catch { /* no continuity report */ }
+    const sceneLines = `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.filter((f) => f.point !== "end" && f.point !== "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}`;
+    const changeLines = boundaries.length ? `\nThe last and first frames either side of each scene change are saved too (b01-end-<scene>, b01-start-<scene>, ...), so that you can see what carries across:\n${frames.filter((f) => f.point === "end" || f.point === "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}` : "";
+    const beat = beatReport(manifest);
+    return { ok: true, data: { frames, ...(continuity ? { continuity } : {}), ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) }, summary: `${sceneLines}${changeLines}${continuityLines.length ? `\nContinuity: ${continuityLines.join("\n")}` : ""}${beat ? `\nBeat: ${beat.line}.` : ""}` };
   } finally {
     // The full-size stills are only inputs; none stays behind however this ends.
     for (const f of readdirSync(dir)) if (/^still-\d+\.png$/.test(f)) rmSync(join(dir, f), { force: true });

package/src/commands/plan.ts

@@ -1,6 +1,6 @@
 import { client, type Ctx, type Result } from "../context";
 import { loadCredentials } from "../credentials";
-import { estimateLength, reviewPlan } from "../pipeline/review";
+import { estimateLength, lengthVariation, reviewPlan, sceneSeconds } from "../pipeline/review";
 import { ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
 import { FILES, issueLines, openProject, type Project } from "../project/project";
 
@@ -15,6 +15,25 @@ export function loadPlan(project: Project): ScenePlan {
   return parsed.data;
 }
 
+// What a plan that follows a reference must satisfy: the reference has to be measured, and the pace should be of the same order.
+function referenceChecks(project: Project, plan: ScenePlan): { mustFix: string[]; improve: string[] } {
+  if (!plan.reference) return { mustFix: [], improve: [] };
+  const id = plan.reference.id;
+  if (!/^r-[0-9a-f]{8}$/.test(id) || !project.exists(`refs/${id}/ref.json`))
+    return { mustFix: [`reference ${id} is not in this project. Run \`reelkit ref download <url-or-file>\` and use the id it prints, or remove "reference" from plan.json.`], improve: [] };
+  const file = `refs/${id}/breakdown.json`;
+  if (!project.exists(file)) return { mustFix: [`reference ${id} has not been analysed yet. Run \`reelkit ref analyze ${id}\`.`], improve: [] };
+  let average: unknown;
+  try { average = project.readJson<{ pacing?: { averageShotSec?: unknown } }>(file).pacing?.averageShotSec; } catch { average = undefined; }
+  if (typeof average !== "number" || !(average > 0)) return { mustFix: [`${file} has no average shot length. Run \`reelkit ref analyze ${id} --redo\`.`], improve: [] };
+  const ours = estimateLength(plan).seconds / plan.scenes.length;
+  const ratio = ours / average;
+  const improve = ratio > 2 || ratio < 0.5
+    ? [`The plan's scenes average ${ours.toFixed(1)}s but the reference's shots average ${average.toFixed(1)}s: ${ratio > 2 ? "the new video will feel much slower" : "the new video will feel much faster"}. Change the number of scenes or the narration length, or say in reference.take why the pace differs.`]
+    : [];
+  return { mustFix: [], improve };
+}
+
 // Hard rules (mustFix) stop the video; soft notes (shouldImprove) are advice for a better script.
 export async function planCheck(ctx: Ctx): Promise<Result> {
   const project = openProject(ctx.cwd);
@@ -34,15 +53,18 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
   const footage = project.footage();
   const assets = project.assets().filter((a) => a.id !== footage?.id);
   const voiceIds = loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
-  const mustFix = validatePlan(parsed.data, { aspect: project.config().aspect, footage, assets, voiceIds });
-  const shouldImprove = mustFix.length ? [] : reviewPlan(parsed.data, { footage });
+  const ref = referenceChecks(project, parsed.data);
+  const mustFix = [...validatePlan(parsed.data, { aspect: project.config().aspect, footage, assets, voiceIds }), ...ref.mustFix];
+  const shouldImprove = mustFix.length ? [] : [...reviewPlan(parsed.data, { footage }), ...ref.improve];
   const length = estimateLength(parsed.data);
   const estimatedSeconds = Math.round(length.seconds);
+  const perScene = sceneSeconds(parsed.data).map((s) => ({ id: s.id, seconds: Math.round(s.seconds * 10) / 10 }));
+  const variation = Math.round(lengthVariation(sceneSeconds(parsed.data).map((s) => s.seconds)) * 100) / 100;
   const lines = [
     mustFix.length ? `Fix these:\n- ${mustFix.join("\n- ")}` : "The plan is valid.",
     `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
     ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : []),
     ...(voiceIds ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
   ];
-  return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words }, summary: lines.join("\n") };
+  return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation }, summary: lines.join("\n") };
 }

package/src/commands/ref.ts

@@ -0,0 +1,315 @@
+import { execFile } from "node:child_process";
+import { createHash, randomUUID } from "node:crypto";
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
+import { basename, extname, resolve } from "node:path";
+import { z } from "zod";
+import { ApiFailure, uploadTo } from "../api/client";
+import { client, type Ctx, type Result } from "../context";
+import { MAX_TRANSCRIBE_SECONDS } from "../contract";
+import { loadCredentials } from "../credentials";
+import { analyzeBeats, cutsOnBeat } from "../project/beats";
+import { openProject, type Project } from "../project/project";
+import {
+  audioDuration, coefficientOfVariation, decodeMono, detectCuts, extractFrame, extractMp3, loudnessLufs, nearestAspect, paletteOfFrames, paletteOfVideo, probeVideo, REF_ASPECTS, stillnessOfVideo, toMp4,
+} from "../project/refmeasure";
+
+// A reference is for learning how a video is built, so ten minutes is more than it needs and keeps every step quick.
+export const MAX_REF_SECONDS = 600;
+const MAX_SCENES = 40;
+const REF_ID = /^r-[0-9a-f]{8}$/;
+
+export const RefRecordSchema = z.object({
+  id: z.string(), source: z.string(), urlHash: z.string().optional(),
+  durationSec: z.number(), width: z.number(), height: z.number(), aspect: z.enum(REF_ASPECTS), fps: z.number(), hasAudio: z.boolean(), createdAt: z.string(),
+});
+export type RefRecord = z.infer<typeof RefRecordSchema>;
+
+export type Breakdown = {
+  source: string; durationSec: number; width: number; height: number; aspect: RefRecord["aspect"]; fps: number;
+  scenes: { index: number; startSec: number; endSec: number; durationSec: number; frames: string[]; palette: string[] }[];
+  // True when the video has more cuts than the 40 scenes listed: the last scene then holds everything after the 39th cut.
+  scenesCapped?: boolean;
+  pacing: { cuts: number; averageShotSec: number; shortestShotSec: number; longestShotSec: number; cutsPerSecond: number; firstCutSec?: number; cutsOnBeat?: number;
+    // How uneven the shot lengths are (standard deviation over the mean); left out with fewer than 3 shots. High means a quarter-second hit beside a long hold.
+    shotLengthVariation?: number };
+  // The share of the video's time in which the picture barely changes from frame to frame, and the longest such stretch. Missing from breakdowns saved by older versions.
+  stillness?: { share: number; longestSec: number };
+  palette: string[];
+  audio: { hasAudio: boolean; loudnessLufs?: number; tempoBpm?: number; beatConfidence: number };
+  beats?: number[];
+  transcript?: { language?: string; text: string; segments: { text: string; startSec: number; endSec: number }[] };
+};
+
+export const REF_NOTICE = "You are responsible for having the right to download this video; it is used only as a reference: none of its footage or music goes into the output.";
+
+// The external program that fetches a link. Tests give their own so that nothing touches the network.
+export type YtdlpRunner = { available(): Promise<boolean>; run(args: string[]): Promise<void> };
+export type RefDeps = { ytdlp?: YtdlpRunner };
+
+const realYtdlp: YtdlpRunner = {
+  available: () => new Promise((done) => { execFile("yt-dlp", ["--version"], (err) => done(!err)); }),
+  run: (args) => new Promise((done, fail) => {
+    // No shell: the link is one element of the argument list and can never become a command.
+    execFile("yt-dlp", args, { maxBuffer: 64 * 1024 * 1024 }, (err, _out, stderr) => {
+      if (!err) return done();
+      const last = String(stderr).trim().split("\n").filter(Boolean).at(-1) ?? err.message;
+      fail(new Error(last.replace(/^ERROR:\s*/, "")));
+    });
+  }),
+};
+
+const YTDLP_MISSING = "yt-dlp is not installed, and links need it. Install it (macOS: `brew install yt-dlp`, or `pipx install yt-dlp`) and run the command again; a video file on this machine works without it.";
+
+// One video, at most 1080p, as mp4, and nothing longer than the limit: the flags that keep a link from fetching a playlist, a channel or a huge file.
+export function ytdlpArgs(url: string, dest: string): string[] {
+  return [
+    "--ignore-config", "--no-playlist", "--max-downloads", "1", "--no-progress", "--no-warnings",
+    "--match-filter", `duration<=${MAX_REF_SECONDS}`,
+    "-f", "bv*[height<=1080][ext=mp4]+ba[ext=m4a]/b[height<=1080][ext=mp4]/bv*[height<=1080]+ba/b[height<=1080]",
+    "--merge-output-format", "mp4", "-o", dest, "--", url,
+  ];
+}
+
+const dirOf = (id: string) => `refs/${id}`;
+const fileOf = (id: string, name: string) => `refs/${id}/${name}`;
+
+function loadRef(project: Project, id: string): RefRecord {
+  if (!REF_ID.test(id)) throw new Error(`"${id}" is not a reference id. Run \`reelkit ref list\` to see them.`);
+  if (!project.exists(fileOf(id, "ref.json"))) throw new Error(`There is no reference ${id} in this project. Run \`reelkit ref list\`, or \`reelkit ref download <url-or-file>\`.`);
+  const parsed = RefRecordSchema.safeParse(project.readJson(fileOf(id, "ref.json")));
+  if (!parsed.success) throw new Error(`refs/${id}/ref.json is not valid. Download the reference again with \`reelkit ref download\`.`);
+  return parsed.data;
+}
+
+function allRefs(project: Project): RefRecord[] {
+  if (!project.exists("refs")) return [];
+  const out: RefRecord[] = [];
+  for (const name of readdirSync(project.path("refs")).sort()) {
+    if (!REF_ID.test(name) || !project.exists(fileOf(name, "ref.json"))) continue;
+    const parsed = RefRecordSchema.safeParse(project.readJsonOr(fileOf(name, "ref.json"), null));
+    if (parsed.success) out.push(parsed.data);
+  }
+  return out;
+}
+
+const fmt = (n: number) => String(Math.round(n * 10) / 10);
+const minutes = (sec: number) => (sec >= 90 ? `${Math.round(sec / 60)} minutes` : `${Math.round(sec)} seconds`);
+const tooLong = (what: string, sec: number) => `${what} is ${minutes(sec)} long and a reference is limited to ${MAX_REF_SECONDS / 60} minutes. Pick a shorter video, or trim it and give the file.`;
+
+// A link has a scheme of two or more letters; "C:\\clip.mp4" is a path.
+const looksLikeLink = (s: string) => /^[a-z][a-z0-9+.-]+:/i.test(s);
+
+export async function refDownload(ctx: Ctx, input: string, opts: { redo?: boolean } = {}, deps: RefDeps = {}): Promise<Result> {
+  const project = openProject(ctx.cwd);
+  input = input.trim();
+  const isLink = looksLikeLink(input);
+  let url: URL | undefined;
+  if (isLink) {
+    try { url = new URL(input); } catch { return { ok: false, summary: `"${input}" is not a link that can be downloaded. Give an http or https link, or a video file.` }; }
+    if (url.protocol !== "http:" && url.protocol !== "https:") return { ok: false, summary: `Only http and https links can be downloaded, not ${url.protocol}. Give an http or https link, or a video file.` };
+  }
+
+  const urlHash = url ? createHash("sha256").update(url.href).digest("hex") : undefined;
+  let id = `r-${randomUUID().replace(/-/g, "").slice(0, 8)}`;
+  if (urlHash) {
+    const have = allRefs(project).find((r) => r.urlHash === urlHash && project.exists(fileOf(r.id, "video.mp4")));
+    if (have && !opts.redo) return { ok: true, data: have, summary: `This link is already downloaded as ${have.id} (refs/${have.id}/video.mp4). Use --redo to download it again. Next: \`reelkit ref analyze ${have.id}\`.\n${REF_NOTICE}` };
+    if (have) { id = have.id; rmSync(project.path(dirOf(id)), { recursive: true, force: true }); }
+  }
+
+  const video = project.path(fileOf(id, "video.mp4"));
+  const cleanup = () => rmSync(project.path(dirOf(id)), { recursive: true, force: true });
+  let source: string;
+  try {
+    if (url) {
+      const runner = deps.ytdlp ?? realYtdlp;
+      if (!(await runner.available())) return { ok: false, summary: YTDLP_MISSING };
+      source = url.href;
+      mkdirSync(project.path(dirOf(id)), { recursive: true });
+      ctx.log(`Downloading ${url.href} with yt-dlp.`);
+      try { await runner.run(ytdlpArgs(url.href, video)); }
+      catch (e) { cleanup(); return { ok: false, summary: `yt-dlp could not download this link: ${e instanceof Error ? e.message : String(e)} Check the link, or download the video yourself and give its file.` }; }
+      if (!existsSync(video)) { cleanup(); return { ok: false, summary: "yt-dlp finished but saved no video. The link may be a page without a video, or longer than 10 minutes. Check the link, or give a video file." }; }
+    } else {
+      const from = resolve(ctx.cwd, input);
+      if (!existsSync(from) || !statSync(from).isFile()) return { ok: false, summary: `${basename(from)} does not exist. Check the path.` };
+      source = basename(from);
+      const pre = await probeVideo(from, basename(from));
+      if (pre.durationSec > MAX_REF_SECONDS) return { ok: false, summary: tooLong(basename(from), pre.durationSec) };
+      mkdirSync(project.path(dirOf(id)), { recursive: true });
+      // mp4 is copied as it is; any other container is converted, so that refs/<id>/video.mp4 is an mp4 whatever was given.
+      if ([".mp4", ".m4v"].includes(extname(from).toLowerCase())) copyFileSync(from, video);
+      else await toMp4(from, video);
+    }
+    const info = await probeVideo(video, source);
+    if (info.durationSec > MAX_REF_SECONDS) { cleanup(); return { ok: false, summary: tooLong("This video", info.durationSec) }; }
+    const record: RefRecord = {
+      id, source, ...(urlHash ? { urlHash } : {}), ...info, aspect: nearestAspect(info.width, info.height), createdAt: new Date().toISOString(),
+    };
+    project.writeJson(fileOf(id, "ref.json"), record);
+    return {
+      ok: true, data: record,
+      summary: `Saved reference ${id}: ${fmt(info.durationSec)}s, ${info.width}x${info.height} (${record.aspect}), ${fmt(info.fps)} fps, ${info.hasAudio ? "with audio" : "no audio"}, at refs/${id}/video.mp4. Next: \`reelkit ref analyze ${id}\`.\n${REF_NOTICE}`,
+    };
+  } catch (e) {
+    cleanup();
+    // Every failure here is already a plain line written for the person who gave the video.
+    return { ok: false, summary: e instanceof Error ? e.message : String(e) };
+  }
+}
+
+// The audio of a reference as a small mp3, made once.
+async function ensureAudio(project: Project, ref: RefRecord): Promise<string> {
+  const rel = fileOf(ref.id, "audio.mp3");
+  if (!project.exists(rel)) await extractMp3(project.path(fileOf(ref.id, "video.mp4")), project.path(rel));
+  return rel;
+}
+
+export async function refAudio(ctx: Ctx, id: string): Promise<Result> {
+  const project = openProject(ctx.cwd);
+  const ref = loadRef(project, id);
+  if (!project.exists(fileOf(id, "video.mp4"))) return { ok: false, summary: `refs/${id}/video.mp4 is missing. Download the reference again with \`reelkit ref download --redo\`.` };
+  if (!ref.hasAudio) return { ok: false, summary: `Reference ${id} has no audio track, so there is no audio to extract.` };
+  const had = project.exists(fileOf(id, "audio.mp3"));
+  const rel = await ensureAudio(project, ref);
+  const bytes = statSync(project.path(rel)).size;
+  return { ok: true, data: { path: rel, bytes, skipped: had }, summary: had ? `The audio of ${id} is already at ${rel}.` : `Saved the audio of ${id} at ${rel} (mono, 16 kHz, ${Math.round(bytes / 1024)} KB).` };
+}
+
+export async function refList(ctx: Ctx): Promise<Result> {
+  const project = openProject(ctx.cwd);
+  const refs = allRefs(project).map((r) => ({ id: r.id, source: r.source, durationSec: r.durationSec, analyzed: project.exists(fileOf(r.id, "breakdown.json")) }));
+  return {
+    ok: true, data: { references: refs },
+    summary: refs.length
+      ? refs.map((r) => `${r.id}  ${fmt(r.durationSec)}s  ${r.analyzed ? "analysed" : "not analysed"}  ${r.source}`).join("\n")
+      : "No references in this project yet. Run `reelkit ref download <url-or-file>`.",
+  };
+}
+
+type TranscriptOutcome = { transcript?: NonNullable<Breakdown["transcript"]>; note: string };
+
+// The audio (never the video) goes to the server and comes back as text. Whatever goes wrong here costs the transcript only: the measurements
+// of the video do not depend on it.
+async function transcribe(ctx: Ctx, project: Project, ref: RefRecord): Promise<TranscriptOutcome> {
+  const retry = `\`reelkit ref analyze ${ref.id} --redo\``;
+  if (!loadCredentials(ctx.env)) return { note: `No transcript: you are not logged in. Run \`reelkit auth login\`, then ${retry}.` };
+  try {
+    const rel = await ensureAudio(project, ref);
+    const path = project.path(rel);
+    const bytes = readFileSync(path);
+    const api = client(ctx);
+    ctx.log("Sending the audio (not the video) to the Reelkit server to be transcribed.");
+    const durationSec = Math.min(MAX_TRANSCRIBE_SECONDS, Math.max(0.1, await audioDuration(path)));
+    const up = await api("transcribeStart", { filename: "audio.mp3", contentType: "audio/mpeg", bytes: bytes.length, durationSec });
+    await uploadTo(up.uploadUrl, new Uint8Array(bytes), "audio/mpeg");
+    const t = await api("transcribeRun", { id: up.id });
+    return {
+      transcript: { ...(t.language ? { language: t.language } : {}), text: t.text, segments: t.segments },
+      note: `The audio (not the video) was sent to the Reelkit server to be transcribed and was deleted there. It used ${Math.ceil(t.durationSec)} seconds of your monthly transcription quota (\`reelkit whoami\`).`,
+    };
+  } catch (e) {
+    const why = e instanceof ApiFailure ? e.message : `the Reelkit server could not be reached (${e instanceof Error ? e.message : String(e)}).`;
+    return { note: `No transcript: ${why.endsWith(".") ? why : `${why}.`} Retry with ${retry}.` };
+  }
+}
+
+// Runs `fn` over the items, `size` at a time, and keeps the order of the results.
+async function inBatches<T, R>(items: T[], size: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const out: R[] = [];
+  for (let i = 0; i < items.length; i += size) out.push(...(await Promise.all(items.slice(i, i + size).map((x, j) => fn(x, i + j)))));
+  return out;
+}
+
+const round2 = (n: number) => Math.round(n * 100) / 100;
+
+async function measure(ctx: Ctx, project: Project, ref: RefRecord): Promise<Breakdown> {
+  const video = project.path(fileOf(ref.id, "video.mp4"));
+  ctx.log("Finding the cuts.");
+  const cuts = await detectCuts(video, ref.durationSec);
+  // The scenes listed are capped; the pacing below is measured on every cut.
+  const sceneCuts = cuts.slice(0, MAX_SCENES - 1);
+  const bounds = [0, ...sceneCuts, ref.durationSec];
+  rmSync(project.path(fileOf(ref.id, "frames")), { recursive: true, force: true });
+  ctx.log(`Saving sample frames of ${bounds.length - 1} scene(s).`);
+  // Each scene is a few short ffmpeg runs; running several scenes at once keeps a video with many cuts quick.
+  const scenes = await inBatches(bounds.slice(0, -1), 6, async (start, i) => {
+    const end = bounds[i + 1]!, len = end - start;
+    const n = String(i + 1).padStart(2, "0");
+    const frames = [`${dirOf(ref.id)}/frames/scene-${n}-1.jpg`, `${dirOf(ref.id)}/frames/scene-${n}-2.jpg`];
+    // A little inside the end, so that the last frame of the video can be read.
+    await extractFrame(video, start + len * 0.25, project.path(frames[0]!));
+    await extractFrame(video, Math.min(start + len * 0.75, ref.durationSec - 0.05), project.path(frames[1]!));
+    return { index: i + 1, startSec: round2(start), endSec: round2(end), durationSec: round2(len), frames, palette: await paletteOfFrames(frames.map((f) => project.path(f))) };
+  });
+  ctx.log("Measuring colours and sound.");
+  const palette = await paletteOfVideo(video, ref.durationSec);
+
+  const shots = [0, ...cuts, ref.durationSec].map((t, i, all) => (i ? t - all[i - 1]! : 0)).slice(1);
+  const pacing: Breakdown["pacing"] = {
+    cuts: cuts.length, averageShotSec: round2(ref.durationSec / shots.length), shortestShotSec: round2(Math.min(...shots)), longestShotSec: round2(Math.max(...shots)),
+    cutsPerSecond: round2(cuts.length / ref.durationSec), ...(cuts.length ? { firstCutSec: round2(cuts[0]!) } : {}),
+    ...(shots.length >= 3 ? { shotLengthVariation: round2(coefficientOfVariation(shots)) } : {}),
+  };
+  ctx.log("Measuring how much of the time the picture is still.");
+  const still = await stillnessOfVideo(video, ref.width, ref.height);
+
+  const audio: Breakdown["audio"] = { hasAudio: ref.hasAudio, beatConfidence: 0 };
+  let beats: number[] | undefined;
+  if (ref.hasAudio) {
+    // A track that cannot be measured is a reference without a tempo, not a failed analysis.
+    const loud = await loudnessLufs(video).catch(() => undefined);
+    if (loud !== undefined) audio.loudnessLufs = loud;
+    try {
+      const rate = 11025;
+      const r = analyzeBeats(await decodeMono(video, rate), rate);
+      audio.beatConfidence = r.confidence;
+      if (r.tempoBpm !== undefined && r.beats) {
+        audio.tempoBpm = r.tempoBpm;
+        beats = r.beats.filter((t) => t <= ref.durationSec);
+        if (cuts.length) pacing.cutsOnBeat = round2(cutsOnBeat(cuts, beats));
+      }
+    } catch { /* no tempo */ }
+  }
+  return { source: ref.source, durationSec: ref.durationSec, width: ref.width, height: ref.height, aspect: ref.aspect, fps: ref.fps, scenes, ...(cuts.length > MAX_SCENES - 1 ? { scenesCapped: true } : {}), pacing, stillness: still, palette, audio, ...(beats ? { beats } : {}) };
+}
+
+// A short text a person or an agent can read at once.
+function digest(id: string, b: Breakdown, notes: string[]): string {
+  const lines = [
+    `Reference ${id}: ${fmt(b.durationSec)}s, ${b.width}x${b.height} (${b.aspect}), ${fmt(b.fps)} fps.`,
+    b.pacing.cuts
+      ? `Shape: ${b.scenes.length}${b.scenesCapped ? "+" : ""} scenes, ${b.pacing.cuts} cuts; average shot ${fmt(b.pacing.averageShotSec)}s (shortest ${fmt(b.pacing.shortestShotSec)}s, longest ${fmt(b.pacing.longestShotSec)}s); first cut at ${fmt(b.pacing.firstCutSec!)}s.`
+      : "Shape: one continuous shot, no cuts found.",
+    ...(b.stillness ? [`Still ${Math.round(b.stillness.share * 100)}% of the time; longest hold ${fmt(b.stillness.longestSec)} s.${b.pacing.shotLengthVariation !== undefined ? ` Shot lengths vary by ${b.pacing.shotLengthVariation} (the spread over the average).` : ""}`] : []),
+    `Palette: ${b.palette.join(", ")}.`,
+    !b.audio.hasAudio ? "Audio: none."
+      : `Audio:${b.audio.loudnessLufs !== undefined ? ` ${b.audio.loudnessLufs} LUFS;` : ""} ${b.audio.tempoBpm !== undefined ? `tempo about ${Math.round(b.audio.tempoBpm)} BPM (confidence ${b.audio.beatConfidence})` : "no clear beat"}${b.pacing.cutsOnBeat !== undefined ? `; ${Math.round(b.pacing.cutsOnBeat * b.pacing.cuts)} of ${b.pacing.cuts} cuts land on a beat` : ""}.`,
+  ];
+  if (b.transcript) lines.push(`Transcript${b.transcript.language ? ` (${b.transcript.language})` : ""}: "${b.transcript.text.slice(0, 200)}${b.transcript.text.length > 200 ? "..." : ""}"`);
+  lines.push(...notes);
+  lines.push(`Look at the frames now: refs/${id}/frames/ has two per scene (scene-01-1.jpg, scene-01-2.jpg, ...). The full numbers are in refs/${id}/breakdown.json.`);
+  return lines.join("\n");
+}
+
+export async function refAnalyze(ctx: Ctx, id: string, opts: { transcript?: boolean; redo?: boolean } = {}): Promise<Result> {
+  const project = openProject(ctx.cwd);
+  const ref = loadRef(project, id);
+  if (!project.exists(fileOf(id, "video.mp4"))) return { ok: false, summary: `refs/${id}/video.mp4 is missing. Download the reference again with \`reelkit ref download --redo\`.` };
+  const file = fileOf(id, "breakdown.json");
+  if (project.exists(file) && !opts.redo) {
+    const have = project.readJson<Breakdown>(file);
+    return { ok: true, data: have, summary: digest(id, have, [`Already analysed: this is the saved breakdown. Use --redo to measure again.`]) };
+  }
+  const breakdown = await measure(ctx, project, ref);
+  const notes: string[] = [];
+  if (opts.transcript === false) notes.push(`No transcript (--no-transcript). Nothing was sent anywhere. To add it later: \`reelkit ref analyze ${id} --redo\`.`);
+  else if (!ref.hasAudio) notes.push("No transcript: the reference has no audio. Nothing was sent anywhere.");
+  else {
+    const t = await transcribe(ctx, project, ref);
+    if (t.transcript) breakdown.transcript = t.transcript;
+    notes.push(t.note);
+  }
+  project.writeJson(file, breakdown);
+  return { ok: true, data: breakdown, summary: digest(id, breakdown, notes) };
+}