reelkit-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +2 -2
  2. package/package.json +5 -1
  3. package/skill/SKILL.md +20 -9
  4. package/skill/THIRD_PARTY.md +102 -0
  5. package/skill/commands/launch-film.md +7 -0
  6. package/skill/reference/asset-reuse.md +13 -2
  7. package/skill/reference/backgrounds.md +63 -0
  8. package/skill/reference/beat-sync.md +25 -19
  9. package/skill/reference/captions.md +11 -5
  10. package/skill/reference/continuity.md +21 -2
  11. package/skill/reference/kit.md +139 -11
  12. package/skill/reference/launch-film.md +190 -0
  13. package/skill/reference/remotion-composition.md +4 -3
  14. package/skill/reference/scene-treatments.md +20 -0
  15. package/skill/reference/scriptwriting.md +4 -1
  16. package/skill/reference/three-d.md +134 -0
  17. package/skill/reference/voice-sync.md +108 -0
  18. package/src/agents.ts +23 -12
  19. package/src/api/client.ts +4 -1
  20. package/src/cli.ts +18 -7
  21. package/src/commands/assets.ts +310 -30
  22. package/src/commands/build.ts +148 -35
  23. package/src/commands/init.ts +1 -1
  24. package/src/commands/install.ts +1 -1
  25. package/src/commands/plan.ts +8 -5
  26. package/src/commands/ref.ts +5 -2
  27. package/src/contract/index.ts +4 -2
  28. package/src/pipeline/beatsnap.ts +72 -0
  29. package/src/pipeline/review.ts +44 -10
  30. package/src/pipeline/schema.ts +51 -4
  31. package/src/pipeline/timing.ts +27 -1
  32. package/src/project/background.ts +33 -0
  33. package/src/project/layers.ts +60 -0
  34. package/src/project/manifest.ts +59 -14
  35. package/src/project/music.ts +19 -5
  36. package/src/project/project.ts +4 -1
  37. package/src/project/soundreport.ts +347 -0
  38. package/src/project/svgcheck.ts +21 -0
  39. package/src/remotion/kit/Assemble3D.tsx +92 -0
  40. package/src/remotion/kit/BrowserFrame.tsx +83 -0
  41. package/src/remotion/kit/Camera.tsx +6 -4
  42. package/src/remotion/kit/Captions.tsx +33 -17
  43. package/src/remotion/kit/Card3D.tsx +211 -0
  44. package/src/remotion/kit/ChapterFrame.tsx +68 -0
  45. package/src/remotion/kit/CounterRoll.tsx +75 -0
  46. package/src/remotion/kit/GlassPanel.tsx +43 -0
  47. package/src/remotion/kit/Grounds.tsx +177 -0
  48. package/src/remotion/kit/Headline.tsx +97 -0
  49. package/src/remotion/kit/Hero3D.tsx +197 -0
  50. package/src/remotion/kit/HudOverlay.tsx +52 -0
  51. package/src/remotion/kit/ImageLayers.tsx +48 -0
  52. package/src/remotion/kit/Music.tsx +4 -4
  53. package/src/remotion/kit/NamedCursor.tsx +54 -0
  54. package/src/remotion/kit/Orbit3D.tsx +49 -0
  55. package/src/remotion/kit/Particles3D.tsx +74 -0
  56. package/src/remotion/kit/Place.tsx +12 -0
  57. package/src/remotion/kit/PromptBox.tsx +84 -0
  58. package/src/remotion/kit/Scene3D.tsx +70 -0
  59. package/src/remotion/kit/SceneFrame.tsx +88 -11
  60. package/src/remotion/kit/SoundCues.tsx +22 -0
  61. package/src/remotion/kit/TerminalLog.tsx +98 -0
  62. package/src/remotion/kit/Text3D.tsx +78 -0
  63. package/src/remotion/kit/TextOnImage.tsx +41 -0
  64. package/src/remotion/kit/Warp3D.tsx +59 -0
  65. package/src/remotion/kit/bg-math.ts +179 -0
  66. package/src/remotion/kit/caption-groups.ts +7 -3
  67. package/src/remotion/kit/caption-style.ts +45 -0
  68. package/src/remotion/kit/docs.ts +132 -11
  69. package/src/remotion/kit/image-layers-math.ts +115 -0
  70. package/src/remotion/kit/index.ts +43 -1
  71. package/src/remotion/kit/inter-bold-typeface.ts +3 -0
  72. package/src/remotion/kit/motion-math.ts +36 -2
  73. package/src/remotion/kit/music-math.ts +27 -10
  74. package/src/remotion/kit/quiet-three.ts +11 -0
  75. package/src/remotion/kit/sample-text.ts +55 -0
  76. package/src/remotion/kit/scene3d-context.ts +5 -0
  77. package/src/remotion/kit/seeded.ts +13 -0
  78. package/src/remotion/kit/sound-cues.ts +89 -0
  79. package/src/remotion/kit/sound-kinds.ts +122 -0
  80. package/src/remotion/kit/theme.ts +2 -0
  81. package/src/remotion/kit/three-fx-math.ts +192 -0
  82. package/src/remotion/kit/three-math.ts +145 -0
  83. package/src/remotion/kit/transition-math.ts +116 -0
  84. package/src/remotion/kit/ui-math.ts +145 -0
  85. package/src/remotion/kit/ui-theme.ts +25 -0
  86. package/src/remotion/kit/word-anchor.ts +107 -0
  87. package/src/render/contact-sheet.ts +39 -0
  88. package/src/render/continuity.ts +14 -4
  89. package/src/render/deps.ts +15 -3
  90. package/src/render/render.ts +15 -8
  91. package/src/render/sound-notes.ts +106 -0
  92. package/src/render/word-check.ts +181 -0
  93. package/src/testing/conformance.ts +12 -0
  94. package/src/testing/fake-api.ts +4 -4
  95. package/src/testing/fixtures.ts +3 -0
@@ -1,21 +1,26 @@
1
1
  import { execFile } from "node:child_process";
2
- import { mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
3
- import { join } from "node:path";
2
+ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
3
+ import { join, resolve } from "node:path";
4
4
  import { promisify } from "node:util";
5
5
  import type { Ctx, Result } from "../context";
6
- import type { AssetManifest, ScenePlan } from "../pipeline/schema";
7
- import { compareFrames, SAMPLE_WIDTH, summariseContinuity, type Boundary } from "../render/continuity";
6
+ import { isVoiceless, type AssetManifest, type ScenePlan } from "../pipeline/schema";
7
+ import { compareAcross, SAMPLE_WIDTH, summariseContinuity, type Boundary } from "../render/continuity";
8
8
  import { buildManifest, missingAssets } from "../project/manifest";
9
+ import { readBackground } from "../project/background";
9
10
  import { beatReport } from "../project/music";
10
11
  import { FILES, openProject, type Project } from "../project/project";
11
12
  import { structureNotes } from "../pipeline/review";
12
13
  import type { SearchRecord } from "./assets";
13
14
  import { libraryUse, shareAfterRender } from "./components";
14
15
  import { mediaUrls, serveDir } from "../project/serve";
16
+ import { contactSheet } from "../render/contact-sheet";
15
17
  import { bundleProject, disposeBundle, ensureRenderBrowser, renderStills, renderVideo, writeEntry, type EnsureBrowser } from "../render/render";
16
18
  import { MASTER_LUFS, MASTER_TRUE_PEAK_DB, masterLoudness, type Master, type Mastered } from "../render/master";
19
+ import { detailLines, measureSound, soundAdvice, spokenWords, voiceAdvice, type SoundReport, type SpokenWord } from "../project/soundreport";
20
+ import { soundNotes } from "../render/sound-notes";
17
21
  import { FILE_NAME, MAIN_FILE, jsxUses, staticCheck, typecheck } from "../render/validate";
18
22
  import { loadPlan } from "./plan";
23
+ import { MAX_WORD_FRAMES, usesWordTiming, WORD_FRAME_AFTER, wordCalls, wordErrors, wordTargets } from "../render/word-check";
19
24
 
20
25
  const exec = promisify(execFile);
21
26
  const MEDIA_PATH = /["'`](assets\/[^"'`\s]+\.(?:mp3|wav|m4a|mp4|mov|webm|png|jpe?g|webp|svg|gif))["'`]/g;
@@ -35,6 +40,13 @@ function inspect(project: Project): { errors: string[]; manifest?: AssetManifest
35
40
  if (!project.exists(m[1])) errors.push(`${name} uses "${m[1]}", which is not in this project. Get it with \`reelkit assets pull <id>\` first.`);
36
41
  }
37
42
  }
43
+ // A word given to wordFrame, onWord or onWordBeat that its scene never says would stop the render, so it is an error here.
44
+ if (!isVoiceless(plan)) {
45
+ const ids = plan.scenes.map((s) => s.id);
46
+ for (const name of names.filter((n) => FILE_NAME.test(n))) {
47
+ errors.push(...wordErrors(name, wordCalls(readFileSync(project.path(join("src", name)), "utf8"), ids, name), plan.scenes));
48
+ }
49
+ }
38
50
  if (errors.length) return { errors };
39
51
  const typeErrors = typecheck(names.map((n) => project.path(join("src", n))));
40
52
  if (typeErrors.length) return { errors: typeErrors };
@@ -43,14 +55,39 @@ function inspect(project: Project): { errors: string[]; manifest?: AssetManifest
43
55
  }
44
56
 
45
57
  // Advice about a composition that passes: never an error.
46
- function compositionNotes(project: Project, plan: ScenePlan): string[] {
58
+ function compositionNotes(project: Project, plan: ScenePlan, manifest?: AssetManifest): string[] {
47
59
  const notes: string[] = [];
48
60
  const names = readdirSync(project.path("src")).filter((f) => FILE_NAME.test(f));
49
61
  const sources = names.map((n) => [n, readFileSync(project.path(join("src", n)), "utf8")] as const);
50
62
  const rendersCaptions = sources.some(([n, src]) => jsxUses(src, n, "Captions").length > 0);
51
63
  if (plan.captions === "none" && rendersCaptions) notes.push('The plan says captions: "none" but the composition renders <Captions>. Remove them, or change the plan if the user wants captions.');
52
64
  if ((plan.captions === "word" || plan.captions === "phrase") && !rendersCaptions) notes.push(`The plan asks for captions (${plan.captions}) but the composition renders no <Captions>. Add <Captions words={s.words} group={manifest.captions} /> in each scene, or set captions to "none" in the plan if the user does not want them.`);
65
+ if (!isVoiceless(plan) && !sources.some(([n, src]) => usesWordTiming(src, n))) {
66
+ notes.push("Nothing on screen is timed to a spoken word: the composition never uses wordFrame, onWord, onWordBeat or a scene's words (other than to draw the subtitles). Place what illustrates a word on that word, e.g. <Entrance delay={onWord(s, \"tasks\")}>; see reference/voice-sync.md.");
67
+ }
53
68
  notes.push(...structureNotes(plan, { footage: project.footage() }));
69
+ if (isVoiceless(plan) && manifest) {
70
+ notes.push(...soundNotes(sources.map(([name, source]) => ({ name, source })), { seconds: manifest.totalFrames / manifest.fps, fps: manifest.fps, musicSet: manifest.music !== undefined }));
71
+ }
72
+ // A ground that was chosen and then never drawn is wasted: say so. A video ground with a low dim under type is hard to read.
73
+ const bg = readBackground(project);
74
+ const recorded = [...(bg.film ? [bg.film.key] : []), ...Object.values(bg.scenes).map((e) => e.key)];
75
+ const usesBackground = (src: string) => /\.background\b|bgPerScene/.test(src) || recorded.some((k) => src.includes(k));
76
+ if (recorded.length && !sources.some(([, src]) => usesBackground(src))) notes.push(`A background is recorded (${recorded[0]}) but the composition never uses it. Draw it with <BgVideo> or <BgImage> from manifest.background, or <BgSequence items={bgPerScene(...)}>; see reference/backgrounds.md.`);
77
+ if (plan.scenes.some((s) => s.onScreenText.length)) {
78
+ for (const [name, src] of sources) {
79
+ const lowDim = jsxUses(src, name, "BgVideo").some((u) => { const m = /\bdim=\{\s*(\d*\.?\d+)\s*\}/.exec(u.example); return m !== null && Number(m[1]) < 0.2; });
80
+ if (lowDim) { notes.push(`${name}: a BgVideo with a dim under 0.2 sits behind on-screen text, which will be hard to read. Raise dim to 0.3 or more, or blur the clip.`); break; }
81
+ }
82
+ }
83
+ // An image scene with words on it is built in layers. Decidable only when the composition names one of the two components, or names the scene's image (its key, or
84
+ // imageKey): a composition that never mentions the picture says nothing here.
85
+ const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
86
+ const layered = sources.some(([n, src]) => jsxUses(src, n, "ImageLayers").length > 0 || jsxUses(src, n, "TextOnImage").length > 0);
87
+ const withText = plan.scenes.filter((sc) => sc.treatment === "illustration" && sc.onScreenText.length > 0 && images[sc.id]);
88
+ if (!layered && withText.length && sources.some(([, src]) => /\bimageKey\b/.test(src) || withText.some((sc) => src.includes(images[sc.id]!)))) {
89
+ notes.push(`Scene ${withText.map((sc) => sc.id).join(", ")} ${withText.length === 1 ? "has" : "have"} an image and on-screen text, but the composition uses neither <ImageLayers> nor <TextOnImage>: the text is not a layer of the picture. Put the words in the picture's calm zone with <TextOnImage layers={s.imageLayers}> inside <ImageLayers layers={s.imageLayers}> (run \`reelkit assets layers --scene <id>\` for a hero picture so the words can pass behind its subject), not on a flat card beside it.`);
90
+ }
54
91
  const { pulled, written } = libraryUse(project);
55
92
  if (!pulled.length && plan.scenes.length >= 4) {
56
93
  const searched = project.readJsonOr<SearchRecord[]>(FILES.searches, []).filter((s) => s.kind === "component").length;
@@ -63,9 +100,9 @@ const failed = (errors: string[]): Result => ({ ok: false, data: { errors }, sum
63
100
 
64
101
  export async function check(ctx: Ctx): Promise<Result> {
65
102
  const project = openProject(ctx.cwd);
66
- const { errors } = inspect(project);
103
+ const { errors, manifest } = inspect(project);
67
104
  if (errors.length) return failed(errors);
68
- const shouldImprove = compositionNotes(project, loadPlan(project));
105
+ const shouldImprove = compositionNotes(project, loadPlan(project), manifest);
69
106
  return {
70
107
  ok: true, data: { errors: [], shouldImprove, library: libraryUse(project) },
71
108
  summary: ["The composition passes. Run `reelkit preview` to see it.", ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : [])].join("\n"),
@@ -102,33 +139,54 @@ export async function ensureBrowserWithNotice(ctx: Ctx, ensure: EnsureBrowser =
102
139
  const ffmpegToJpeg = (png: string, jpg: string) => exec("ffmpeg", ["-y", "-loglevel", "error", "-i", png, "-vf", "scale=540:-2", "-q:v", "5", jpg]).then(() => undefined);
103
140
 
104
141
  type PreviewPoint = { sceneId: string; point: "early" | "late" | "middle" | "end" | "start"; frame: number; name: string };
142
+ type WordFrameFile = { sceneId: string; word: string; label: string; frame: number; wordStart: number; name: string };
143
+
144
+ // Whether any file of the composition draws a Scene3D, which is the only case that needs the software GL in the browser.
145
+ export const usesScene3D = (project: Project): boolean => readdirSync(project.path("src")).filter((f) => f.endsWith(".tsx")).some((n) => /<Scene3D[\s>/]/.test(readFileSync(project.path(join("src", n)), "utf8")));
105
146
 
106
- // Two frames per scene, at 30% and 90% of its length, so what enters late in a scene is seen as well as what opens it.
107
- // A scene under 4 frames has no room for two distinct frames and gets its middle one. The scene's number leads each name
108
- // so the files sort by scene order and then by time, whatever the scene ids are.
109
- export function previewPoints(scenes: { id: string; startFrame: number; durationFrames: number }[]): PreviewPoint[] {
147
+ // How many frames at each end of a scene SceneFrame may be fading or moving: 8 unless the composition asks for longer with a literal
148
+ // transitionFrames={N} (at most 20, as SceneFrame allows). A value that is not a literal cannot be read, so the default is used.
149
+ export const DEFAULT_FADE = 8;
150
+ export function fadeFrames(sources: string[]): number {
151
+ let n = DEFAULT_FADE;
152
+ for (const src of sources) for (const m of src.matchAll(/transitionFrames=\{\s*(\d+)\s*\}/g)) n = Math.max(n, Math.min(20, Number(m[1])));
153
+ return n;
154
+ }
155
+
156
+ // The frames of a scene, counted from its start, where it is fully visible: SceneFrame fades or moves it at both ends for up to `fade` frames,
157
+ // but never more than half its length.
158
+ const visibleRange = (d: number, fade: number) => { const edge = Math.max(1, Math.min(fade, Math.floor(d / 2))); return { lo: edge, hi: d - edge, edge }; };
159
+
160
+ // Two frames per scene, at 30% and 90% of its length, so what enters late in a scene is seen as well as what opens it, both moved into the
161
+ // part where the scene is fully visible (a quick scene's 90% falls inside its fade and would look dim). A scene too short to have two distinct
162
+ // fully visible frames gets its middle one. The scene's number leads each name so the files sort by scene order and then by time.
163
+ export function previewPoints(scenes: { id: string; startFrame: number; durationFrames: number }[], fade = DEFAULT_FADE): PreviewPoint[] {
110
164
  return scenes.flatMap((s, i) => {
111
165
  const at = (point: PreviewPoint["point"], offset: number): PreviewPoint => ({ sceneId: s.id, point, frame: s.startFrame + offset, name: `${String(i + 1).padStart(2, "0")}-${s.id}-${point}` });
112
- if (s.durationFrames < 4) return [at("middle", Math.floor(s.durationFrames / 2))];
113
- const inside = (share: number) => Math.min(s.durationFrames - 1, Math.round(s.durationFrames * share));
166
+ const { lo, hi } = visibleRange(s.durationFrames, fade);
167
+ if (s.durationFrames < 4 || hi - lo < 2) return [at("middle", Math.floor(s.durationFrames / 2))];
168
+ const inside = (share: number) => Math.max(lo, Math.min(hi, Math.round(s.durationFrames * share)));
114
169
  return [at("early", inside(0.3)), at("late", inside(0.9))];
115
170
  });
116
171
  }
117
172
 
118
- type BoundaryPoint = { from: string; to: string; end: PreviewPoint; start: PreviewPoint };
173
+ type BoundaryPoint = { from: string; to: string; end: PreviewPoint; start: PreviewPoint; settled: PreviewPoint };
119
174
 
120
- // SceneFrame fades each scene in and out over up to 8 frames, so the very last and very first frames of two scenes are nearly empty.
121
- // The frames that show what a scene really holds are the last one at full strength and the first one at full strength: those are the two
122
- // either side of every change. The files are named by the change, b01-end-<scene> then b01-start-<scene>, so that they sort in order.
123
- export function boundaryPoints(scenes: { id: string; startFrame: number; durationFrames: number }[]): BoundaryPoint[] {
124
- const edge = (d: number) => Math.max(1, Math.min(8, Math.floor(d / 2)));
175
+ // SceneFrame fades or moves each scene in and out, so the very last and very first frames of two scenes are nearly empty. The frames that show
176
+ // what a scene really holds are the last one at full strength and the first one at full strength. A third frame, six frames after the first,
177
+ // catches a move that is still settling; it is measured but not saved. The files are named by the change, b01-end-<scene> then b01-start-<scene>.
178
+ export function boundaryPoints(scenes: { id: string; startFrame: number; durationFrames: number }[], fade = DEFAULT_FADE): BoundaryPoint[] {
125
179
  return scenes.slice(0, -1).map((a, i) => {
126
180
  const b = scenes[i + 1]!;
127
181
  const n = `b${String(i + 1).padStart(2, "0")}`;
182
+ const ea = visibleRange(a.durationFrames, fade), eb = visibleRange(b.durationFrames, fade);
183
+ const startOffset = Math.min(b.durationFrames - 1, eb.edge);
184
+ const settledOffset = Math.min(b.durationFrames - 1, Math.max(startOffset, Math.min(startOffset + 6, eb.hi)));
128
185
  return {
129
186
  from: a.id, to: b.id,
130
- end: { sceneId: a.id, point: "end", frame: a.startFrame + Math.max(0, a.durationFrames - edge(a.durationFrames)), name: `${n}-end-${a.id}` },
131
- start: { sceneId: b.id, point: "start", frame: b.startFrame + Math.min(b.durationFrames - 1, edge(b.durationFrames)), name: `${n}-start-${b.id}` },
187
+ end: { sceneId: a.id, point: "end", frame: a.startFrame + Math.max(0, ea.hi), name: `${n}-end-${a.id}` },
188
+ start: { sceneId: b.id, point: "start", frame: b.startFrame + startOffset, name: `${n}-start-${b.id}` },
189
+ settled: { sceneId: b.id, point: "start", frame: b.startFrame + settledOffset, name: `${n}-settled-${b.id}` },
132
190
  };
133
191
  });
134
192
  }
@@ -147,17 +205,25 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
147
205
  const dir = project.path("out/preview");
148
206
  rmSync(dir, { recursive: true, force: true });
149
207
  mkdirSync(dir, { recursive: true });
150
- const sceneFrames = previewPoints(manifest.scenes);
151
- const boundaries = boundaryPoints(manifest.scenes);
208
+ const plan = loadPlan(project);
209
+ const intended = new Set(plan.scenes.filter((s) => s.cutIn).map((s) => s.id));
210
+ const fade = fadeFrames(readdirSync(project.path("src")).filter((f) => f.endsWith(".tsx")).map((n) => readFileSync(project.path(join("src", n)), "utf8")));
211
+ const sceneFrames = previewPoints(manifest.scenes, fade);
212
+ const boundaries = boundaryPoints(manifest.scenes, fade);
152
213
  const points = [...sceneFrames, ...boundaries.flatMap((b) => [b.end, b.start])];
214
+ // A narrated film also gets one frame just after each word its code times something to: the thing the word names should already be there.
215
+ const srcNames = readdirSync(project.path("src")).filter((f) => FILE_NAME.test(f));
216
+ const wordPlan = isVoiceless(plan) ? undefined : wordTargets(srcNames.flatMap((n) => wordCalls(readFileSync(project.path(join("src", n)), "utf8"), plan.scenes.map((sc) => sc.id), n)), manifest);
217
+ const wordFiles: WordFrameFile[] = (wordPlan?.targets ?? []).map((t, i) => ({ ...t, name: `w${String(i + 1).padStart(2, "0")}-${t.sceneId}-${t.word.toLowerCase().replace(/[^a-z0-9]+/g, "") || "word"}` }));
218
+ const measuredFrames = [...points.map((p) => p.frame), ...boundaries.map((b) => b.settled.frame), ...wordFiles.map((w) => w.frame)];
153
219
  try {
154
220
  try {
155
221
  await ensureBrowserWithNotice(ctx, deps.ensureBrowser);
156
- await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, [...new Set(points.map((p) => p.frame))], dir));
222
+ await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, [...new Set(measuredFrames)], dir, { uses3D: usesScene3D(project) }));
157
223
  } catch (e) {
158
224
  return failed([`Render failed: ${e instanceof Error ? e.message : String(e)}`]);
159
225
  }
160
- const frames: { sceneId: string; point: PreviewPoint["point"]; path: string }[] = [];
226
+ const frames: { sceneId: string; point: PreviewPoint["point"] | "word"; path: string; word?: string; label?: string }[] = [];
161
227
  try {
162
228
  for (const m of points) {
163
229
  // Shrunk to phone width: small enough to read cheaply, and what is unreadable here is unreadable on a phone.
@@ -165,9 +231,23 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
165
231
  await toJpeg(join(dir, `still-${m.frame}.png`), project.path(path));
166
232
  frames.push({ sceneId: m.sceneId, point: m.point, path });
167
233
  }
234
+ for (const w of wordFiles) {
235
+ const path = `out/preview/${w.name}.jpg`;
236
+ await toJpeg(join(dir, `still-${w.frame}.png`), project.path(path));
237
+ frames.push({ sceneId: w.sceneId, point: "word", path, word: w.word, label: w.label });
238
+ }
168
239
  } catch {
169
240
  return failed(["Could not write the preview frames: ffmpeg failed. Check that ffmpeg is installed (macOS: `brew install ffmpeg`) and run `reelkit preview` again."]);
170
241
  }
242
+ // One picture of all the frames, so that a look at it comes before opening single frames. It is a convenience: failing to make it changes nothing else.
243
+ let sheetLine = "";
244
+ let sheet: { path: string; cols: number; rows: number } | undefined;
245
+ try {
246
+ const names = frames.map((f) => (f.point === "word" ? f.label! : `${f.sceneId} ${f.point === "early" || f.point === "late" || f.point === "middle" ? f.point : `change ${f.point}`}`));
247
+ const r = await contactSheet(frames.map((f) => project.path(f.path)), names, project.path("out/preview/sheet.jpg"), manifest.width / manifest.height, { newRowAt: wordFiles.length ? points.length : undefined });
248
+ sheet = { path: "out/preview/sheet.jpg", cols: r.cols, rows: r.rows };
249
+ sheetLine = `Contact sheet, one picture of all ${frames.length} frames in the order below${r.labelled ? ", each labelled with its scene and early, late or change" : ""} (${r.cols} across${wordFiles.length ? `; the word frames are in the last row${r.labelled ? ", labelled word > scene" : ""}` : ""}): out/preview/sheet.jpg. Look at it first, then open single frames where something needs a closer look.\n`;
250
+ } catch { /* no sheet */ }
171
251
  // Whether anything survives each scene change is advice. If it cannot be measured, the preview is still the preview.
172
252
  let continuity: { score: number | null; boundaries: Boundary[] } | undefined;
173
253
  let continuityLines: string[] = [];
@@ -175,9 +255,9 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
175
255
  const h = Math.round((SAMPLE_WIDTH * manifest.height) / manifest.width);
176
256
  const measured: Boundary[] = [];
177
257
  for (const b of boundaries) {
178
- const [a, c] = await Promise.all([decodeGray(join(dir, `still-${b.end.frame}.png`), SAMPLE_WIDTH, h), decodeGray(join(dir, `still-${b.start.frame}.png`), SAMPLE_WIDTH, h)]);
179
- const r = compareFrames(a, c, SAMPLE_WIDTH, h);
180
- measured.push({ from: b.from, to: b.to, survived: Math.round(r.survived * 100) / 100, kind: r.kind });
258
+ const [a, c, d] = await Promise.all([b.end.frame, b.start.frame, b.settled.frame].map((f) => decodeGray(join(dir, `still-${f}.png`), SAMPLE_WIDTH, h)));
259
+ const r = compareAcross(a!, [c!, d!], SAMPLE_WIDTH, h);
260
+ measured.push({ from: b.from, to: b.to, survived: Math.round(r.survived * 100) / 100, kind: intended.has(b.to) && r.kind !== "empty" ? "cut (intended)" : r.kind });
181
261
  }
182
262
  if (measured.length) {
183
263
  const summary = summariseContinuity(measured);
@@ -185,24 +265,27 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
185
265
  continuityLines = summary.lines;
186
266
  }
187
267
  } catch { /* no continuity report */ }
188
- const sceneLines = `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.filter((f) => f.point !== "end" && f.point !== "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}`;
268
+ const wordLines = wordPlan && (wordFiles.length || wordPlan.skipped)
269
+ ? `\nWord frames: ${wordFiles.length} frame${wordFiles.length === 1 ? "" : "s"}, each ${WORD_FRAME_AFTER} frames AFTER its word starts, in a row of their own on the sheet (labelled "word ▸ scene"). Each word frame should already show the thing that word names: if it does not, the thing is late (or on the wrong word) and its timing needs fixing.${wordFiles.length ? `\n${wordFiles.map((w) => `${w.label}: out/preview/${w.name}.jpg`).join("\n")}` : ""}${wordPlan.skipped ? `\n${wordPlan.skipped} word timing${wordPlan.skipped === 1 ? " was" : "s were"} not shown (${[wordPlan.undecidable ? `${wordPlan.undecidable} name${wordPlan.undecidable === 1 ? "s" : ""} a word or a scene the code does not give as a literal` : "", wordPlan.capped ? `${wordPlan.capped} beyond the first ${MAX_WORD_FRAMES}` : ""].filter(Boolean).join(", ")}).` : ""}`
270
+ : wordPlan ? "\nWord frames: none, because the composition has no wordFrame, onWord or onWordBeat call with a literal scene and word." : "";
271
+ const sceneLines = `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.filter((f) => f.point !== "end" && f.point !== "start" && f.point !== "word").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}`;
189
272
  const changeLines = boundaries.length ? `\nThe last and first frames either side of each scene change are saved too (b01-end-<scene>, b01-start-<scene>, ...), so that you can see what carries across:\n${frames.filter((f) => f.point === "end" || f.point === "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}` : "";
190
- const beat = beatReport(manifest);
191
- return { ok: true, data: { frames, ...(continuity ? { continuity } : {}), ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) }, summary: `${sceneLines}${changeLines}${continuityLines.length ? `\nContinuity: ${continuityLines.join("\n")}` : ""}${beat ? `\nBeat: ${beat.line}.` : ""}` };
273
+ const beat = beatReport(manifest, { finer: isVoiceless(plan) });
274
+ return { ok: true, data: { frames, ...(wordPlan ? { wordFrames: { shown: wordFiles.map((w) => ({ scene: w.sceneId, word: w.word, wordStart: w.wordStart, frame: w.frame })), skipped: wordPlan.skipped } } : {}), ...(sheet ? { sheet } : {}), ...(continuity ? { continuity } : {}), ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) }, summary: `${sheetLine}${sceneLines}${changeLines}${wordLines}${continuityLines.length ? `\nContinuity: ${continuityLines.join("\n")}` : ""}${beat ? `\nBeat: ${beat.line}.` : ""}` };
192
275
  } finally {
193
276
  // The full-size stills are only inputs; none stays behind however this ends.
194
277
  for (const f of readdirSync(dir)) if (/^still-\d+\.png$/.test(f)) rmSync(join(dir, f), { force: true });
195
278
  }
196
279
  }
197
280
 
198
- export async function render(ctx: Ctx, deps: { ensureBrowser?: EnsureBrowser; master?: Master; noShare?: boolean } = {}): Promise<Result> {
281
+ export async function render(ctx: Ctx, deps: { ensureBrowser?: EnsureBrowser; master?: Master; noShare?: boolean; measure?: (path: string, loudness?: number, words?: SpokenWord[]) => Promise<SoundReport> } = {}): Promise<Result> {
199
282
  const project = openProject(ctx.cwd);
200
283
  const { errors, manifest } = inspect(project);
201
284
  if (!manifest) return failed(errors);
202
285
  const path = "out/video.mp4";
203
286
  try {
204
287
  await ensureBrowserWithNotice(ctx, deps.ensureBrowser);
205
- await withBundle(project, (serveUrl, urls) => renderVideo(serveUrl, { manifest, urls }, project.path(path)));
288
+ await withBundle(project, (serveUrl, urls) => renderVideo(serveUrl, { manifest, urls }, project.path(path), { uses3D: usesScene3D(project) }));
206
289
  } catch (e) {
207
290
  return failed([`Render failed: ${e instanceof Error ? e.message : String(e)}`]);
208
291
  }
@@ -216,9 +299,39 @@ export async function render(ctx: Ctx, deps: { ensureBrowser?: EnsureBrowser; ma
216
299
  } catch (e) {
217
300
  masterNote = ` It was not mastered to ${MASTER_LUFS} LUFS (${e instanceof Error ? e.message : String(e)}); the video is kept as rendered.`;
218
301
  }
302
+ // How the finished film sounds against what it shows. It is advice, so a measurement that fails leaves the render as it was.
303
+ let sound: SoundReport | undefined;
304
+ let soundLine: string | undefined;
305
+ try {
306
+ const voiceless = isVoiceless(loadPlan(project));
307
+ sound = await (deps.measure ?? measureSound)(project.path(path), loudness?.outputLufs, voiceless ? undefined : spokenWords(manifest));
308
+ if (voiceless) soundLine = soundAdvice(sound);
309
+ else if (sound.voice) soundLine = voiceAdvice(sound.voice);
310
+ } catch { sound = undefined; }
219
311
  const share = await shareAfterRender(ctx, project, { noShare: deps.noShare });
220
312
  return {
221
- ok: true, data: { path, seconds, ...(loudness ? { loudness } : {}), ...(share.data ? { shared: share.data.shared.map((s) => s.name) } : {}) },
222
- summary: [`Video: ${project.path(path)} (${seconds.toFixed(1)}s, ${manifest.width}x${manifest.height}).${masterNote}`, ...share.lines].join("\n"),
313
+ ok: true, data: { path, seconds, ...(loudness ? { loudness } : {}),
314
+ ...(sound ? { sound: { hitsPerSecond: sound.hitsPerSecond, swellsPerSecond: sound.swellsPerSecond, pictureChanges: sound.pictureChanges, changesWithHit: sound.changesWithHit, firstSoundSec: sound.firstSoundSec }, ...(sound.voice ? { voice: sound.voice } : {}) } : {}), ...(share.data ? { shared: share.data.shared.map((s) => s.name) } : {}) },
315
+ summary: [`Video: ${project.path(path)} (${seconds.toFixed(1)}s, ${manifest.width}x${manifest.height}).${masterNote}`, ...(soundLine ? [soundLine] : []), ...share.lines].join("\n"),
223
316
  };
224
317
  }
318
+
319
+ // `reelkit sound [file]`: the same measurement on any film, by default the last render. `--detail` lists every picture change with its nearest hit, every swell and the first sound.
320
+ export async function soundCommand(ctx: Ctx, file?: string, opts: { detail?: boolean } = {}, deps: { measure?: (path: string, loudness?: number, words?: SpokenWord[]) => Promise<SoundReport> } = {}): Promise<Result> {
321
+ let project: Project | undefined;
322
+ try { project = openProject(ctx.cwd); } catch { project = undefined; }
323
+ const path = file ? resolve(ctx.cwd, file) : project ? project.path("out/video.mp4") : resolve(ctx.cwd, "out/video.mp4");
324
+ if (!existsSync(path)) return { ok: false, summary: `${file ?? "out/video.mp4"} does not exist. ${file ? "Check the path." : "Run `reelkit render` first, or give the file."}` };
325
+ let narrated = false;
326
+ try { narrated = project ? !isVoiceless(loadPlan(project)) : false; } catch { narrated = false; }
327
+ // A narrated film is also measured against its words: the manifest has their times.
328
+ let words: SpokenWord[] | undefined;
329
+ if (narrated && project?.exists(FILES.manifest) && (!file || resolve(ctx.cwd, file) === resolve(project.path("out/video.mp4")))) { try { words = spokenWords(project.readJson<AssetManifest>(FILES.manifest)); } catch { words = undefined; } }
330
+ let report: SoundReport;
331
+ try { report = await (deps.measure ?? measureSound)(path, undefined, words); } catch (e) { return { ok: false, summary: `Could not measure the sound: ${e instanceof Error ? e.message : String(e)}` }; }
332
+ const numbers = `${report.hitsPerSecond} hits a second, ${report.swellsPerSecond} swells a second, ${report.changesWithHit} of ${report.pictureChanges} picture changes with a hit within 0.1s, first sound at ${report.firstSoundSec ?? "never"}s${report.loudnessLufs !== undefined ? `, ${report.loudnessLufs} LUFS` : ""}.`;
333
+ const { detail, ...counts } = report;
334
+ const body = narrated ? `${numbers}${report.voice ? `\n${voiceAdvice(report.voice)}` : ""}` : `${soundAdvice(report)}\n${numbers}`;
335
+ if (opts.detail && detail) return { ok: true, data: { path, ...counts, detail }, summary: `${body}\n${detailLines(detail).join("\n")}` };
336
+ return { ok: true, data: { path, ...counts }, summary: `${body}\nRun with --detail to see each event.` };
337
+ }
@@ -31,7 +31,7 @@ export async function init(ctx: Ctx, name: string | undefined, opts: { aspect?:
31
31
  summary: [
32
32
  `Project ready in ${dir} (${project.config().aspect}).${slug ? ` Next: cd ${slug}` : ""}`,
33
33
  project.config().shareComponents === false
34
- ? "Nothing from this project is shared automatically."
34
+ ? "This project is private: nothing from it is sent to the library, not its components and not the images, clips or graphics generated for it, whatever the plan says."
35
35
  : "After a render, new components written in this project are sent to the Reelkit library for review (their source, a description and an example; nothing else). To turn this off: `reelkit init <name> --private` for a new project, `reelkit render --no-share`, or REELKIT_NO_SHARE=1.",
36
36
  ].join("\n"),
37
37
  };
@@ -17,7 +17,7 @@ export async function install(ctx: Ctx, opts: { agent?: string; force?: boolean
17
17
  }
18
18
  const name = (id: string) => AGENTS.find((a) => a.id === id)!.name;
19
19
  const lines = [
20
- ...report.installed.map((i) => `Installed the Reelkit skill for ${name(i.agent)}: ${i.path}`),
20
+ ...report.installed.map((i) => `Installed the Reelkit skill for ${name(i.agent)}: ${i.path}${i.commands.length ? ` and the commands ${i.commands.map((c) => `/${c}`).join(" and ")}` : ""}`),
21
21
  ...report.skipped.filter((s) => agents || s.reason !== "not installed on this machine").map((s) => `Skipped ${name(s.agent)}: ${s.reason}`),
22
22
  ];
23
23
  return { ok: true, data: report, summary: lines.join("\n") };
@@ -1,7 +1,8 @@
1
1
  import { client, type Ctx, type Result } from "../context";
2
2
  import { loadCredentials } from "../credentials";
3
3
  import { estimateLength, lengthVariation, reviewPlan, sceneSeconds } from "../pipeline/review";
4
- import { ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
4
+ import { isVoiceless, ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
5
+ import { snappedSeconds } from "../project/manifest";
5
6
  import { FILES, issueLines, openProject, type Project } from "../project/project";
6
7
 
7
8
  export function loadPlan(project: Project): ScenePlan {
@@ -52,7 +53,8 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
52
53
  }
53
54
  const footage = project.footage();
54
55
  const assets = project.assets().filter((a) => a.id !== footage?.id);
55
- const voiceIds = loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
56
+ const voiceless = isVoiceless(parsed.data);
57
+ const voiceIds = !voiceless && loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
56
58
  const ref = referenceChecks(project, parsed.data);
57
59
  const mustFix = [...validatePlan(parsed.data, { aspect: project.config().aspect, footage, assets, voiceIds }), ...ref.mustFix];
58
60
  const shouldImprove = mustFix.length ? [] : [...reviewPlan(parsed.data, { footage }), ...ref.improve];
@@ -60,11 +62,12 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
60
62
  const estimatedSeconds = Math.round(length.seconds);
61
63
  const perScene = sceneSeconds(parsed.data).map((s) => ({ id: s.id, seconds: Math.round(s.seconds * 10) / 10 }));
62
64
  const variation = Math.round(lengthVariation(sceneSeconds(parsed.data).map((s) => s.seconds)) * 100) / 100;
65
+ const snapped = voiceless ? snappedSeconds(project, parsed.data) : undefined;
63
66
  const lines = [
64
67
  mustFix.length ? `Fix these:\n- ${mustFix.join("\n- ")}` : "The plan is valid.",
65
- `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
68
+ voiceless ? `Length: ${(Math.round(length.seconds * 10) / 10).toFixed(1)}s over ${parsed.data.scenes.length} scenes${snapped !== undefined ? `; about ${snapped.toFixed(1)}s once the cuts are on the beat` : ""}.` : `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
66
69
  ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : []),
67
- ...(voiceIds ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
70
+ ...(voiceIds || voiceless ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
68
71
  ];
69
- return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation }, summary: lines.join("\n") };
72
+ return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation, ...(snapped !== undefined ? { snappedSeconds: Math.round(snapped * 10) / 10 } : {}) }, summary: lines.join("\n") };
70
73
  }
@@ -60,11 +60,14 @@ const realYtdlp: YtdlpRunner = {
60
60
 
61
61
  const YTDLP_MISSING = "yt-dlp is not installed, and links need it. Install it (macOS: `brew install yt-dlp`, or `pipx install yt-dlp`) and run the command again; a video file on this machine works without it.";
62
62
 
63
+ // The duration filter uses "<=?": some sites (Instagram) give no duration before the download, and without the "?" such a video is
64
+ // skipped as if it were too long. A video that turns out longer is refused after it is probed.
65
+ // No "--max-downloads 1": yt-dlp exits with an error code after reaching it even when the one video was saved.
63
66
  // One video, at most 1080p, as mp4, and nothing longer than the limit: the flags that keep a link from fetching a playlist, a channel or a huge file.
64
67
  export function ytdlpArgs(url: string, dest: string): string[] {
65
68
  return [
66
- "--ignore-config", "--no-playlist", "--max-downloads", "1", "--no-progress", "--no-warnings",
67
- "--match-filter", `duration<=${MAX_REF_SECONDS}`,
69
+ "--ignore-config", "--no-playlist", "--no-progress", "--no-warnings",
70
+ "--match-filter", `duration<=?${MAX_REF_SECONDS}`,
68
71
  "-f", "bv*[height<=1080][ext=mp4]+ba[ext=m4a]/b[height<=1080][ext=mp4]/bv*[height<=1080]+ba/b[height<=1080]",
69
72
  "--merge-output-format", "mp4", "-o", dest, "--", url,
70
73
  ];
@@ -210,10 +210,12 @@ export const routes = {
210
210
  voiceover: route("POST", "/voiceover", true,
211
211
  z.object({ text: text(5000).refine((t) => /\S/.test(t), "must contain a non-space character"), voiceId: z.string().regex(/^[A-Za-z0-9]{1,64}$/).optional(), speed: z.number().min(0.7).max(1.3).optional() }),
212
212
  z.object({ url: z.string(), ext: z.enum(["mp3", "wav", "m4a"]), contentType: z.string(), durationSec: z.number().positive(), words: z.array(WordTimingSchema), chars: z.number() })),
213
+ // `format` is "png" (a raster image, the default) or "svg" (a vector graphic). An SVG counts as one image against the image quota, takes about 45 s, and the
214
+ // server returns only a cleaned SVG: no scripts, no event attributes, no outside references. Its `ext` is "svg" and its contentType `image/svg+xml`.
213
215
  images: route("POST", "/images", true,
214
- z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12) }),
216
+ z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12), format: z.enum(["png", "svg"]).default("png") }),
215
217
  // libraryId is set when the image was also added to the shared library.
216
- z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp"]), contentType: z.string(), libraryId: z.string().optional() })),
218
+ z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp", "svg"]), contentType: z.string(), libraryId: z.string().optional() })),
217
219
  clipStart: route("POST", "/clips", true,
218
220
  z.object({
219
221
  prompt: text(2000).min(1), aspect: AspectSchema, greenScreen: z.boolean().default(false), durationSec: z.union([z.literal(5), z.literal(10)]).default(5),
@@ -12,6 +12,9 @@ export type SnapInput = {
12
12
  // Beats as frame numbers, ascending. They should reach past the end of the video.
13
13
  beatFrames: number[];
14
14
  fps: number;
15
+ // "next" holds a scene to the next whole beat (narration cannot be cut). "nearest" moves each end to the closest point on a finer grid
16
+ // (a film with no voice can be shortened as well as lengthened, so its total stays near the plan).
17
+ grid?: "next" | "nearest";
15
18
  };
16
19
 
17
20
  const firstAtOrAfter = (beats: number[], frame: number): number | undefined => beats.find((b) => b >= frame);
@@ -21,6 +24,7 @@ const firstAtOrAfter = (beats: number[], frame: number): number | undefined => b
21
24
  export function snapToBeats(input: SnapInput): number[] {
22
25
  const { naturalFrames, beatFrames, fps } = input;
23
26
  if (beatFrames.length < 2) return [...naturalFrames];
27
+ if (input.grid === "nearest") return snapToNearest(naturalFrames, beatFrames);
24
28
  const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
25
29
  const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
26
30
  let cursor = 0;
@@ -40,6 +44,74 @@ export function snapToBeats(input: SnapInput): number[] {
40
44
  });
41
45
  }
42
46
 
47
+ // How far a scene change of a narrated film may move to reach a beat, in frames, either way.
48
+ export const NARRATED_SNAP_FRAMES = 4;
49
+ // A boundary never comes closer than this to the last word of the scene it ends, in frames.
50
+ export const NARRATED_WORD_CLEARANCE = 2;
51
+
52
+ // Scene changes for a narrated film. The voice sets where a scene ends; a boundary is moved to the nearest beat only when that is at most
53
+ // `maxMove` frames away either way and leaves at least `clearance` frames after the scene's last word (a boundary never cuts into a word). The
54
+ // next scene's audio starts on its first frame, so it moves with the boundary. Every other boundary stays where the narration puts it, and the
55
+ // last scene is not touched: the film's end is not a scene change. Each boundary is judged from where its scene actually starts, so
56
+ // nothing adds up. `lastWordEndFrames[i]` is when scene i's last word ends, in frames from its own start.
57
+ export function snapNarrated(input: { naturalFrames: number[]; lastWordEndFrames: number[]; beatFrames: number[]; maxMove?: number; clearance?: number }): { frames: number[]; onBeat: boolean[] } {
58
+ const { naturalFrames, lastWordEndFrames, beatFrames } = input;
59
+ const maxMove = input.maxMove ?? NARRATED_SNAP_FRAMES, clearance = input.clearance ?? NARRATED_WORD_CLEARANCE;
60
+ let cursor = 0;
61
+ const onBeat: boolean[] = [];
62
+ const frames = naturalFrames.map((natural, i) => {
63
+ let end = cursor + natural;
64
+ if (i < naturalFrames.length - 1) {
65
+ const floor = cursor + (lastWordEndFrames[i] ?? 0) + clearance;
66
+ let best: number | undefined;
67
+ for (const b of beatFrames) {
68
+ if (Math.abs(b - end) > maxMove || b < floor) continue;
69
+ if (best === undefined || Math.abs(b - end) < Math.abs(best - end) || (Math.abs(b - end) === Math.abs(best - end) && b > best)) best = b;
70
+ }
71
+ if (best !== undefined) end = best;
72
+ onBeat.push(beatFrames.includes(end));
73
+ }
74
+ const length = end - cursor;
75
+ cursor = end;
76
+ return length;
77
+ });
78
+ return { frames, onBeat };
79
+ }
80
+
81
+ // The beat frames with `divisions - 1` evenly spaced points between each pair, rounded to frames. The last beat is followed by one more
82
+ // at the median spacing, so a point just after it exists too.
83
+ export function gridPoints(beatFrames: number[], divisions: number): number[] {
84
+ if (beatFrames.length < 2) return [...beatFrames];
85
+ const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
86
+ const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
87
+ const all = [...beatFrames, beatFrames[beatFrames.length - 1]! + period];
88
+ const out = new Set<number>();
89
+ for (let i = 0; i + 1 < all.length; i++) for (let k = 0; k < divisions; k++) out.add(Math.round(all[i]! + ((all[i + 1]! - all[i]!) * k) / divisions));
90
+ out.add(all[all.length - 1]!);
91
+ return [...out].sort((a, b) => a - b);
92
+ }
93
+
94
+ // A scene of at least 1.5 beats ends on the nearest whole beat; a shorter one on the nearest half beat, and under half a beat the nearest
95
+ // quarter. The target is the planned end (the sum of the natural lengths), so rounding errors do not add up; a scene is never made empty.
96
+ function snapToNearest(naturalFrames: number[], beatFrames: number[]): number[] {
97
+ const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
98
+ const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
99
+ const grids = new Map<number, number[]>();
100
+ const grid = (d: number) => grids.get(d) ?? (grids.set(d, gridPoints(beatFrames, d)), grids.get(d)!);
101
+ let cursor = 0, planned = 0;
102
+ return naturalFrames.map((natural) => {
103
+ planned += natural;
104
+ const beats = natural / period;
105
+ const points = grid(beats >= 1.5 ? 1 : beats >= 0.5 ? 2 : 4).filter((f) => f > cursor);
106
+ let target = planned;
107
+ if (points.length) target = points.reduce((best, f) => (Math.abs(f - planned) < Math.abs(best - planned) ? f : best));
108
+ else target = Math.max(planned, cursor + 1);
109
+ const frames = target - cursor;
110
+ cursor = target;
111
+ return frames;
112
+ });
113
+ }
114
+
43
115
  // Beats in seconds, continued at the tempo's own spacing until `untilSec`, for a video longer than the track (the track loops).
44
116
  export function extendBeats(beats: number[], bpm: number, untilSec: number): number[] {
45
117
  if (!beats.length) return [];
@@ -1,16 +1,23 @@
1
- import { WORDS_PER_SEC, type AssetRecord, type ScenePlan } from "./schema";
1
+ import { gapSec, isVoiceless, LAST_TAIL_SEC, PACE_SPEED, VOICE_WORDS_PER_SEC, type AssetRecord, type ScenePlan } from "./schema";
2
2
 
3
3
  const words = (s: string) => s.trim().split(/\s+/).filter(Boolean).length;
4
4
 
5
- // The same estimate the length advice below is based on: narration words at the voice's usual pace.
5
+ // What the voice speaks at, in words a second, for this plan's pace.
6
+ const speechRate = (plan: ScenePlan) => VOICE_WORDS_PER_SEC * PACE_SPEED[plan.pace ?? "normal"];
7
+
8
+ // The estimate the length advice below is based on: each scene's words at the voice's real speed (2.25 words a second at the normal pace, the
9
+ // pauses within a sentence included), the plan's silence between sentences after every scene but the last, and the last scene's tail after its
10
+ // last word. It matches the manifest `reelkit assets voiceover` builds. A video with no voice has no words to count: its length is the sum of the
11
+ // scenes' own seconds.
6
12
  export function estimateLength(plan: ScenePlan): { words: number; seconds: number } {
13
+ if (isVoiceless(plan)) return { words: 0, seconds: plan.scenes.reduce((n, s) => n + (s.seconds ?? 0), 0) };
7
14
  const total = plan.scenes.reduce((n, s) => n + words(s.narration), 0);
8
- return { words: total, seconds: total / WORDS_PER_SEC };
15
+ return { words: total, seconds: total / speechRate(plan) + (plan.scenes.length - 1) * gapSec(plan) + LAST_TAIL_SEC };
9
16
  }
10
17
 
11
- // Each scene's estimated length in seconds, from its narration at the same pace as the whole-video estimate above.
18
+ // Each scene's estimated length in seconds: its speech and what follows it (the gap, or the tail for the last scene), so they add up to the whole estimate.
12
19
  export function sceneSeconds(plan: ScenePlan): { id: string; seconds: number }[] {
13
- return plan.scenes.map((s) => ({ id: s.id, seconds: words(s.narration) / WORDS_PER_SEC }));
20
+ return plan.scenes.map((s, i) => ({ id: s.id, seconds: isVoiceless(plan) ? (s.seconds ?? 0) : words(s.narration) / speechRate(plan) + (i === plan.scenes.length - 1 ? LAST_TAIL_SEC : gapSec(plan)) }));
14
21
  }
15
22
 
16
23
  // How uneven the scene lengths are: the standard deviation over the mean (0 when they are all the same).
@@ -29,12 +36,14 @@ export function rhythmNote(plan: ScenePlan): string | undefined {
29
36
  const cv = lengthVariation(values);
30
37
  if (cv >= 0.25 && values.some((v) => v < mean / 2)) return undefined;
31
38
  const shortest = lengths.reduce((a, b) => (b.seconds < a.seconds ? b : a)), longest = lengths.reduce((a, b) => (b.seconds > a.seconds ? b : a));
32
- return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). Make one or two scenes much shorter (a hit of a few words) and let one run long.`;
39
+ const fix = isVoiceless(plan) ? "Make one or two scenes much shorter (under a second) and let one run long." : "Make one or two scenes much shorter (a hit of a few words) and let one run long.";
40
+ return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). ${fix}`;
33
41
  }
34
42
 
35
43
  // Soft quality checks on a plan that already passes the hard rules in validatePlan.
36
44
  // These are things worth improving in the script; they never fail a video on their own.
37
45
  export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
46
+ if (isVoiceless(plan)) return reviewVoicelessPlan(plan, ctx);
38
47
  const issues: string[] = [];
39
48
  const { words: total, seconds } = estimateLength(plan);
40
49
 
@@ -53,7 +62,30 @@ export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): str
53
62
  // A hit of a few words is wanted in the rhythm of a video, so only a scene with almost nothing to say is flagged.
54
63
  if (!ctx.footage && n < 3) issues.push(`Scene ${s.id} has only ${n} word${n === 1 ? "" : "s"} of narration; give it at least a short phrase.`);
55
64
  if (n > 45) issues.push(`Scene ${s.id} has ${n} words of narration; split it or cut it to under 45.`);
56
- if (s.onScreenText.length > 3) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 3.`);
65
+ if (s.onScreenText.length > 5) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 5 (the items of one list on screen count as one text, but the plan cannot say which they are).`);
66
+ for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
67
+ }
68
+ const rhythm = rhythmNote(plan);
69
+ if (rhythm) issues.push(rhythm);
70
+ issues.push(...structureNotes(plan, ctx));
71
+ return issues;
72
+ }
73
+
74
+ // The same kind of advice for a video with no voice: no words to count, so length and rhythm come from `seconds`, and every word the
75
+ // viewer reads is on screen. A video of this kind needs at least one quick hit, so a plan with no scene under a second is flagged.
76
+ function reviewVoicelessPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
77
+ const issues: string[] = [];
78
+ const { seconds } = estimateLength(plan);
79
+ if (seconds < 10) issues.push(`The video runs ${seconds.toFixed(1)}s. A video with no voice needs at least 10 seconds to get across; aim for 15 to 45.`);
80
+ if (seconds > 60) issues.push(`The video runs ${seconds.toFixed(1)}s. Cut it to 60 seconds or less; aim for 15 to 45.`);
81
+ if (!plan.scenes.some((s) => (s.seconds ?? 0) < 1)) issues.push("No scene is under 1 second. A film like this needs at least one quick hit; shorten one or two scenes to under a second.");
82
+ if (!ctx.footage) {
83
+ const illustrations = plan.scenes.filter((s) => s.treatment === "illustration").length;
84
+ if (illustrations > Math.ceil(plan.scenes.length / 2)) issues.push(`${illustrations} of ${plan.scenes.length} scenes are illustrations. Use illustrations for at most half the scenes.`);
85
+ }
86
+ for (const s of plan.scenes) {
87
+ if (!s.onScreenText.length && !s.notes.trim()) issues.push(`Scene ${s.id} has neither onScreenText nor notes. Say what is on screen, since no voice explains it.`);
88
+ if (s.onScreenText.length > 5) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 5 (the items of one list on screen count as one text, but the plan cannot say which they are).`);
57
89
  for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
58
90
  }
59
91
  const rhythm = rhythmNote(plan);
@@ -75,14 +107,16 @@ export function firstSentence(text: string): string {
75
107
  export function structureNotes(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
76
108
  const notes: string[] = [];
77
109
  if (ctx.footage) return notes;
78
- if (plan.scenes.length >= 4 && !plan.scenes.some(hasPicture)) {
110
+ // A film with no voice is made of type and interface pieces on purpose (reference/launch-film.md), so it is not asked for a picture.
111
+ const voiceless = isVoiceless(plan);
112
+ if (!voiceless && plan.scenes.length >= 4 && !plan.scenes.some(hasPicture)) {
79
113
  notes.push("Every scene is type and shapes: no illustration, no clip and none of the user's own files. Give at least one scene a picture so the video has something to look at; see reference/scene-treatments.md.");
80
114
  }
81
115
  const first = plan.scenes[0]!;
82
- if (!hasPicture(first)) {
116
+ if (!voiceless && !hasPicture(first)) {
83
117
  notes.push(`The opening has no picture, clip or user asset (scene ${first.id}). The first second decides whether the video is watched, so open on something to look at; see reference/scriptwriting.md.`);
84
118
  }
85
- const n = words(firstSentence(first.narration));
119
+ const n = isVoiceless(plan) ? 0 : words(firstSentence(first.narration));
86
120
  if (n > 12) notes.push(`The first sentence of the opening is ${n} words. Open with a sentence of 12 words or fewer; see reference/scriptwriting.md.`);
87
121
  return notes;
88
122
  }