reelkit-cli 0.6.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +5 -3
  2. package/package.json +52 -9
  3. package/skill/SKILL.md +40 -19
  4. package/skill/THIRD_PARTY.md +104 -2
  5. package/skill/commands/launch-film.md +7 -0
  6. package/skill/reference/art-styles.md +70 -0
  7. package/skill/reference/asset-reuse.md +13 -2
  8. package/skill/reference/backgrounds.md +63 -0
  9. package/skill/reference/beat-sync.md +25 -19
  10. package/skill/reference/brand-motion.md +62 -0
  11. package/skill/reference/captions.md +11 -5
  12. package/skill/reference/clips.md +3 -3
  13. package/skill/reference/component-authoring.md +1 -1
  14. package/skill/reference/continuity.md +26 -7
  15. package/skill/reference/delivery-review.md +40 -0
  16. package/skill/reference/hebrew-rtl.md +3 -4
  17. package/skill/reference/{remotion-composition.md → hyperframes-composition.md} +18 -11
  18. package/skill/reference/kit.md +144 -12
  19. package/skill/reference/launch-film.md +194 -0
  20. package/skill/reference/motion-design.md +18 -16
  21. package/skill/reference/scene-treatments.md +20 -0
  22. package/skill/reference/scriptwriting.md +4 -1
  23. package/skill/reference/sound-design.md +34 -12
  24. package/skill/reference/studio-editing.md +55 -0
  25. package/skill/reference/styles.md +9 -6
  26. package/skill/reference/three-d.md +135 -0
  27. package/skill/reference/voice-sync.md +108 -0
  28. package/src/agents.ts +23 -12
  29. package/src/api/client.ts +4 -1
  30. package/src/cli.ts +18 -7
  31. package/src/commands/assets.ts +314 -30
  32. package/src/commands/build.ts +148 -35
  33. package/src/commands/init.ts +1 -1
  34. package/src/commands/install.ts +1 -1
  35. package/src/commands/plan.ts +8 -5
  36. package/src/commands/ref.ts +5 -2
  37. package/src/contract/index.ts +5 -3
  38. package/src/hyperframes/Root.tsx +1 -0
  39. package/src/hyperframes/fonts.ts +54 -0
  40. package/src/hyperframes/frame.tsx +46 -0
  41. package/src/hyperframes/host.tsx +38 -0
  42. package/src/hyperframes/kit/Assemble3D.tsx +92 -0
  43. package/src/hyperframes/kit/BrandTransform3D.tsx +12 -0
  44. package/src/hyperframes/kit/BrowserFrame.tsx +83 -0
  45. package/src/{remotion → hyperframes}/kit/Camera.tsx +7 -5
  46. package/src/{remotion → hyperframes}/kit/Captions.tsx +34 -18
  47. package/src/hyperframes/kit/Card3D.tsx +211 -0
  48. package/src/{remotion → hyperframes}/kit/Carry.tsx +1 -1
  49. package/src/hyperframes/kit/ChapterFrame.tsx +68 -0
  50. package/src/{remotion → hyperframes}/kit/ClipLayer.tsx +1 -1
  51. package/src/{remotion → hyperframes}/kit/Counter.tsx +1 -1
  52. package/src/hyperframes/kit/CounterRoll.tsx +75 -0
  53. package/src/{remotion → hyperframes}/kit/Entrance.tsx +1 -1
  54. package/src/{remotion → hyperframes}/kit/FootageLayer.tsx +1 -1
  55. package/src/hyperframes/kit/GlassPanel.tsx +43 -0
  56. package/src/hyperframes/kit/Grounds.tsx +177 -0
  57. package/src/hyperframes/kit/Headline.tsx +97 -0
  58. package/src/hyperframes/kit/Hero3D.tsx +197 -0
  59. package/src/hyperframes/kit/HudOverlay.tsx +52 -0
  60. package/src/hyperframes/kit/ImageLayers.tsx +48 -0
  61. package/src/{remotion → hyperframes}/kit/KenBurnsImage.tsx +1 -1
  62. package/src/{remotion → hyperframes}/kit/KeyedClip.tsx +1 -1
  63. package/src/{remotion → hyperframes}/kit/Layers.tsx +1 -1
  64. package/src/{remotion → hyperframes}/kit/LowerThird.tsx +1 -1
  65. package/src/hyperframes/kit/Music.tsx +19 -0
  66. package/src/hyperframes/kit/NamedCursor.tsx +54 -0
  67. package/src/hyperframes/kit/Orbit3D.tsx +49 -0
  68. package/src/hyperframes/kit/Particles3D.tsx +74 -0
  69. package/src/hyperframes/kit/Place.tsx +12 -0
  70. package/src/hyperframes/kit/PromptBox.tsx +84 -0
  71. package/src/hyperframes/kit/Scene3D.tsx +70 -0
  72. package/src/hyperframes/kit/SceneFrame.tsx +96 -0
  73. package/src/{remotion → hyperframes}/kit/ScreenOverlay.tsx +1 -1
  74. package/src/{remotion → hyperframes}/kit/Sfx.tsx +1 -1
  75. package/src/hyperframes/kit/SoundCues.tsx +22 -0
  76. package/src/hyperframes/kit/TerminalLog.tsx +98 -0
  77. package/src/hyperframes/kit/Text3D.tsx +78 -0
  78. package/src/hyperframes/kit/TextOnImage.tsx +41 -0
  79. package/src/{remotion → hyperframes}/kit/TitleCard.tsx +1 -1
  80. package/src/{remotion → hyperframes}/kit/Voiceover.tsx +1 -1
  81. package/src/hyperframes/kit/Warp3D.tsx +59 -0
  82. package/src/hyperframes/kit/bg-math.ts +179 -0
  83. package/src/hyperframes/kit/brand-transform.ts +25 -0
  84. package/src/{remotion → hyperframes}/kit/caption-groups.ts +7 -3
  85. package/src/hyperframes/kit/caption-style.ts +45 -0
  86. package/src/hyperframes/kit/docs.ts +249 -0
  87. package/src/hyperframes/kit/image-layers-math.ts +115 -0
  88. package/src/hyperframes/kit/index.ts +74 -0
  89. package/src/hyperframes/kit/inter-bold-typeface.ts +3 -0
  90. package/src/{remotion → hyperframes}/kit/motion-math.ts +36 -2
  91. package/src/hyperframes/kit/music-math.ts +59 -0
  92. package/src/hyperframes/kit/quiet-three.ts +11 -0
  93. package/src/hyperframes/kit/sample-text.ts +55 -0
  94. package/src/hyperframes/kit/scene3d-context.ts +5 -0
  95. package/src/hyperframes/kit/seeded.ts +13 -0
  96. package/src/hyperframes/kit/sound-cues.ts +89 -0
  97. package/src/hyperframes/kit/sound-kinds.ts +135 -0
  98. package/src/{remotion → hyperframes}/kit/theme.ts +43 -39
  99. package/src/hyperframes/kit/three-fx-math.ts +192 -0
  100. package/src/hyperframes/kit/three-math.ts +145 -0
  101. package/src/hyperframes/kit/transition-math.ts +116 -0
  102. package/src/hyperframes/kit/ui-math.ts +145 -0
  103. package/src/hyperframes/kit/ui-theme.ts +25 -0
  104. package/src/hyperframes/kit/word-anchor.ts +107 -0
  105. package/src/hyperframes/math.ts +62 -0
  106. package/src/hyperframes/three.tsx +10 -0
  107. package/src/pipeline/beatsnap.ts +72 -0
  108. package/src/pipeline/review.ts +44 -10
  109. package/src/pipeline/schema.ts +51 -4
  110. package/src/pipeline/timing.ts +27 -1
  111. package/src/project/background.ts +33 -0
  112. package/src/project/chromakey.ts +1 -1
  113. package/src/project/layers.ts +60 -0
  114. package/src/project/manifest.ts +59 -14
  115. package/src/project/music.ts +19 -5
  116. package/src/project/project.ts +4 -1
  117. package/src/project/serve.ts +2 -2
  118. package/src/project/soundreport.ts +347 -0
  119. package/src/project/svgcheck.ts +21 -0
  120. package/src/render/component-preview.ts +11 -55
  121. package/src/render/contact-sheet.ts +39 -0
  122. package/src/render/continuity.ts +14 -4
  123. package/src/render/deps.ts +15 -3
  124. package/src/render/render.ts +62 -57
  125. package/src/render/serve.ts +31 -0
  126. package/src/render/sound-notes.ts +106 -0
  127. package/src/render/static-check.ts +15 -4
  128. package/src/render/validate.ts +4 -4
  129. package/src/render/word-check.ts +181 -0
  130. package/src/render/worker.ts +71 -0
  131. package/src/testing/conformance.ts +12 -0
  132. package/src/testing/fake-api.ts +4 -4
  133. package/src/testing/fixtures.ts +4 -1
  134. package/src/remotion/Root.tsx +0 -31
  135. package/src/remotion/kit/Music.tsx +0 -19
  136. package/src/remotion/kit/SceneFrame.tsx +0 -19
  137. package/src/remotion/kit/docs.ts +0 -124
  138. package/src/remotion/kit/index.ts +0 -29
  139. package/src/remotion/kit/music-math.ts +0 -42
  140. /package/src/{remotion → hyperframes}/kit/Icon.tsx +0 -0
  141. /package/src/{remotion → hyperframes}/kit/beat.ts +0 -0
  142. /package/src/{remotion → hyperframes}/kit/brand-icons.ts +0 -0
  143. /package/src/{remotion → hyperframes}/kit/media.ts +0 -0
  144. /package/src/{remotion → hyperframes}/types.ts +0 -0
@@ -1,21 +1,26 @@
1
1
  import { execFile } from "node:child_process";
2
- import { mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
3
- import { join } from "node:path";
2
+ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
3
+ import { join, resolve } from "node:path";
4
4
  import { promisify } from "node:util";
5
5
  import type { Ctx, Result } from "../context";
6
- import type { AssetManifest, ScenePlan } from "../pipeline/schema";
7
- import { compareFrames, SAMPLE_WIDTH, summariseContinuity, type Boundary } from "../render/continuity";
6
+ import { isVoiceless, type AssetManifest, type ScenePlan } from "../pipeline/schema";
7
+ import { compareAcross, SAMPLE_WIDTH, summariseContinuity, type Boundary } from "../render/continuity";
8
8
  import { buildManifest, missingAssets } from "../project/manifest";
9
+ import { readBackground } from "../project/background";
9
10
  import { beatReport } from "../project/music";
10
11
  import { FILES, openProject, type Project } from "../project/project";
11
12
  import { structureNotes } from "../pipeline/review";
12
13
  import type { SearchRecord } from "./assets";
13
14
  import { libraryUse, shareAfterRender } from "./components";
14
15
  import { mediaUrls, serveDir } from "../project/serve";
16
+ import { contactSheet } from "../render/contact-sheet";
15
17
  import { bundleProject, disposeBundle, ensureRenderBrowser, renderStills, renderVideo, writeEntry, type EnsureBrowser } from "../render/render";
16
18
  import { MASTER_LUFS, MASTER_TRUE_PEAK_DB, masterLoudness, type Master, type Mastered } from "../render/master";
19
+ import { detailLines, measureSound, soundAdvice, spokenWords, voiceAdvice, type SoundReport, type SpokenWord } from "../project/soundreport";
20
+ import { soundNotes } from "../render/sound-notes";
17
21
  import { FILE_NAME, MAIN_FILE, jsxUses, staticCheck, typecheck } from "../render/validate";
18
22
  import { loadPlan } from "./plan";
23
+ import { MAX_WORD_FRAMES, usesWordTiming, WORD_FRAME_AFTER, wordCalls, wordErrors, wordTargets } from "../render/word-check";
19
24
 
20
25
  const exec = promisify(execFile);
21
26
  const MEDIA_PATH = /["'`](assets\/[^"'`\s]+\.(?:mp3|wav|m4a|mp4|mov|webm|png|jpe?g|webp|svg|gif))["'`]/g;
@@ -35,6 +40,13 @@ function inspect(project: Project): { errors: string[]; manifest?: AssetManifest
35
40
  if (!project.exists(m[1])) errors.push(`${name} uses "${m[1]}", which is not in this project. Get it with \`reelkit assets pull <id>\` first.`);
36
41
  }
37
42
  }
43
+ // A word given to wordFrame, onWord or onWordBeat that its scene never says would stop the render, so it is an error here.
44
+ if (!isVoiceless(plan)) {
45
+ const ids = plan.scenes.map((s) => s.id);
46
+ for (const name of names.filter((n) => FILE_NAME.test(n))) {
47
+ errors.push(...wordErrors(name, wordCalls(readFileSync(project.path(join("src", name)), "utf8"), ids, name), plan.scenes));
48
+ }
49
+ }
38
50
  if (errors.length) return { errors };
39
51
  const typeErrors = typecheck(names.map((n) => project.path(join("src", n))));
40
52
  if (typeErrors.length) return { errors: typeErrors };
@@ -43,14 +55,39 @@ function inspect(project: Project): { errors: string[]; manifest?: AssetManifest
43
55
  }
44
56
 
45
57
  // Advice about a composition that passes: never an error.
46
- function compositionNotes(project: Project, plan: ScenePlan): string[] {
58
+ function compositionNotes(project: Project, plan: ScenePlan, manifest?: AssetManifest): string[] {
47
59
  const notes: string[] = [];
48
60
  const names = readdirSync(project.path("src")).filter((f) => FILE_NAME.test(f));
49
61
  const sources = names.map((n) => [n, readFileSync(project.path(join("src", n)), "utf8")] as const);
50
62
  const rendersCaptions = sources.some(([n, src]) => jsxUses(src, n, "Captions").length > 0);
51
63
  if (plan.captions === "none" && rendersCaptions) notes.push('The plan says captions: "none" but the composition renders <Captions>. Remove them, or change the plan if the user wants captions.');
52
64
  if ((plan.captions === "word" || plan.captions === "phrase") && !rendersCaptions) notes.push(`The plan asks for captions (${plan.captions}) but the composition renders no <Captions>. Add <Captions words={s.words} group={manifest.captions} /> in each scene, or set captions to "none" in the plan if the user does not want them.`);
65
+ if (!isVoiceless(plan) && !sources.some(([n, src]) => usesWordTiming(src, n))) {
66
+ notes.push("Nothing on screen is timed to a spoken word: the composition never uses wordFrame, onWord, onWordBeat or a scene's words (other than to draw the subtitles). Place what illustrates a word on that word, e.g. <Entrance delay={onWord(s, \"tasks\")}>; see reference/voice-sync.md.");
67
+ }
53
68
  notes.push(...structureNotes(plan, { footage: project.footage() }));
69
+ if (isVoiceless(plan) && manifest) {
70
+ notes.push(...soundNotes(sources.map(([name, source]) => ({ name, source })), { seconds: manifest.totalFrames / manifest.fps, fps: manifest.fps, musicSet: manifest.music !== undefined }));
71
+ }
72
+ // A ground that was chosen and then never drawn is wasted: say so. A video ground with a low dim under type is hard to read.
73
+ const bg = readBackground(project);
74
+ const recorded = [...(bg.film ? [bg.film.key] : []), ...Object.values(bg.scenes).map((e) => e.key)];
75
+ const usesBackground = (src: string) => /\.background\b|bgPerScene/.test(src) || recorded.some((k) => src.includes(k));
76
+ if (recorded.length && !sources.some(([, src]) => usesBackground(src))) notes.push(`A background is recorded (${recorded[0]}) but the composition never uses it. Draw it with <BgVideo> or <BgImage> from manifest.background, or <BgSequence items={bgPerScene(...)}>; see reference/backgrounds.md.`);
77
+ if (plan.scenes.some((s) => s.onScreenText.length)) {
78
+ for (const [name, src] of sources) {
79
+ const lowDim = jsxUses(src, name, "BgVideo").some((u) => { const m = /\bdim=\{\s*(\d*\.?\d+)\s*\}/.exec(u.example); return m !== null && Number(m[1]) < 0.2; });
80
+ if (lowDim) { notes.push(`${name}: a BgVideo with a dim under 0.2 sits behind on-screen text, which will be hard to read. Raise dim to 0.3 or more, or blur the clip.`); break; }
81
+ }
82
+ }
83
+ // An image scene with words on it is built in layers. Decidable only when the composition names one of the two components, or names the scene's image (its key, or
84
+ // imageKey): a composition that never mentions the picture says nothing here.
85
+ const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
86
+ const layered = sources.some(([n, src]) => jsxUses(src, n, "ImageLayers").length > 0 || jsxUses(src, n, "TextOnImage").length > 0);
87
+ const withText = plan.scenes.filter((sc) => sc.treatment === "illustration" && sc.onScreenText.length > 0 && images[sc.id]);
88
+ if (!layered && withText.length && sources.some(([, src]) => /\bimageKey\b/.test(src) || withText.some((sc) => src.includes(images[sc.id]!)))) {
89
+ notes.push(`Scene ${withText.map((sc) => sc.id).join(", ")} ${withText.length === 1 ? "has" : "have"} an image and on-screen text, but the composition uses neither <ImageLayers> nor <TextOnImage>: the text is not a layer of the picture. Put the words in the picture's calm zone with <TextOnImage layers={s.imageLayers}> inside <ImageLayers layers={s.imageLayers}> (run \`reelkit assets layers --scene <id>\` for a hero picture so the words can pass behind its subject), not on a flat card beside it.`);
90
+ }
54
91
  const { pulled, written } = libraryUse(project);
55
92
  if (!pulled.length && plan.scenes.length >= 4) {
56
93
  const searched = project.readJsonOr<SearchRecord[]>(FILES.searches, []).filter((s) => s.kind === "component").length;
@@ -63,9 +100,9 @@ const failed = (errors: string[]): Result => ({ ok: false, data: { errors }, sum
63
100
 
64
101
  export async function check(ctx: Ctx): Promise<Result> {
65
102
  const project = openProject(ctx.cwd);
66
- const { errors } = inspect(project);
103
+ const { errors, manifest } = inspect(project);
67
104
  if (errors.length) return failed(errors);
68
- const shouldImprove = compositionNotes(project, loadPlan(project));
105
+ const shouldImprove = compositionNotes(project, loadPlan(project), manifest);
69
106
  return {
70
107
  ok: true, data: { errors: [], shouldImprove, library: libraryUse(project) },
71
108
  summary: ["The composition passes. Run `reelkit preview` to see it.", ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : [])].join("\n"),
@@ -102,33 +139,54 @@ export async function ensureBrowserWithNotice(ctx: Ctx, ensure: EnsureBrowser =
102
139
  const ffmpegToJpeg = (png: string, jpg: string) => exec("ffmpeg", ["-y", "-loglevel", "error", "-i", png, "-vf", "scale=540:-2", "-q:v", "5", jpg]).then(() => undefined);
103
140
 
104
141
  type PreviewPoint = { sceneId: string; point: "early" | "late" | "middle" | "end" | "start"; frame: number; name: string };
142
+ type WordFrameFile = { sceneId: string; word: string; label: string; frame: number; wordStart: number; name: string };
143
+
144
+ // Whether any file of the composition draws a Scene3D, which is the only case that needs the software GL in the browser.
145
+ export const usesScene3D = (project: Project): boolean => readdirSync(project.path("src")).filter((f) => f.endsWith(".tsx")).some((n) => /<Scene3D[\s>/]/.test(readFileSync(project.path(join("src", n)), "utf8")));
105
146
 
106
- // Two frames per scene, at 30% and 90% of its length, so what enters late in a scene is seen as well as what opens it.
107
- // A scene under 4 frames has no room for two distinct frames and gets its middle one. The scene's number leads each name
108
- // so the files sort by scene order and then by time, whatever the scene ids are.
109
- export function previewPoints(scenes: { id: string; startFrame: number; durationFrames: number }[]): PreviewPoint[] {
147
+ // How many frames at each end of a scene SceneFrame may be fading or moving: 8 unless the composition asks for longer with a literal
148
+ // transitionFrames={N} (at most 20, as SceneFrame allows). A value that is not a literal cannot be read, so the default is used.
149
+ export const DEFAULT_FADE = 8;
150
+ export function fadeFrames(sources: string[]): number {
151
+ let n = DEFAULT_FADE;
152
+ for (const src of sources) for (const m of src.matchAll(/transitionFrames=\{\s*(\d+)\s*\}/g)) n = Math.max(n, Math.min(20, Number(m[1])));
153
+ return n;
154
+ }
155
+
156
+ // The frames of a scene, counted from its start, where it is fully visible: SceneFrame fades or moves it at both ends for up to `fade` frames,
157
+ // but never more than half its length.
158
+ const visibleRange = (d: number, fade: number) => { const edge = Math.max(1, Math.min(fade, Math.floor(d / 2))); return { lo: edge, hi: d - edge, edge }; };
159
+
160
+ // Two frames per scene, at 30% and 90% of its length, so what enters late in a scene is seen as well as what opens it, both moved into the
161
+ // part where the scene is fully visible (a quick scene's 90% falls inside its fade and would look dim). A scene too short to have two distinct
162
+ // fully visible frames gets its middle one. The scene's number leads each name so the files sort by scene order and then by time.
163
+ export function previewPoints(scenes: { id: string; startFrame: number; durationFrames: number }[], fade = DEFAULT_FADE): PreviewPoint[] {
110
164
  return scenes.flatMap((s, i) => {
111
165
  const at = (point: PreviewPoint["point"], offset: number): PreviewPoint => ({ sceneId: s.id, point, frame: s.startFrame + offset, name: `${String(i + 1).padStart(2, "0")}-${s.id}-${point}` });
112
- if (s.durationFrames < 4) return [at("middle", Math.floor(s.durationFrames / 2))];
113
- const inside = (share: number) => Math.min(s.durationFrames - 1, Math.round(s.durationFrames * share));
166
+ const { lo, hi } = visibleRange(s.durationFrames, fade);
167
+ if (s.durationFrames < 4 || hi - lo < 2) return [at("middle", Math.floor(s.durationFrames / 2))];
168
+ const inside = (share: number) => Math.max(lo, Math.min(hi, Math.round(s.durationFrames * share)));
114
169
  return [at("early", inside(0.3)), at("late", inside(0.9))];
115
170
  });
116
171
  }
117
172
 
118
- type BoundaryPoint = { from: string; to: string; end: PreviewPoint; start: PreviewPoint };
173
+ type BoundaryPoint = { from: string; to: string; end: PreviewPoint; start: PreviewPoint; settled: PreviewPoint };
119
174
 
120
- // SceneFrame fades each scene in and out over up to 8 frames, so the very last and very first frames of two scenes are nearly empty.
121
- // The frames that show what a scene really holds are the last one at full strength and the first one at full strength: those are the two
122
- // either side of every change. The files are named by the change, b01-end-<scene> then b01-start-<scene>, so that they sort in order.
123
- export function boundaryPoints(scenes: { id: string; startFrame: number; durationFrames: number }[]): BoundaryPoint[] {
124
- const edge = (d: number) => Math.max(1, Math.min(8, Math.floor(d / 2)));
175
+ // SceneFrame fades or moves each scene in and out, so the very last and very first frames of two scenes are nearly empty. The frames that show
176
+ // what a scene really holds are the last one at full strength and the first one at full strength. A third frame, six frames after the first,
177
+ // catches a move that is still settling; it is measured but not saved. The files are named by the change, b01-end-<scene> then b01-start-<scene>.
178
+ export function boundaryPoints(scenes: { id: string; startFrame: number; durationFrames: number }[], fade = DEFAULT_FADE): BoundaryPoint[] {
125
179
  return scenes.slice(0, -1).map((a, i) => {
126
180
  const b = scenes[i + 1]!;
127
181
  const n = `b${String(i + 1).padStart(2, "0")}`;
182
+ const ea = visibleRange(a.durationFrames, fade), eb = visibleRange(b.durationFrames, fade);
183
+ const startOffset = Math.min(b.durationFrames - 1, eb.edge);
184
+ const settledOffset = Math.min(b.durationFrames - 1, Math.max(startOffset, Math.min(startOffset + 6, eb.hi)));
128
185
  return {
129
186
  from: a.id, to: b.id,
130
- end: { sceneId: a.id, point: "end", frame: a.startFrame + Math.max(0, a.durationFrames - edge(a.durationFrames)), name: `${n}-end-${a.id}` },
131
- start: { sceneId: b.id, point: "start", frame: b.startFrame + Math.min(b.durationFrames - 1, edge(b.durationFrames)), name: `${n}-start-${b.id}` },
187
+ end: { sceneId: a.id, point: "end", frame: a.startFrame + Math.max(0, ea.hi), name: `${n}-end-${a.id}` },
188
+ start: { sceneId: b.id, point: "start", frame: b.startFrame + startOffset, name: `${n}-start-${b.id}` },
189
+ settled: { sceneId: b.id, point: "start", frame: b.startFrame + settledOffset, name: `${n}-settled-${b.id}` },
132
190
  };
133
191
  });
134
192
  }
@@ -147,17 +205,25 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
147
205
  const dir = project.path("out/preview");
148
206
  rmSync(dir, { recursive: true, force: true });
149
207
  mkdirSync(dir, { recursive: true });
150
- const sceneFrames = previewPoints(manifest.scenes);
151
- const boundaries = boundaryPoints(manifest.scenes);
208
+ const plan = loadPlan(project);
209
+ const intended = new Set(plan.scenes.filter((s) => s.cutIn).map((s) => s.id));
210
+ const fade = fadeFrames(readdirSync(project.path("src")).filter((f) => f.endsWith(".tsx")).map((n) => readFileSync(project.path(join("src", n)), "utf8")));
211
+ const sceneFrames = previewPoints(manifest.scenes, fade);
212
+ const boundaries = boundaryPoints(manifest.scenes, fade);
152
213
  const points = [...sceneFrames, ...boundaries.flatMap((b) => [b.end, b.start])];
214
+ // A narrated film also gets one frame just after each word its code times something to: the thing the word names should already be there.
215
+ const srcNames = readdirSync(project.path("src")).filter((f) => FILE_NAME.test(f));
216
+ const wordPlan = isVoiceless(plan) ? undefined : wordTargets(srcNames.flatMap((n) => wordCalls(readFileSync(project.path(join("src", n)), "utf8"), plan.scenes.map((sc) => sc.id), n)), manifest);
217
+ const wordFiles: WordFrameFile[] = (wordPlan?.targets ?? []).map((t, i) => ({ ...t, name: `w${String(i + 1).padStart(2, "0")}-${t.sceneId}-${t.word.toLowerCase().replace(/[^a-z0-9]+/g, "") || "word"}` }));
218
+ const measuredFrames = [...points.map((p) => p.frame), ...boundaries.map((b) => b.settled.frame), ...wordFiles.map((w) => w.frame)];
153
219
  try {
154
220
  try {
155
221
  await ensureBrowserWithNotice(ctx, deps.ensureBrowser);
156
- await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, [...new Set(points.map((p) => p.frame))], dir));
222
+ await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, [...new Set(measuredFrames)], dir, { uses3D: usesScene3D(project) }));
157
223
  } catch (e) {
158
224
  return failed([`Render failed: ${e instanceof Error ? e.message : String(e)}`]);
159
225
  }
160
- const frames: { sceneId: string; point: PreviewPoint["point"]; path: string }[] = [];
226
+ const frames: { sceneId: string; point: PreviewPoint["point"] | "word"; path: string; word?: string; label?: string }[] = [];
161
227
  try {
162
228
  for (const m of points) {
163
229
  // Shrunk to phone width: small enough to read cheaply, and what is unreadable here is unreadable on a phone.
@@ -165,9 +231,23 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
165
231
  await toJpeg(join(dir, `still-${m.frame}.png`), project.path(path));
166
232
  frames.push({ sceneId: m.sceneId, point: m.point, path });
167
233
  }
234
+ for (const w of wordFiles) {
235
+ const path = `out/preview/${w.name}.jpg`;
236
+ await toJpeg(join(dir, `still-${w.frame}.png`), project.path(path));
237
+ frames.push({ sceneId: w.sceneId, point: "word", path, word: w.word, label: w.label });
238
+ }
168
239
  } catch {
169
240
  return failed(["Could not write the preview frames: ffmpeg failed. Check that ffmpeg is installed (macOS: `brew install ffmpeg`) and run `reelkit preview` again."]);
170
241
  }
242
+ // One picture of all the frames, so that a look at it comes before opening single frames. It is a convenience: failing to make it changes nothing else.
243
+ let sheetLine = "";
244
+ let sheet: { path: string; cols: number; rows: number } | undefined;
245
+ try {
246
+ const names = frames.map((f) => (f.point === "word" ? f.label! : `${f.sceneId} ${f.point === "early" || f.point === "late" || f.point === "middle" ? f.point : `change ${f.point}`}`));
247
+ const r = await contactSheet(frames.map((f) => project.path(f.path)), names, project.path("out/preview/sheet.jpg"), manifest.width / manifest.height, { newRowAt: wordFiles.length ? points.length : undefined });
248
+ sheet = { path: "out/preview/sheet.jpg", cols: r.cols, rows: r.rows };
249
+ sheetLine = `Contact sheet, one picture of all ${frames.length} frames in the order below${r.labelled ? ", each labelled with its scene and early, late or change" : ""} (${r.cols} across${wordFiles.length ? `; the word frames are in the last row${r.labelled ? ", labelled word > scene" : ""}` : ""}): out/preview/sheet.jpg. Look at it first, then open single frames where something needs a closer look.\n`;
250
+ } catch { /* no sheet */ }
171
251
  // Whether anything survives each scene change is advice. If it cannot be measured, the preview is still the preview.
172
252
  let continuity: { score: number | null; boundaries: Boundary[] } | undefined;
173
253
  let continuityLines: string[] = [];
@@ -175,9 +255,9 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
175
255
  const h = Math.round((SAMPLE_WIDTH * manifest.height) / manifest.width);
176
256
  const measured: Boundary[] = [];
177
257
  for (const b of boundaries) {
178
- const [a, c] = await Promise.all([decodeGray(join(dir, `still-${b.end.frame}.png`), SAMPLE_WIDTH, h), decodeGray(join(dir, `still-${b.start.frame}.png`), SAMPLE_WIDTH, h)]);
179
- const r = compareFrames(a, c, SAMPLE_WIDTH, h);
180
- measured.push({ from: b.from, to: b.to, survived: Math.round(r.survived * 100) / 100, kind: r.kind });
258
+ const [a, c, d] = await Promise.all([b.end.frame, b.start.frame, b.settled.frame].map((f) => decodeGray(join(dir, `still-${f}.png`), SAMPLE_WIDTH, h)));
259
+ const r = compareAcross(a!, [c!, d!], SAMPLE_WIDTH, h);
260
+ measured.push({ from: b.from, to: b.to, survived: Math.round(r.survived * 100) / 100, kind: intended.has(b.to) && r.kind !== "empty" ? "cut (intended)" : r.kind });
181
261
  }
182
262
  if (measured.length) {
183
263
  const summary = summariseContinuity(measured);
@@ -185,24 +265,27 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
185
265
  continuityLines = summary.lines;
186
266
  }
187
267
  } catch { /* no continuity report */ }
188
- const sceneLines = `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.filter((f) => f.point !== "end" && f.point !== "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}`;
268
+ const wordLines = wordPlan && (wordFiles.length || wordPlan.skipped)
269
+ ? `\nWord frames: ${wordFiles.length} frame${wordFiles.length === 1 ? "" : "s"}, each ${WORD_FRAME_AFTER} frames AFTER its word starts, in a row of their own on the sheet (labelled "word ▸ scene"). Each word frame should already show the thing that word names: if it does not, the thing is late (or on the wrong word) and its timing needs fixing.${wordFiles.length ? `\n${wordFiles.map((w) => `${w.label}: out/preview/${w.name}.jpg`).join("\n")}` : ""}${wordPlan.skipped ? `\n${wordPlan.skipped} word timing${wordPlan.skipped === 1 ? " was" : "s were"} not shown (${[wordPlan.undecidable ? `${wordPlan.undecidable} name${wordPlan.undecidable === 1 ? "s" : ""} a word or a scene the code does not give as a literal` : "", wordPlan.capped ? `${wordPlan.capped} beyond the first ${MAX_WORD_FRAMES}` : ""].filter(Boolean).join(", ")}).` : ""}`
270
+ : wordPlan ? "\nWord frames: none, because the composition has no wordFrame, onWord or onWordBeat call with a literal scene and word." : "";
271
+ const sceneLines = `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.filter((f) => f.point !== "end" && f.point !== "start" && f.point !== "word").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}`;
189
272
  const changeLines = boundaries.length ? `\nThe last and first frames either side of each scene change are saved too (b01-end-<scene>, b01-start-<scene>, ...), so that you can see what carries across:\n${frames.filter((f) => f.point === "end" || f.point === "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}` : "";
190
- const beat = beatReport(manifest);
191
- return { ok: true, data: { frames, ...(continuity ? { continuity } : {}), ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) }, summary: `${sceneLines}${changeLines}${continuityLines.length ? `\nContinuity: ${continuityLines.join("\n")}` : ""}${beat ? `\nBeat: ${beat.line}.` : ""}` };
273
+ const beat = beatReport(manifest, { finer: isVoiceless(plan) });
274
+ return { ok: true, data: { frames, ...(wordPlan ? { wordFrames: { shown: wordFiles.map((w) => ({ scene: w.sceneId, word: w.word, wordStart: w.wordStart, frame: w.frame })), skipped: wordPlan.skipped } } : {}), ...(sheet ? { sheet } : {}), ...(continuity ? { continuity } : {}), ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) }, summary: `${sheetLine}${sceneLines}${changeLines}${wordLines}${continuityLines.length ? `\nContinuity: ${continuityLines.join("\n")}` : ""}${beat ? `\nBeat: ${beat.line}.` : ""}` };
192
275
  } finally {
193
276
  // The full-size stills are only inputs; none stays behind however this ends.
194
277
  for (const f of readdirSync(dir)) if (/^still-\d+\.png$/.test(f)) rmSync(join(dir, f), { force: true });
195
278
  }
196
279
  }
197
280
 
198
- export async function render(ctx: Ctx, deps: { ensureBrowser?: EnsureBrowser; master?: Master; noShare?: boolean } = {}): Promise<Result> {
281
+ export async function render(ctx: Ctx, deps: { ensureBrowser?: EnsureBrowser; master?: Master; noShare?: boolean; measure?: (path: string, loudness?: number, words?: SpokenWord[]) => Promise<SoundReport> } = {}): Promise<Result> {
199
282
  const project = openProject(ctx.cwd);
200
283
  const { errors, manifest } = inspect(project);
201
284
  if (!manifest) return failed(errors);
202
285
  const path = "out/video.mp4";
203
286
  try {
204
287
  await ensureBrowserWithNotice(ctx, deps.ensureBrowser);
205
- await withBundle(project, (serveUrl, urls) => renderVideo(serveUrl, { manifest, urls }, project.path(path)));
288
+ await withBundle(project, (serveUrl, urls) => renderVideo(serveUrl, { manifest, urls }, project.path(path), { uses3D: usesScene3D(project) }));
206
289
  } catch (e) {
207
290
  return failed([`Render failed: ${e instanceof Error ? e.message : String(e)}`]);
208
291
  }
@@ -216,9 +299,39 @@ export async function render(ctx: Ctx, deps: { ensureBrowser?: EnsureBrowser; ma
216
299
  } catch (e) {
217
300
  masterNote = ` It was not mastered to ${MASTER_LUFS} LUFS (${e instanceof Error ? e.message : String(e)}); the video is kept as rendered.`;
218
301
  }
302
+ // How the finished film sounds against what it shows. It is advice, so a measurement that fails leaves the render as it was.
303
+ let sound: SoundReport | undefined;
304
+ let soundLine: string | undefined;
305
+ try {
306
+ const voiceless = isVoiceless(loadPlan(project));
307
+ sound = await (deps.measure ?? measureSound)(project.path(path), loudness?.outputLufs, voiceless ? undefined : spokenWords(manifest));
308
+ if (voiceless) soundLine = soundAdvice(sound);
309
+ else if (sound.voice) soundLine = voiceAdvice(sound.voice);
310
+ } catch { sound = undefined; }
219
311
  const share = await shareAfterRender(ctx, project, { noShare: deps.noShare });
220
312
  return {
221
- ok: true, data: { path, seconds, ...(loudness ? { loudness } : {}), ...(share.data ? { shared: share.data.shared.map((s) => s.name) } : {}) },
222
- summary: [`Video: ${project.path(path)} (${seconds.toFixed(1)}s, ${manifest.width}x${manifest.height}).${masterNote}`, ...share.lines].join("\n"),
313
+ ok: true, data: { path, seconds, ...(loudness ? { loudness } : {}),
314
+ ...(sound ? { sound: { hitsPerSecond: sound.hitsPerSecond, swellsPerSecond: sound.swellsPerSecond, pictureChanges: sound.pictureChanges, changesWithHit: sound.changesWithHit, firstSoundSec: sound.firstSoundSec }, ...(sound.voice ? { voice: sound.voice } : {}) } : {}), ...(share.data ? { shared: share.data.shared.map((s) => s.name) } : {}) },
315
+ summary: [`Video: ${project.path(path)} (${seconds.toFixed(1)}s, ${manifest.width}x${manifest.height}).${masterNote}`, ...(soundLine ? [soundLine] : []), ...share.lines].join("\n"),
223
316
  };
224
317
  }
318
+
319
+ // `reelkit sound [file]`: the same measurement on any film, by default the last render. `--detail` lists every picture change with its nearest hit, every swell and the first sound.
320
+ export async function soundCommand(ctx: Ctx, file?: string, opts: { detail?: boolean } = {}, deps: { measure?: (path: string, loudness?: number, words?: SpokenWord[]) => Promise<SoundReport> } = {}): Promise<Result> {
321
+ let project: Project | undefined;
322
+ try { project = openProject(ctx.cwd); } catch { project = undefined; }
323
+ const path = file ? resolve(ctx.cwd, file) : project ? project.path("out/video.mp4") : resolve(ctx.cwd, "out/video.mp4");
324
+ if (!existsSync(path)) return { ok: false, summary: `${file ?? "out/video.mp4"} does not exist. ${file ? "Check the path." : "Run `reelkit render` first, or give the file."}` };
325
+ let narrated = false;
326
+ try { narrated = project ? !isVoiceless(loadPlan(project)) : false; } catch { narrated = false; }
327
+ // A narrated film is also measured against its words: the manifest has their times.
328
+ let words: SpokenWord[] | undefined;
329
+ if (narrated && project?.exists(FILES.manifest) && (!file || resolve(ctx.cwd, file) === resolve(project.path("out/video.mp4")))) { try { words = spokenWords(project.readJson<AssetManifest>(FILES.manifest)); } catch { words = undefined; } }
330
+ let report: SoundReport;
331
+ try { report = await (deps.measure ?? measureSound)(path, undefined, words); } catch (e) { return { ok: false, summary: `Could not measure the sound: ${e instanceof Error ? e.message : String(e)}` }; }
332
+ const numbers = `${report.hitsPerSecond} hits a second, ${report.swellsPerSecond} swells a second, ${report.changesWithHit} of ${report.pictureChanges} picture changes with a hit within 0.1s, first sound at ${report.firstSoundSec ?? "never"}s${report.loudnessLufs !== undefined ? `, ${report.loudnessLufs} LUFS` : ""}.`;
333
+ const { detail, ...counts } = report;
334
+ const body = narrated ? `${numbers}${report.voice ? `\n${voiceAdvice(report.voice)}` : ""}` : `${soundAdvice(report)}\n${numbers}`;
335
+ if (opts.detail && detail) return { ok: true, data: { path, ...counts, detail }, summary: `${body}\n${detailLines(detail).join("\n")}` };
336
+ return { ok: true, data: { path, ...counts }, summary: `${body}\nRun with --detail to see each event.` };
337
+ }
@@ -31,7 +31,7 @@ export async function init(ctx: Ctx, name: string | undefined, opts: { aspect?:
31
31
  summary: [
32
32
  `Project ready in ${dir} (${project.config().aspect}).${slug ? ` Next: cd ${slug}` : ""}`,
33
33
  project.config().shareComponents === false
34
- ? "Nothing from this project is shared automatically."
34
+ ? "This project is private: nothing from it is sent to the library, not its components and not the images, clips or graphics generated for it, whatever the plan says."
35
35
  : "After a render, new components written in this project are sent to the Reelkit library for review (their source, a description and an example; nothing else). To turn this off: `reelkit init <name> --private` for a new project, `reelkit render --no-share`, or REELKIT_NO_SHARE=1.",
36
36
  ].join("\n"),
37
37
  };
@@ -17,7 +17,7 @@ export async function install(ctx: Ctx, opts: { agent?: string; force?: boolean
17
17
  }
18
18
  const name = (id: string) => AGENTS.find((a) => a.id === id)!.name;
19
19
  const lines = [
20
- ...report.installed.map((i) => `Installed the Reelkit skill for ${name(i.agent)}: ${i.path}`),
20
+ ...report.installed.map((i) => `Installed the Reelkit skill for ${name(i.agent)}: ${i.path}${i.commands.length ? ` and the commands ${i.commands.map((c) => `/${c}`).join(" and ")}` : ""}`),
21
21
  ...report.skipped.filter((s) => agents || s.reason !== "not installed on this machine").map((s) => `Skipped ${name(s.agent)}: ${s.reason}`),
22
22
  ];
23
23
  return { ok: true, data: report, summary: lines.join("\n") };
@@ -1,7 +1,8 @@
1
1
  import { client, type Ctx, type Result } from "../context";
2
2
  import { loadCredentials } from "../credentials";
3
3
  import { estimateLength, lengthVariation, reviewPlan, sceneSeconds } from "../pipeline/review";
4
- import { ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
4
+ import { isVoiceless, ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
5
+ import { snappedSeconds } from "../project/manifest";
5
6
  import { FILES, issueLines, openProject, type Project } from "../project/project";
6
7
 
7
8
  export function loadPlan(project: Project): ScenePlan {
@@ -52,7 +53,8 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
52
53
  }
53
54
  const footage = project.footage();
54
55
  const assets = project.assets().filter((a) => a.id !== footage?.id);
55
- const voiceIds = loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
56
+ const voiceless = isVoiceless(parsed.data);
57
+ const voiceIds = !voiceless && loadCredentials(ctx.env) ? (await client(ctx)("voices", {})).voices.map((v) => v.id) : undefined;
56
58
  const ref = referenceChecks(project, parsed.data);
57
59
  const mustFix = [...validatePlan(parsed.data, { aspect: project.config().aspect, footage, assets, voiceIds }), ...ref.mustFix];
58
60
  const shouldImprove = mustFix.length ? [] : [...reviewPlan(parsed.data, { footage }), ...ref.improve];
@@ -60,11 +62,12 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
60
62
  const estimatedSeconds = Math.round(length.seconds);
61
63
  const perScene = sceneSeconds(parsed.data).map((s) => ({ id: s.id, seconds: Math.round(s.seconds * 10) / 10 }));
62
64
  const variation = Math.round(lengthVariation(sceneSeconds(parsed.data).map((s) => s.seconds)) * 100) / 100;
65
+ const snapped = voiceless ? snappedSeconds(project, parsed.data) : undefined;
63
66
  const lines = [
64
67
  mustFix.length ? `Fix these:\n- ${mustFix.join("\n- ")}` : "The plan is valid.",
65
- `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
68
+ voiceless ? `Length: ${(Math.round(length.seconds * 10) / 10).toFixed(1)}s over ${parsed.data.scenes.length} scenes${snapped !== undefined ? `; about ${snapped.toFixed(1)}s once the cuts are on the beat` : ""}.` : `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
66
69
  ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : []),
67
- ...(voiceIds ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
70
+ ...(voiceIds || voiceless ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
68
71
  ];
69
- return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation }, summary: lines.join("\n") };
72
+ return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation, ...(snapped !== undefined ? { snappedSeconds: Math.round(snapped * 10) / 10 } : {}) }, summary: lines.join("\n") };
70
73
  }
@@ -60,11 +60,14 @@ const realYtdlp: YtdlpRunner = {
60
60
 
61
61
  const YTDLP_MISSING = "yt-dlp is not installed, and links need it. Install it (macOS: `brew install yt-dlp`, or `pipx install yt-dlp`) and run the command again; a video file on this machine works without it.";
62
62
 
63
+ // The duration filter uses "<=?": some sites (Instagram) give no duration before the download, and without the "?" such a video is
64
+ // skipped as if it were too long. A video that turns out longer is refused after it is probed.
65
+ // No "--max-downloads 1": yt-dlp exits with an error code after reaching it even when the one video was saved.
63
66
  // One video, at most 1080p, as mp4, and nothing longer than the limit: the flags that keep a link from fetching a playlist, a channel or a huge file.
64
67
  export function ytdlpArgs(url: string, dest: string): string[] {
65
68
  return [
66
- "--ignore-config", "--no-playlist", "--max-downloads", "1", "--no-progress", "--no-warnings",
67
- "--match-filter", `duration<=${MAX_REF_SECONDS}`,
69
+ "--ignore-config", "--no-playlist", "--no-progress", "--no-warnings",
70
+ "--match-filter", `duration<=?${MAX_REF_SECONDS}`,
68
71
  "-f", "bv*[height<=1080][ext=mp4]+ba[ext=m4a]/b[height<=1080][ext=mp4]/bv*[height<=1080]+ba/b[height<=1080]",
69
72
  "--merge-output-format", "mp4", "-o", dest, "--", url,
70
73
  ];
@@ -14,7 +14,7 @@
14
14
  // - A library item `id` (pull, commit) matches `^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`; a `deviceCode` is 1 to 200 characters of plain text.
15
15
  // - A `voiceId` is 1 to 64 letters and digits and must be one of the voices the server lists (`voices`), otherwise 400 invalid_request.
16
16
  // - A user's component is uploaded for review through the same two calls as any other file (`kind: "component"`). The server checks its source
17
- // at commit with the same static check a pulled component gets (only react, remotion and reelkit/kit imports, no network, no globals that reach
17
+ // at commit with the same static check a pulled component gets (only react, reelkit/frame and reelkit/kit imports, no network, no globals that reach
18
18
  // outside the video) and refuses a source that fails it with 400 `invalid_request` and the first problem. It stays in `review`, is never found
19
19
  // by search or the public listing, and is published only by the owner. A component upload is: `contentType` `text/plain`, `bytes` up to 65,536,
20
20
  // `shareable` true, `filename` matching `^[A-Z][A-Za-z0-9]*\.tsx$`, a `description` of at least 20 characters, and `meta.example` a string of at most
@@ -210,10 +210,12 @@ export const routes = {
210
210
  voiceover: route("POST", "/voiceover", true,
211
211
  z.object({ text: text(5000).refine((t) => /\S/.test(t), "must contain a non-space character"), voiceId: z.string().regex(/^[A-Za-z0-9]{1,64}$/).optional(), speed: z.number().min(0.7).max(1.3).optional() }),
212
212
  z.object({ url: z.string(), ext: z.enum(["mp3", "wav", "m4a"]), contentType: z.string(), durationSec: z.number().positive(), words: z.array(WordTimingSchema), chars: z.number() })),
213
+ // `format` is "png" (a raster image, the default) or "svg" (a vector graphic). An SVG counts as one image against the image quota, takes about 45 s, and the
214
+ // server returns only a cleaned SVG: no scripts, no event attributes, no outside references. Its `ext` is "svg" and its contentType `image/svg+xml`.
213
215
  images: route("POST", "/images", true,
214
- z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12) }),
216
+ z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12), format: z.enum(["png", "svg"]).default("png") }),
215
217
  // libraryId is set when the image was also added to the shared library.
216
- z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp"]), contentType: z.string(), libraryId: z.string().optional() })),
218
+ z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp", "svg"]), contentType: z.string(), libraryId: z.string().optional() })),
217
219
  clipStart: route("POST", "/clips", true,
218
220
  z.object({
219
221
  prompt: text(2000).min(1), aspect: AspectSchema, greenScreen: z.boolean().default(false), durationSec: z.union([z.literal(5), z.literal(10)]).default(5),
@@ -0,0 +1 @@
1
+ export { mountVideo, mountVideo as makeRoot } from "./host";
@@ -0,0 +1,54 @@
1
+ import { delayRender, continueRender } from "./frame";
2
+ const loaded = new Set<string>();
3
+ export function loadFont(family: string, slug: string) {
4
+ if (typeof document !== "undefined" && !loaded.has(slug)) {
5
+ loaded.add(slug);
6
+ const handle = delayRender(`font ${family}`);
7
+ const link = document.createElement("link");
8
+ link.onload = () => continueRender(handle);
9
+ link.onerror = () => { continueRender(handle); throw new Error(`Font stylesheet failed: ${slug}`); };
10
+ link.rel = "stylesheet"; link.href = `./fonts/${slug}.css`; document.head.appendChild(link);
11
+ }
12
+ return { fontFamily: `"${family}"` };
13
+ }
14
+ export const fontFamilies = {
15
+ "Inter": "inter",
16
+ "Sora": "sora",
17
+ "ArchivoBlack": "archivo-black",
18
+ "BebasNeue": "bebas-neue",
19
+ "GreatVibes": "great-vibes",
20
+ "JetBrainsMono": "jetbrains-mono",
21
+ "Kanit": "kanit",
22
+ "Modak": "modak",
23
+ "Montserrat": "montserrat",
24
+ "Roboto": "roboto",
25
+ "RobotoSlab": "roboto-slab",
26
+ "Sacramento": "sacramento",
27
+ "Unbounded": "unbounded",
28
+ "BungeeShade": "bungee-shade",
29
+ "Cinzel": "cinzel",
30
+ "Creepster": "creepster",
31
+ "Heebo": "heebo",
32
+ "KaushanScript": "kaushan-script",
33
+ "LilitaOne": "lilita-one",
34
+ "MrsSaintDelafield": "mrs-saint-delafield",
35
+ "Orbitron": "orbitron",
36
+ "Outfit": "outfit",
37
+ "PermanentMarker": "permanent-marker",
38
+ "PlayfairDisplay": "playfair-display",
39
+ "Rubik": "rubik",
40
+ "Rye": "rye",
41
+ "Syne": "syne",
42
+ "Yellowtail": "yellowtail",
43
+ "Alef": "alef",
44
+ "AmaticSC": "amatic-sc",
45
+ "Assistant": "assistant",
46
+ "DavidLibre": "david-libre",
47
+ "FrankRuhlLibre": "frank-ruhl-libre",
48
+ "Fredoka": "fredoka",
49
+ "Karantina": "karantina",
50
+ "NotoSansHebrew": "noto-sans-hebrew",
51
+ "SecularOne": "secular-one",
52
+ "SuezOne": "suez-one",
53
+ "VarelaRound": "varela-round"
54
+ } as const;
@@ -0,0 +1,46 @@
1
+ import React, { createContext, useContext, useId } from "react";
2
+ export * from "./math";
3
+ export type VideoConfig = { width: number; height: number; fps: number; durationInFrames: number; id?: string };
4
+ export type FrameState = { frame: number; offset: number; duration: number; config: VideoConfig; loop?: number; collecting?: boolean };
5
+ export const FrameContext = createContext<FrameState>({ frame: 0, offset: 0, duration: 1, config: { width: 1080, height: 1920, fps: 30, durationInFrames: 1 } });
6
+ export const useCurrentFrame = () => useContext(FrameContext).frame;
7
+ export const useVideoConfig = () => { const s = useContext(FrameContext); return { ...s.config, durationInFrames: s.duration }; };
8
+ export const AbsoluteFill = React.forwardRef<HTMLDivElement, React.HTMLAttributes<HTMLDivElement>>(({ style, ...props }, ref) => <div ref={ref} {...props} style={{ position: "absolute", inset: 0, display: "flex", flexDirection: "column", ...style }} />);
9
+ export const Img = React.forwardRef<HTMLImageElement, React.ImgHTMLAttributes<HTMLImageElement>>((props, ref) => <img {...props} ref={ref} />);
10
+ export const Sequence: React.FC<{ from?: number; durationInFrames?: number; layout?: "none" | "absolute-fill"; children?: React.ReactNode; name?: string; style?: React.CSSProperties; premountFor?: number }> = ({ from = 0, durationInFrames, layout, children, style }) => {
11
+ const s = useContext(FrameContext), duration = Math.min(durationInFrames ?? s.duration - from, s.duration - from);
12
+ if (!s.collecting && (s.frame < from || s.frame >= from + duration)) return null;
13
+ const content = <FrameContext.Provider value={{ ...s, frame: s.frame - from, offset: s.offset + from, duration }}>{children}</FrameContext.Provider>;
14
+ return layout === "none" ? content : <AbsoluteFill style={style}>{content}</AbsoluteFill>;
15
+ };
16
+ export const Freeze: React.FC<{ frame: number; children?: React.ReactNode }> = ({ frame, children }) => { const s = useContext(FrameContext); return <FrameContext.Provider value={{ ...s, frame }}>{children}</FrameContext.Provider>; };
17
+ export const Loop: React.FC<{ durationInFrames: number; children?: React.ReactNode; times?: number; layout?: "none" | "absolute-fill" }> = ({ durationInFrames, children }) => {
18
+ const s = useContext(FrameContext); return <FrameContext.Provider value={{ ...s, frame: ((s.frame % durationInFrames) + durationInFrames) % durationInFrames, loop: durationInFrames }}>{children}</FrameContext.Provider>;
19
+ };
20
+ export type AudioRecord = { id: string; src: string; start: number; end: number; mediaStart: number; layer: number; volume: number; volumeKeyframes: { time: number; volume: number }[]; type: "audio" | "video"; loop: boolean };
21
+ export const audioRecords = new Map<string, AudioRecord>();
22
+ type MediaProps = { src: string; volume?: number | ((frame: number) => number); startFrom?: number; endAt?: number; trimBefore?: number; trimAfter?: number; playbackRate?: number; loopVolumeCurveBehavior?: "extend" | "repeat" };
23
+ function recordAudio(id: string, props: MediaProps & { loop?: boolean }, type: "audio" | "video", state: FrameState) {
24
+ const { config: { fps } } = state;
25
+ const volume = typeof props.volume === "function" ? props.volume(state.frame) : props.volume ?? 1;
26
+ const start = state.offset / fps, end = Math.min(state.config.durationInFrames, state.offset + state.duration) / fps;
27
+ let record = audioRecords.get(id);
28
+ if (!record) { record = { id, src: props.src, start, end, mediaStart: (props.trimBefore ?? props.startFrom ?? 0) / fps, layer: audioRecords.size, volume, volumeKeyframes: [], type, loop: props.loop ?? false }; audioRecords.set(id, record); }
29
+ const time = (state.frame + state.offset) / fps;
30
+ if (time >= start && time < end && typeof props.volume === "function") record.volumeKeyframes.push({ time, volume });
31
+ }
32
+ export const Audio: React.FC<MediaProps & { loop?: boolean }> = (props) => {
33
+ const s = useContext(FrameContext), id = useId(); recordAudio(id, props, "audio", s);
34
+ return <audio id={`hf-audio-${id.replace(/[^a-z0-9]/gi, "")}`} src={props.src} preload="none" data-start={s.offset / s.config.fps} data-duration={s.duration / s.config.fps} />;
35
+ };
36
+ export const OffthreadVideo: React.FC<Omit<React.VideoHTMLAttributes<HTMLVideoElement>, "volume"> & MediaProps & { transparent?: boolean; pauseWhenBuffering?: boolean }> = ({ volume, startFrom, endAt, trimBefore, trimAfter, transparent, pauseWhenBuffering, playbackRate = 1, ...props }) => {
37
+ const s = useContext(FrameContext), id = useId();
38
+ if (!props.muted) recordAudio(id, { ...props, src: props.src!, volume, startFrom, trimBefore }, "video", s);
39
+ return <video {...props} muted playsInline preload="auto" data-hf-frame-video="true" data-hf-time={Math.max(0, s.frame / s.config.fps * playbackRate + (trimBefore ?? startFrom ?? 0) / s.config.fps)} data-hf-loop={props.loop || s.loop ? "true" : undefined} />;
40
+ };
41
+ export const Video = OffthreadVideo;
42
+ let nextHandle = 0;
43
+ const pending = new Map<number, { promise: Promise<void>; resolve: () => void }>();
44
+ export function delayRender(_label?: string): number { const id = ++nextHandle; let resolve!: () => void; const promise = new Promise<void>((r) => { resolve = r; }); pending.set(id, { promise, resolve }); return id; }
45
+ export function continueRender(handle: number): void { pending.get(handle)?.resolve(); pending.delete(handle); }
46
+ export async function waitForRender(): Promise<void> { while (pending.size) await Promise.all([...pending.values()].map((p) => p.promise)); }
@@ -0,0 +1,38 @@
1
+ import React from "react";
2
+ import type { HfProtocol } from "@hyperframes/engine";
3
+ import { createRoot } from "react-dom/client";
4
+ import { flushSync } from "react-dom";
5
+ import { audioRecords, FrameContext, waitForRender } from "./frame";
6
+ import { ManifestContext, UrlsContext } from "./kit/media";
7
+ import type { VideoProps } from "./types";
8
+
9
+ declare global { interface Window { __reelkitProps: VideoProps; __reelkitReady: Promise<void>; __reelkitAudio: typeof audioRecords; __hf?: HfProtocol; } }
10
+ async function settleMedia() {
11
+ await waitForRender();
12
+ await document.fonts.ready;
13
+ await Promise.all([...document.images].map(async (img) => { if (!img.complete) await new Promise<void>((resolve, reject) => { img.addEventListener("load", () => resolve(), { once: true }); img.addEventListener("error", () => reject(new Error(`Image failed: ${img.src}`)), { once: true }); }); if (img.naturalWidth === 0) throw new Error(`Image failed: ${img.src}`); await img.decode(); }));
14
+ await Promise.all([...document.querySelectorAll<HTMLVideoElement>("video[data-hf-frame-video]")].map(async (video) => {
15
+ if (video.readyState < 1) await new Promise<void>((resolve, reject) => { video.addEventListener("loadedmetadata", () => resolve(), { once: true }); video.addEventListener("error", () => reject(new Error(`Video failed: ${video.src}`)), { once: true }); });
16
+ const time = Number(video.dataset.hfTime), duration = video.duration;
17
+ const target = video.dataset.hfLoop ? time % duration : Math.min(time, Math.max(0, duration - 0.00001));
18
+ if (video.readyState < 2 || Math.abs(video.currentTime - target) > 0.00001) await new Promise<void>((resolve, reject) => { video.addEventListener("seeked", () => resolve(), { once: true }); video.addEventListener("error", () => reject(new Error(`Video seek failed: ${video.src}`)), { once: true }); video.currentTime = target; });
19
+ video.pause();
20
+ }));
21
+ await new Promise<void>((resolve) => requestAnimationFrame(() => requestAnimationFrame(() => resolve())));
22
+ }
23
+ export function mountVideo(Video: React.ComponentType<VideoProps>, previewProps?: VideoProps) {
24
+ const props = previewProps ?? window.__reelkitProps, m = props.manifest;
25
+ const stage = document.getElementById("stage")!;
26
+ stage.style.width = `${m.width}px`; stage.style.height = `${m.height}px`;
27
+ stage.dataset.compositionId = "Video"; stage.dataset.noTimeline = "true"; stage.dataset.duration = String(m.totalFrames / m.fps);
28
+ stage.dataset.width = String(m.width); stage.dataset.height = String(m.height);
29
+ const root = createRoot(stage);
30
+ window.__reelkitAudio = audioRecords;
31
+ const seek = (time: number) => {
32
+ const frame = Math.min(m.totalFrames - 1, Math.max(0, Math.round(time * m.fps)));
33
+ flushSync(() => root.render(<FrameContext.Provider value={{ frame, offset: 0, duration: m.totalFrames, config: { width: m.width, height: m.height, fps: m.fps, durationInFrames: m.totalFrames } }}><UrlsContext.Provider value={props.urls}><ManifestContext.Provider value={m}><Video {...props} /></ManifestContext.Provider></UrlsContext.Provider></FrameContext.Provider>));
34
+ window.__reelkitReady = settleMedia();
35
+ };
36
+ window.__hf = { duration: m.totalFrames / m.fps, seek };
37
+ seek(0);
38
+ }