reelkit-cli 0.6.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/package.json +52 -9
- package/skill/SKILL.md +40 -19
- package/skill/THIRD_PARTY.md +104 -2
- package/skill/commands/launch-film.md +7 -0
- package/skill/reference/art-styles.md +70 -0
- package/skill/reference/asset-reuse.md +13 -2
- package/skill/reference/backgrounds.md +63 -0
- package/skill/reference/beat-sync.md +25 -19
- package/skill/reference/brand-motion.md +62 -0
- package/skill/reference/captions.md +11 -5
- package/skill/reference/clips.md +3 -3
- package/skill/reference/component-authoring.md +1 -1
- package/skill/reference/continuity.md +26 -7
- package/skill/reference/delivery-review.md +40 -0
- package/skill/reference/hebrew-rtl.md +3 -4
- package/skill/reference/{remotion-composition.md → hyperframes-composition.md} +18 -11
- package/skill/reference/kit.md +144 -12
- package/skill/reference/launch-film.md +194 -0
- package/skill/reference/motion-design.md +18 -16
- package/skill/reference/scene-treatments.md +20 -0
- package/skill/reference/scriptwriting.md +4 -1
- package/skill/reference/sound-design.md +34 -12
- package/skill/reference/studio-editing.md +55 -0
- package/skill/reference/styles.md +9 -6
- package/skill/reference/three-d.md +135 -0
- package/skill/reference/voice-sync.md +108 -0
- package/src/agents.ts +23 -12
- package/src/api/client.ts +4 -1
- package/src/cli.ts +18 -7
- package/src/commands/assets.ts +314 -30
- package/src/commands/build.ts +148 -35
- package/src/commands/init.ts +1 -1
- package/src/commands/install.ts +1 -1
- package/src/commands/plan.ts +8 -5
- package/src/commands/ref.ts +5 -2
- package/src/contract/index.ts +5 -3
- package/src/hyperframes/Root.tsx +1 -0
- package/src/hyperframes/fonts.ts +54 -0
- package/src/hyperframes/frame.tsx +46 -0
- package/src/hyperframes/host.tsx +38 -0
- package/src/hyperframes/kit/Assemble3D.tsx +92 -0
- package/src/hyperframes/kit/BrandTransform3D.tsx +12 -0
- package/src/hyperframes/kit/BrowserFrame.tsx +83 -0
- package/src/{remotion → hyperframes}/kit/Camera.tsx +7 -5
- package/src/{remotion → hyperframes}/kit/Captions.tsx +34 -18
- package/src/hyperframes/kit/Card3D.tsx +211 -0
- package/src/{remotion → hyperframes}/kit/Carry.tsx +1 -1
- package/src/hyperframes/kit/ChapterFrame.tsx +68 -0
- package/src/{remotion → hyperframes}/kit/ClipLayer.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/Counter.tsx +1 -1
- package/src/hyperframes/kit/CounterRoll.tsx +75 -0
- package/src/{remotion → hyperframes}/kit/Entrance.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/FootageLayer.tsx +1 -1
- package/src/hyperframes/kit/GlassPanel.tsx +43 -0
- package/src/hyperframes/kit/Grounds.tsx +177 -0
- package/src/hyperframes/kit/Headline.tsx +97 -0
- package/src/hyperframes/kit/Hero3D.tsx +197 -0
- package/src/hyperframes/kit/HudOverlay.tsx +52 -0
- package/src/hyperframes/kit/ImageLayers.tsx +48 -0
- package/src/{remotion → hyperframes}/kit/KenBurnsImage.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/KeyedClip.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/Layers.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/LowerThird.tsx +1 -1
- package/src/hyperframes/kit/Music.tsx +19 -0
- package/src/hyperframes/kit/NamedCursor.tsx +54 -0
- package/src/hyperframes/kit/Orbit3D.tsx +49 -0
- package/src/hyperframes/kit/Particles3D.tsx +74 -0
- package/src/hyperframes/kit/Place.tsx +12 -0
- package/src/hyperframes/kit/PromptBox.tsx +84 -0
- package/src/hyperframes/kit/Scene3D.tsx +70 -0
- package/src/hyperframes/kit/SceneFrame.tsx +96 -0
- package/src/{remotion → hyperframes}/kit/ScreenOverlay.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/Sfx.tsx +1 -1
- package/src/hyperframes/kit/SoundCues.tsx +22 -0
- package/src/hyperframes/kit/TerminalLog.tsx +98 -0
- package/src/hyperframes/kit/Text3D.tsx +78 -0
- package/src/hyperframes/kit/TextOnImage.tsx +41 -0
- package/src/{remotion → hyperframes}/kit/TitleCard.tsx +1 -1
- package/src/{remotion → hyperframes}/kit/Voiceover.tsx +1 -1
- package/src/hyperframes/kit/Warp3D.tsx +59 -0
- package/src/hyperframes/kit/bg-math.ts +179 -0
- package/src/hyperframes/kit/brand-transform.ts +25 -0
- package/src/{remotion → hyperframes}/kit/caption-groups.ts +7 -3
- package/src/hyperframes/kit/caption-style.ts +45 -0
- package/src/hyperframes/kit/docs.ts +249 -0
- package/src/hyperframes/kit/image-layers-math.ts +115 -0
- package/src/hyperframes/kit/index.ts +74 -0
- package/src/hyperframes/kit/inter-bold-typeface.ts +3 -0
- package/src/{remotion → hyperframes}/kit/motion-math.ts +36 -2
- package/src/hyperframes/kit/music-math.ts +59 -0
- package/src/hyperframes/kit/quiet-three.ts +11 -0
- package/src/hyperframes/kit/sample-text.ts +55 -0
- package/src/hyperframes/kit/scene3d-context.ts +5 -0
- package/src/hyperframes/kit/seeded.ts +13 -0
- package/src/hyperframes/kit/sound-cues.ts +89 -0
- package/src/hyperframes/kit/sound-kinds.ts +135 -0
- package/src/{remotion → hyperframes}/kit/theme.ts +43 -39
- package/src/hyperframes/kit/three-fx-math.ts +192 -0
- package/src/hyperframes/kit/three-math.ts +145 -0
- package/src/hyperframes/kit/transition-math.ts +116 -0
- package/src/hyperframes/kit/ui-math.ts +145 -0
- package/src/hyperframes/kit/ui-theme.ts +25 -0
- package/src/hyperframes/kit/word-anchor.ts +107 -0
- package/src/hyperframes/math.ts +62 -0
- package/src/hyperframes/three.tsx +10 -0
- package/src/pipeline/beatsnap.ts +72 -0
- package/src/pipeline/review.ts +44 -10
- package/src/pipeline/schema.ts +51 -4
- package/src/pipeline/timing.ts +27 -1
- package/src/project/background.ts +33 -0
- package/src/project/chromakey.ts +1 -1
- package/src/project/layers.ts +60 -0
- package/src/project/manifest.ts +59 -14
- package/src/project/music.ts +19 -5
- package/src/project/project.ts +4 -1
- package/src/project/serve.ts +2 -2
- package/src/project/soundreport.ts +347 -0
- package/src/project/svgcheck.ts +21 -0
- package/src/render/component-preview.ts +11 -55
- package/src/render/contact-sheet.ts +39 -0
- package/src/render/continuity.ts +14 -4
- package/src/render/deps.ts +15 -3
- package/src/render/render.ts +62 -57
- package/src/render/serve.ts +31 -0
- package/src/render/sound-notes.ts +106 -0
- package/src/render/static-check.ts +15 -4
- package/src/render/validate.ts +4 -4
- package/src/render/word-check.ts +181 -0
- package/src/render/worker.ts +71 -0
- package/src/testing/conformance.ts +12 -0
- package/src/testing/fake-api.ts +4 -4
- package/src/testing/fixtures.ts +4 -1
- package/src/remotion/Root.tsx +0 -31
- package/src/remotion/kit/Music.tsx +0 -19
- package/src/remotion/kit/SceneFrame.tsx +0 -19
- package/src/remotion/kit/docs.ts +0 -124
- package/src/remotion/kit/index.ts +0 -29
- package/src/remotion/kit/music-math.ts +0 -42
- /package/src/{remotion → hyperframes}/kit/Icon.tsx +0 -0
- /package/src/{remotion → hyperframes}/kit/beat.ts +0 -0
- /package/src/{remotion → hyperframes}/kit/brand-icons.ts +0 -0
- /package/src/{remotion → hyperframes}/kit/media.ts +0 -0
- /package/src/{remotion → hyperframes}/types.ts +0 -0
package/src/commands/assets.ts
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
3
|
-
import { basename, dirname, resolve } from "node:path";
|
|
2
|
+
import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { basename, dirname, relative, resolve } from "node:path";
|
|
4
4
|
import { ApiFailure, download, uploadTo } from "../api/client";
|
|
5
5
|
import { client, type Ctx, type Result } from "../context";
|
|
6
6
|
import { LibraryKindSchema, MAX_CUTOUT_SECONDS, MAX_UPLOAD_BYTES, UploadKindSchema, type CutoutType } from "../contract";
|
|
7
|
-
import { PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
|
|
7
|
+
import { gapSec, isVoiceless, LAST_TAIL_SEC, PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
|
|
8
8
|
import { GreenTooDullError, keyGreen } from "../project/chromakey";
|
|
9
|
-
import { buildManifest, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
|
|
9
|
+
import { buildManifest, sentenceGaps, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
|
|
10
10
|
import { loadPlan } from "./plan";
|
|
11
11
|
import { FILE_NAME } from "../render/validate";
|
|
12
|
+
import { migrateFrameImports } from "../render/static-check";
|
|
12
13
|
import { levelOf } from "../project/loudness";
|
|
13
|
-
import { measureMusic, type MusicRecord } from "../project/music";
|
|
14
|
+
import { beatReport, measureMusic, type MusicRecord } from "../project/music";
|
|
15
|
+
import { audioDuration } from "../project/refmeasure";
|
|
16
|
+
import { BACKGROUND_CLIP_SENTENCE, BACKGROUND_IMAGE_SENTENCE, isVideoPath, lookFromNotes, readBackground, setBackground } from "../project/background";
|
|
14
17
|
import { probeFile } from "../project/probe";
|
|
15
|
-
import {
|
|
18
|
+
import { frameWithAlpha, measureTextZone, stillToVideo, tempDir, type LayerEntry } from "../project/layers";
|
|
19
|
+
import { coverageVerdict } from "../hyperframes/kit/image-layers-math";
|
|
20
|
+
import { svgProblem } from "../project/svgcheck";
|
|
21
|
+
import { FILES, isPrivateProject, openProject, type Project } from "../project/project";
|
|
16
22
|
|
|
17
23
|
// gainDb is the levelling of a sound (sfx or music), in dB, measured when it was pulled: see src/project/loudness.ts.
|
|
18
24
|
export type LibraryEntry = { path: string; kind: string; title: string; meta: Record<string, unknown>; gainDb?: number };
|
|
@@ -49,7 +55,7 @@ const shellArg = (s: string) => (/^[\w./~@%+=:,-]+$/.test(s) ? s : JSON.stringif
|
|
|
49
55
|
|
|
50
56
|
// Sends a video to the server and starts its cutout. The seconds are reserved when the job is run, so this is the point where it becomes
|
|
51
57
|
// paid for: nothing is charged by a failure before it, and the id it returns is the only handle on the job after it.
|
|
52
|
-
async function startCutout(ctx: Ctx, input: CutoutInput): Promise<string> {
|
|
58
|
+
export async function startCutout(ctx: Ctx, input: CutoutInput): Promise<string> {
|
|
53
59
|
const api = client(ctx);
|
|
54
60
|
ctx.log(`Sending ${input.filename} to the Reelkit server to cut the subject out.`);
|
|
55
61
|
const up = await api("cutoutStart", { filename: input.filename.slice(-200), contentType: input.contentType, bytes: input.bytes, durationSec: input.durationSec });
|
|
@@ -63,7 +69,7 @@ type CutoutOutcome = { kind: "done" } | { kind: "failed"; message: string } | {
|
|
|
63
69
|
|
|
64
70
|
// Asks every few seconds until the job ends (at most ten minutes), then saves the result at `dest`. A job that is not found or a login that
|
|
65
71
|
// is wrong has nothing to resume and is thrown; any other trouble may be a hiccup on a job that is paid for, so it is an outcome.
|
|
66
|
-
async function pollCutout(ctx: Ctx, id: string, dest: string, deps: CutoutDeps): Promise<CutoutOutcome> {
|
|
72
|
+
export async function pollCutout(ctx: Ctx, id: string, dest: string, deps: CutoutDeps): Promise<CutoutOutcome> {
|
|
67
73
|
const api = client(ctx);
|
|
68
74
|
const now = deps.now ?? Date.now;
|
|
69
75
|
const started = now();
|
|
@@ -101,7 +107,7 @@ function cutoutRefusal(file: string, probe: Awaited<ReturnType<typeof probeFile>
|
|
|
101
107
|
|
|
102
108
|
// Registers one of the user's own files. It stays on this machine unless --share is given.
|
|
103
109
|
export async function assetsUpload(
|
|
104
|
-
ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; green?: boolean; cutout?: boolean; resume?: string; share?: boolean; kind?: string; tags?: string },
|
|
110
|
+
ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; green?: boolean; cutout?: boolean; resume?: string; share?: boolean; kind?: string; tags?: string; background?: boolean },
|
|
105
111
|
deps: { key?: typeof keyGreen } & CutoutDeps = {},
|
|
106
112
|
): Promise<Result> {
|
|
107
113
|
const project = openProject(ctx.cwd);
|
|
@@ -116,6 +122,8 @@ export async function assetsUpload(
|
|
|
116
122
|
} catch (e) {
|
|
117
123
|
return { ok: false, summary: e instanceof Error ? e.message : String(e) };
|
|
118
124
|
}
|
|
125
|
+
if (opts.background && probe.kind === "audio") return { ok: false, summary: `${file} is audio, so it cannot be the film's background. Use a video or an image.` };
|
|
126
|
+
if (opts.background && (opts.footage || opts.green || opts.cutout)) return { ok: false, summary: "--background makes the file the ground behind everything; it does not go with --footage, --green or --cutout." };
|
|
119
127
|
if (opts.footage && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so it cannot be the footage.` };
|
|
120
128
|
if (opts.green && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so there is no green screen to key out.` };
|
|
121
129
|
const refusal = opts.cutout ? cutoutRefusal(file, probe) : undefined;
|
|
@@ -143,7 +151,12 @@ export async function assetsUpload(
|
|
|
143
151
|
}
|
|
144
152
|
const record: AssetRecord = { id, filename, key, ...probe, ...(opts.describe ? { description: opts.describe } : {}), ...(keyedKey ? { keyedKey } : {}), ...(cutoutId ? { cutoutId } : {}), createdAt: new Date().toISOString() };
|
|
145
153
|
project.writeJson(FILES.assetIndex, [...project.assets(), record]);
|
|
154
|
+
if (probe.kind === "image" && !/\.svg$/i.test(filename)) await recordZone(project, key, true);
|
|
146
155
|
if (opts.footage) project.writeJson(FILES.config, { ...project.config(), footage: id });
|
|
156
|
+
if (opts.background) {
|
|
157
|
+
setBackground(project, { key, id, title: filename, ...(probe.kind === "video" && probe.durationSec ? { durationSec: probe.durationSec } : {}) });
|
|
158
|
+
if (project.exists(FILES.plan)) tryManifest(project, loadPlan(project));
|
|
159
|
+
}
|
|
147
160
|
|
|
148
161
|
let libraryId: string | undefined;
|
|
149
162
|
if (opts.share && kind?.success) {
|
|
@@ -163,7 +176,7 @@ export async function assetsUpload(
|
|
|
163
176
|
}
|
|
164
177
|
return {
|
|
165
178
|
ok: true, data: { ...record, ...(libraryId ? { libraryId } : {}) },
|
|
166
|
-
summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.${keyedKey ? ` The green is keyed out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"].` : ""}`,
|
|
179
|
+
summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${opts.background ? `, set as the film's background: use manifest.background (urls["${key}"])` : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.${keyedKey ? ` The green is keyed out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"].` : ""}`,
|
|
167
180
|
};
|
|
168
181
|
}
|
|
169
182
|
|
|
@@ -215,6 +228,15 @@ function recordSearch(ctx: Ctx, q: string, kind: string | null, items: { id: str
|
|
|
215
228
|
} catch { /* not a project, or the file is unreadable: the search itself still works */ }
|
|
216
229
|
}
|
|
217
230
|
|
|
231
|
+
// The length and, for music, the tempo of a result, at the end of its line, when the library has them. Sounds are only useful once their length is known:
|
|
232
|
+
// a riser must be as long as the build it scores, and a track must have the tempo the cuts will follow.
|
|
233
|
+
export function lengthOf(item: { kind: string; meta: Record<string, unknown> }): string {
|
|
234
|
+
if (item.kind !== "music" && item.kind !== "sfx") return "";
|
|
235
|
+
const d = item.meta.durationSec, bpm = item.meta.bpm;
|
|
236
|
+
const parts = [typeof d === "number" && d > 0 ? `${Math.round(d * 10) / 10}s` : "", item.kind === "music" && typeof bpm === "number" && bpm > 0 ? `${Math.round(bpm)} BPM` : ""].filter(Boolean);
|
|
237
|
+
return parts.length ? ` (${parts.join(", ")})` : "";
|
|
238
|
+
}
|
|
239
|
+
|
|
218
240
|
export async function assetsSearch(ctx: Ctx, query: string, opts: { kind?: string; limit?: string }): Promise<Result> {
|
|
219
241
|
const kind = opts.kind === undefined ? undefined : LibraryKindSchema.safeParse(opts.kind);
|
|
220
242
|
if (kind && !kind.success) return { ok: false, summary: `Unknown kind "${opts.kind}". Use one of: ${LibraryKindSchema.options.join(", ")}.` };
|
|
@@ -223,7 +245,7 @@ export async function assetsSearch(ctx: Ctx, query: string, opts: { kind?: strin
|
|
|
223
245
|
return {
|
|
224
246
|
ok: true, data: { items },
|
|
225
247
|
summary: items.length
|
|
226
|
-
? items.map((i) => `${`${Math.round(i.match * 100)}%`.padStart(4)} ${i.id} ${i.kind} ${i.title}: ${i.description} [${i.tags.join(", ")}]`).join("\n")
|
|
248
|
+
? items.map((i) => `${`${Math.round(i.match * 100)}%`.padStart(4)} ${i.id} ${i.kind} ${i.title}: ${i.description} [${i.tags.join(", ")}]${lengthOf(i)}`).join("\n")
|
|
227
249
|
: "No matches in the library. Generate it instead.",
|
|
228
250
|
};
|
|
229
251
|
}
|
|
@@ -244,10 +266,13 @@ function setSceneClip(project: Project, sceneId: string, record: ClipRecord) {
|
|
|
244
266
|
project.writeJson(FILES.clips, { ...project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {}), [sceneId]: record });
|
|
245
267
|
}
|
|
246
268
|
|
|
247
|
-
export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean; music?: boolean }, deps: ClipDeps = {}): Promise<Result> {
|
|
269
|
+
export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean; music?: boolean; background?: boolean; layers?: boolean }, deps: ClipDeps = {}): Promise<Result> {
|
|
270
|
+
if (opts.layers && (!opts.scene || opts.background || opts.music)) return { ok: false, summary: "--layers goes with --scene: it splits the scene's image into layers right after it is pulled." };
|
|
248
271
|
const project = openProject(ctx.cwd);
|
|
249
272
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id)) return { ok: false, summary: `"${id}" is not a library id. Copy the id from \`reelkit assets search\`.` };
|
|
250
|
-
if (opts.
|
|
273
|
+
if (opts.background && opts.music) return { ok: false, summary: "--background makes the item the ground behind the picture and --music makes it the track. Use one." };
|
|
274
|
+
if (opts.background && opts.scene && !loadPlan(project).scenes.some((s) => s.id === opts.scene)) return { ok: false, summary: `Scene ${opts.scene} is not in plan.json.` };
|
|
275
|
+
if (opts.scene && !opts.background) {
|
|
251
276
|
// Checked before anything is downloaded.
|
|
252
277
|
const scenes = loadPlan(project).scenes;
|
|
253
278
|
const illustrations = scenes.filter((s) => s.treatment === "illustration").map((s) => s.id);
|
|
@@ -256,11 +281,13 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
|
|
|
256
281
|
return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}${clipScenes.length ? ` The clip scenes are: ${clipScenes.join(", ")}.` : ""}` };
|
|
257
282
|
}
|
|
258
283
|
}
|
|
259
|
-
if (opts.music && opts.scene) return { ok: false, summary: "--music makes the item the video's track, not a scene's picture. Use --music or --scene, not both." };
|
|
284
|
+
if (opts.music && opts.scene && !opts.background) return { ok: false, summary: "--music makes the item the video's track, not a scene's picture. Use --music or --scene, not both." };
|
|
260
285
|
const pulled = await client(ctx)("libraryPull", { id });
|
|
261
286
|
if (opts.music && pulled.item.kind !== "music") return { ok: false, summary: `${id} is ${pulled.item.kind}, not music, so it cannot be the video's track. Search with --kind music.` };
|
|
262
287
|
if (unsafeName(pulled.filename, pulled.item.kind)) return { ok: false, summary: `The library returned an unsafe file name for ${id}, so nothing was written. Report this item.` };
|
|
263
|
-
if (opts.
|
|
288
|
+
if (opts.background && !["image", "clip", "overlay"].includes(pulled.item.kind)) return { ok: false, summary: `${id} is ${pulled.item.kind}, so it cannot be a background. Search with --kind clip, overlay or image.` };
|
|
289
|
+
if (opts.background && opts.scene && isVideoPath(pulled.filename)) return { ok: false, summary: `${id} is a video. A video is the ground of the whole film, so use --background without --scene; a scene's ground must be a picture.` };
|
|
290
|
+
if (opts.scene && !opts.background) {
|
|
264
291
|
const treatment = loadPlan(project).scenes.find((x) => x.id === opts.scene)?.treatment;
|
|
265
292
|
if (treatment === "clip" && pulled.item.kind !== "clip") return { ok: false, summary: `${id} is ${pulled.item.kind}, not a clip, so it cannot be scene ${opts.scene}'s clip.` };
|
|
266
293
|
if (treatment !== "clip" && pulled.item.kind !== "image") return { ok: false, summary: `${id} is ${pulled.item.kind}, not an image, so it cannot be a scene's image.` };
|
|
@@ -270,6 +297,9 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
|
|
|
270
297
|
const path = `src/${pulled.filename}`;
|
|
271
298
|
if (project.exists(path) && !opts.force) return { ok: false, summary: `${path} already exists. Use --force to replace it with the library version.` };
|
|
272
299
|
await download(pulled.url, project.path(path));
|
|
300
|
+
const source = readFileSync(project.path(path), "utf8");
|
|
301
|
+
const migrated = migrateFrameImports(source);
|
|
302
|
+
if (migrated !== source) writeFileSync(project.path(path), migrated);
|
|
273
303
|
recordPull(project, id, { path, kind: "component", title: pulled.item.title, meta: pulled.item.meta });
|
|
274
304
|
const name = pulled.filename.replace(/\.tsx$/, "");
|
|
275
305
|
const example = typeof pulled.item.meta.example === "string" ? `\nExample: ${pulled.item.meta.example}` : "";
|
|
@@ -281,26 +311,39 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
|
|
|
281
311
|
// measured is used as it is.
|
|
282
312
|
const level = pulled.item.kind === "sfx" || pulled.item.kind === "music" ? await levelOf(project.path(path)) : undefined;
|
|
283
313
|
recordPull(project, id, { path, kind: pulled.item.kind, title: pulled.item.title, meta: pulled.item.meta, ...(level ? { gainDb: level.gainDb } : {}) });
|
|
314
|
+
if (opts.background) {
|
|
315
|
+
const video = isVideoPath(path);
|
|
316
|
+
let durationSec = typeof pulled.item.meta.durationSec === "number" ? pulled.item.meta.durationSec : undefined;
|
|
317
|
+
if (video && durationSec === undefined) { try { durationSec = Math.round((await audioDuration(project.path(path))) * 100) / 100; } catch { durationSec = undefined; } }
|
|
318
|
+
const previous = readBackground(project)[opts.scene ? "scenes" : "film"];
|
|
319
|
+
const had = opts.scene ? (previous as Record<string, { id?: string }>)[opts.scene] : (previous as { id?: string } | undefined);
|
|
320
|
+
setBackground(project, { key: path, id, title: pulled.item.title, ...(video && durationSec ? { durationSec } : {}) }, opts.scene);
|
|
321
|
+
const note = project.exists(FILES.plan) ? tryManifest(project, loadPlan(project)).note : undefined;
|
|
322
|
+
const where = opts.scene ? `scene ${opts.scene}'s background` : "the film's background";
|
|
323
|
+
return { ok: true, data: { path, kind: pulled.item.kind, background: true, ...(opts.scene ? { scene: opts.scene } : {}) }, summary: [`Pulled ${id} as ${where}${had?.id && had.id !== id ? ` (it replaces ${had.id})` : ""}. In the composition use ${opts.scene ? "the scene's `background.key`" : "`manifest.background`"}: <${video ? "BgVideo" : "BgImage"} src={urls["${path}"]} />. Read reference/backgrounds.md.`, ...(note ? [note] : [])].join("\n") };
|
|
324
|
+
}
|
|
284
325
|
// A clip filmed on green also comes keyed, so it can be laid over a background.
|
|
285
326
|
const green = pulled.item.kind === "clip" && pulled.item.meta.greenScreen === true;
|
|
286
327
|
const keyedPath = green ? keyedPathOf(path) : undefined;
|
|
287
328
|
if (keyedPath) await (deps.key ?? keyGreen)(project.path(path), project.path(keyedPath));
|
|
288
329
|
const sceneIsClip = pulled.item.kind === "clip";
|
|
289
330
|
let note: string | undefined;
|
|
331
|
+
let layered: Result | undefined;
|
|
290
332
|
if (opts.scene) {
|
|
291
333
|
if (sceneIsClip) {
|
|
292
334
|
const durationSec = typeof pulled.item.meta.durationSec === "number" ? pulled.item.meta.durationSec : 5;
|
|
293
335
|
setSceneClip(project, opts.scene, { key: path, greenScreen: green, durationSec, ...(keyedPath ? { keyedKey: keyedPath } : {}) });
|
|
294
|
-
} else setSceneImage(project, opts.scene, path);
|
|
336
|
+
} else { setSceneImage(project, opts.scene, path); await recordZone(project, path, true); if (opts.layers) layered = await assetsLayers(ctx, undefined, { scene: opts.scene, redo: true }, deps); }
|
|
295
337
|
if (project.exists(FILES.plan)) note = tryManifest(project, loadPlan(project)).note;
|
|
296
338
|
}
|
|
297
339
|
if (pulled.item.kind === "music" && opts.music) return pullMusic(project, id, pulled.item.title, path, level?.gainDb ?? 0);
|
|
298
340
|
const musicHint = pulled.item.kind === "music" ? ` To make it the video's track, with scene changes on its beat, run \`reelkit assets pull ${id} --music\`.` : "";
|
|
299
341
|
const refs = keyedPath ? ` The original is urls["${path}"]; the keyed copy with a transparent background is urls["${keyedPath}"].` : ` Reference it in the composition as urls["${path}"].`;
|
|
300
342
|
return {
|
|
301
|
-
ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}) },
|
|
343
|
+
ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}), ...(layered ? { layers: layered.data } : {}) },
|
|
302
344
|
summary: [
|
|
303
345
|
opts.scene ? `Pulled ${id} as the ${sceneIsClip ? "clip" : "image"} for scene ${opts.scene}.${keyedPath ? refs : ""}` : `Pulled ${id} to ${path}.${refs}${musicHint}`,
|
|
346
|
+
...(layered ? [layered.summary] : []),
|
|
304
347
|
...(note ? [note] : []),
|
|
305
348
|
].join("\n"),
|
|
306
349
|
};
|
|
@@ -316,7 +359,7 @@ async function pullMusic(project: Project, id: string, title: string, path: stri
|
|
|
316
359
|
project.writeJson(FILES.music, record);
|
|
317
360
|
const note = project.exists(FILES.plan) ? tryManifest(project, loadPlan(project)).note : undefined;
|
|
318
361
|
const beat = record.bpm
|
|
319
|
-
? `about ${Math.round(record.bpm)} BPM; scene changes will land on its beat.`
|
|
362
|
+
? `about ${Math.round(record.bpm)} BPM; scene changes will land on its beat${project.exists(FILES.plan) && !isVoiceless(loadPlan(project)) ? " where that is within 4 frames of where the voice puts them (the others follow the voice)" : ""}.`
|
|
320
363
|
: "no clear tempo was found, so scene changes stay where the narration puts them.";
|
|
321
364
|
return {
|
|
322
365
|
ok: true, data: { path, kind: "music", music: record },
|
|
@@ -324,11 +367,18 @@ async function pullMusic(project: Project, id: string, title: string, path: stri
|
|
|
324
367
|
};
|
|
325
368
|
}
|
|
326
369
|
|
|
370
|
+
// Female voices first, then neutral and unmarked, then male, each group keeping the server's order: the first one listed is the default offer.
|
|
371
|
+
export function orderVoices<T extends { gender?: string }>(voices: readonly T[]): T[] {
|
|
372
|
+
const rank = (v: T) => (/^f/i.test(v.gender ?? "") ? 0 : /^m/i.test(v.gender ?? "") ? 2 : 1);
|
|
373
|
+
return voices.map((v, i) => ({ v, i })).sort((a, b) => rank(a.v) - rank(b.v) || a.i - b.i).map((x) => x.v);
|
|
374
|
+
}
|
|
375
|
+
|
|
327
376
|
export async function assetsVoices(ctx: Ctx): Promise<Result> {
|
|
328
|
-
const
|
|
377
|
+
const listed = await client(ctx)("voices", {});
|
|
378
|
+
const voices = orderVoices(listed.voices);
|
|
329
379
|
return {
|
|
330
380
|
ok: true, data: { voices },
|
|
331
|
-
summary: voices.map((v) => `${v.id} ${v.name} (${[v.gender, v.accent, v.language].filter(Boolean).join(", ")}): ${v.description}`).join("\n"),
|
|
381
|
+
summary: [...voices.map((v) => `${v.id} ${v.name} (${[v.gender, v.accent, v.language].filter(Boolean).join(", ")}): ${v.description}`), "The plan needs a voiceId from this list (`reelkit plan check` asks for one); the first voice listed is the default offer to the user."].join("\n"),
|
|
332
382
|
};
|
|
333
383
|
}
|
|
334
384
|
|
|
@@ -336,6 +386,8 @@ export async function assetsVoices(ctx: Ctx): Promise<Result> {
|
|
|
336
386
|
export async function assetsVoiceover(ctx: Ctx, opts: { scene?: string; all?: boolean; redo?: boolean }): Promise<Result> {
|
|
337
387
|
const project = openProject(ctx.cwd);
|
|
338
388
|
const plan = loadPlan(project);
|
|
389
|
+
// Said as success so that an agent following the older steps carries on instead of stopping.
|
|
390
|
+
if (isVoiceless(plan)) return { ok: true, data: { recorded: [], voice: "none" }, summary: "This video has no voice (voice is \"none\" in plan.json), so there is nothing to record. Go on to the music and the composition." };
|
|
339
391
|
if (!opts.all && !opts.scene) return { ok: false, summary: "Say which scene with --scene <id>, or use --all." };
|
|
340
392
|
const wanted = opts.all ? plan.scenes : plan.scenes.filter((s) => s.id === opts.scene);
|
|
341
393
|
if (!wanted.length) return { ok: false, summary: `No scene with id ${opts.scene}. The scenes are: ${plan.scenes.map((s) => s.id).join(", ")}.` };
|
|
@@ -356,39 +408,115 @@ export async function assetsVoiceover(ctx: Ctx, opts: { scene?: string; all?: bo
|
|
|
356
408
|
}
|
|
357
409
|
const { manifest, note } = tryManifest(project, plan);
|
|
358
410
|
const left = plan.scenes.filter((s) => !done[s.id] || voiceoverStale(done[s.id], s, plan)).map((s) => s.id);
|
|
411
|
+
const gaps = manifest ? sentenceGaps(manifest) : [];
|
|
412
|
+
const beat = manifest ? beatReport(manifest) : undefined;
|
|
359
413
|
const recorded = `Recorded ${todo.length} scene(s)${rerecorded ? ` (${rerecorded} re-recorded because the script or voice changed)` : ""}.`;
|
|
360
414
|
return {
|
|
361
|
-
ok: true, data: { recorded: todo.map((s) => s.id), rerecorded, voiceovers: done, totalSeconds: manifest ? manifest.totalFrames / manifest.fps : null },
|
|
415
|
+
ok: true, data: { recorded: todo.map((s) => s.id), rerecorded, voiceovers: done, totalSeconds: manifest ? manifest.totalFrames / manifest.fps : null, ...(manifest ? { sentenceGaps: gaps, ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) } : {}) },
|
|
362
416
|
summary: [
|
|
363
417
|
manifest
|
|
364
|
-
? `${recorded} The video runs ${(manifest.totalFrames / manifest.fps).toFixed(1)}s. Voiceover files are in assets/.${left.length ? ` Still out of date: ${left.join(", ")}. Run \`reelkit assets voiceover --all\`.` : ""}`
|
|
418
|
+
? `${recorded} The video runs ${(manifest.totalFrames / manifest.fps).toFixed(1)}s (the last scene keeps ${LAST_TAIL_SEC}s after its last word). Voiceover files are in assets/.${left.length ? ` Still out of date: ${left.join(", ")}. Run \`reelkit assets voiceover --all\`.` : ""}\nSilence between sentences (plan gap "${plan.gap ?? "normal"}", ${gapSec(plan)}s): ${gaps.map((g) => `${g.toFixed(2)}s`).join(", ") || "none"}.${beat ? `\nBeat: ${beat.line}.` : ""}`
|
|
365
419
|
: `${recorded}${left.length ? ` Still to record: ${left.join(", ")}.` : ""}`,
|
|
366
420
|
...(note ? [note] : []),
|
|
367
421
|
].join("\n"),
|
|
368
422
|
};
|
|
369
423
|
}
|
|
370
424
|
|
|
371
|
-
|
|
425
|
+
// Said at the end of every image prompt that does not ask for text or a logo: a generated picture with letters in it is almost always wrong.
|
|
426
|
+
// Added to the prompt of a scene that has on-screen text, so that the picture has a clear single subject and calm room for the words.
|
|
427
|
+
export const TEXT_ROOM_SENTENCE = "One clear subject, with calm empty space on one side of the picture where words can sit.";
|
|
428
|
+
export const NO_TEXT_SENTENCE = "No text, letters, numbers, logos or brand marks anywhere in the picture.";
|
|
429
|
+
// What a prompt asks for when it wants writing or a logo in the picture. A clause that forbids them ("no text, no logos") does not count.
|
|
430
|
+
const WANTS_TEXT = /\b(text|texts|letters?|lettering|typography|words?|caption|captions|title|headline|slogan|quote|says|saying|reads|reading|written|writing|sign|signage|label|logo|logos|wordmark|brand ?mark|brand name)\b/i;
|
|
431
|
+
const FORBIDS = /\b(?:no|without|never|avoid|free of|not any|zero)\b[^.;\n]*/gi;
|
|
432
|
+
export const asksForText = (prompt: string): boolean => WANTS_TEXT.test(prompt.replace(FORBIDS, " "));
|
|
433
|
+
// The prompt as sent: with NO_TEXT_SENTENCE after it, unless the prompt itself asks for text or a logo.
|
|
434
|
+
export const withNoText = (prompt: string): string => (asksForText(prompt) ? prompt : `${prompt.trim()}${/[.!?]$/.test(prompt.trim()) ? "" : "."} ${NO_TEXT_SENTENCE}`);
|
|
435
|
+
|
|
436
|
+
export async function assetsGenImage(ctx: Ctx, prompt: string | undefined, opts: { scene?: string; redo?: boolean; background?: boolean; layers?: boolean }, deps: CutoutDeps = {}): Promise<Result> {
|
|
372
437
|
const project = openProject(ctx.cwd);
|
|
373
438
|
const plan = loadPlan(project);
|
|
439
|
+
if (opts.background) return genBackgroundImage(ctx, project, plan, prompt, { ...opts, format: "png" });
|
|
440
|
+
if (!opts.scene) return { ok: false, summary: "Say which scene the image is for with --scene <id>, or use --background for the film's ground." };
|
|
374
441
|
const scene = plan.scenes.find((s) => s.id === opts.scene);
|
|
375
442
|
if (!scene || scene.treatment !== "illustration" || !scene.imagePrompt) return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene, so it does not take an image.` };
|
|
376
443
|
// A scene that already has its image is not charged for again, whatever prompt is given; only --redo asks for a new one.
|
|
377
444
|
const have = project.readJsonOr<Record<string, string>>(FILES.images, {})[scene.id];
|
|
378
|
-
if (have && project.exists(have) && !opts.redo)
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
445
|
+
if (have && project.exists(have) && !opts.redo) {
|
|
446
|
+
const layered = opts.layers ? await assetsLayers(ctx, undefined, { scene: scene.id }, deps) : undefined;
|
|
447
|
+
return { ok: true, data: { path: have, skipped: true }, summary: `Scene ${scene.id} already has an image at ${have}. Use --redo to generate a new one.${layered ? ` ${layered.summary}` : ""}` };
|
|
448
|
+
}
|
|
449
|
+
// Anything tied to the user's footage or uploads stays private, whatever the plan says. So does a prompt changed by hand, and every image of a
|
|
450
|
+
// project made with `reelkit init --private`.
|
|
451
|
+
const priv = isPrivateProject(project);
|
|
452
|
+
const shareable = scene.shareable && !priv && !project.footage() && scene.userAssetIds.length === 0 && !prompt;
|
|
453
|
+
const base = prompt ?? scene.imagePrompt;
|
|
454
|
+
const img = await client(ctx)("images", { prompt: withNoText(scene.onScreenText.length && !prompt ? `${base.trim()}${/[.!?]$/.test(base.trim()) ? "" : "."} ${TEXT_ROOM_SENTENCE}` : base), aspect: plan.aspect, shareable, tags: scene.imageTags, format: "png" });
|
|
382
455
|
const path = `assets/img-${scene.id}.${img.ext}`;
|
|
383
456
|
await download(img.url, project.path(path));
|
|
384
457
|
setSceneImage(project, scene.id, path);
|
|
458
|
+
await recordZone(project, path, true);
|
|
459
|
+
const layered = opts.layers ? await assetsLayers(ctx, undefined, { scene: scene.id, redo: true }, deps) : undefined;
|
|
385
460
|
const { note } = tryManifest(project, plan);
|
|
386
461
|
return {
|
|
387
|
-
ok: true, data: { path, libraryId: img.libraryId ?? null },
|
|
388
|
-
summary: [`Generated the image for scene ${scene.id} at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}.`, ...(note ? [note] : [])].join("\n"),
|
|
462
|
+
ok: true, data: { path, libraryId: img.libraryId ?? null, ...(layered ? { layers: layered.data } : {}) },
|
|
463
|
+
summary: [`Generated the image for scene ${scene.id} at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}.`, ...(layered ? [layered.summary] : []), ...(note ? [note] : [])].join("\n"),
|
|
389
464
|
};
|
|
390
465
|
}
|
|
391
466
|
|
|
467
|
+
// Downloads a generated vector graphic into memory first, checks it, and only then writes it, so a file that fails the check never reaches the project.
|
|
468
|
+
async function saveSvg(url: string, dest: string): Promise<string | undefined> {
|
|
469
|
+
const res = await fetch(url);
|
|
470
|
+
if (!res.ok) return "The generated graphic could not be downloaded. Run the command again.";
|
|
471
|
+
const text = await res.text();
|
|
472
|
+
const problem = svgProblem(text);
|
|
473
|
+
if (problem) return problem;
|
|
474
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
475
|
+
writeFileSync(dest, text);
|
|
476
|
+
return undefined;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// A background made for the film (or for one scene, as a picture): a generic ambient image, never shared, saved and recorded as the ground.
|
|
480
|
+
async function genBackgroundImage(ctx: Ctx, project: Project, plan: ScenePlan, prompt: string | undefined, opts: { scene?: string; redo?: boolean; format: "png" | "svg" }): Promise<Result> {
|
|
481
|
+
if (opts.scene && !plan.scenes.some((s) => s.id === opts.scene)) return { ok: false, summary: `Scene ${opts.scene} is not in plan.json.` };
|
|
482
|
+
const have = readBackground(project)[opts.scene ? "scenes" : "film"];
|
|
483
|
+
const existing = opts.scene ? (have as Record<string, { key: string }>)[opts.scene] : (have as { key: string } | undefined);
|
|
484
|
+
if (existing && project.exists(existing.key) && !opts.redo) return { ok: true, data: { path: existing.key, skipped: true }, summary: `${opts.scene ? `Scene ${opts.scene}` : "The film"} already has a background at ${existing.key}. Use --redo to generate a new one.` };
|
|
485
|
+
const sent = `${prompt ?? `${lookFromNotes(plan.scenes[0]?.notes)}.`} ${BACKGROUND_IMAGE_SENTENCE}`;
|
|
486
|
+
const img = await client(ctx)("images", { prompt: sent, aspect: plan.aspect, shareable: false, tags: [], format: opts.format });
|
|
487
|
+
const path = `assets/bg-${opts.scene ?? "film"}.${img.ext}`;
|
|
488
|
+
if (img.ext === "svg") { const bad = await saveSvg(img.url, project.path(path)); if (bad) return { ok: false, summary: bad }; }
|
|
489
|
+
else await download(img.url, project.path(path));
|
|
490
|
+
setBackground(project, { key: path, title: "generated background" }, opts.scene);
|
|
491
|
+
const { note } = tryManifest(project, plan);
|
|
492
|
+
return { ok: true, data: { path, prompt: sent }, summary: [`Generated ${opts.scene ? `scene ${opts.scene}'s` : "the film's"} background at ${path}. The prompt sent was: ${sent}`, ...(note ? [note] : [])].join("\n") };
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// A vector graphic: icons, logo-like marks, simple illustrations, diagrams. It is one image against the image quota and takes about 45 seconds.
|
|
496
|
+
export async function assetsGenSvg(ctx: Ctx, prompt: string, opts: { scene?: string; redo?: boolean; private?: boolean; background?: boolean }): Promise<Result> {
|
|
497
|
+
const project = openProject(ctx.cwd);
|
|
498
|
+
const plan = loadPlan(project);
|
|
499
|
+
if (!prompt.trim()) return { ok: false, summary: "Say what the graphic shows: `reelkit assets gen svg \"a red rocket icon, flat, transparent background\"`." };
|
|
500
|
+
if (opts.background) return genBackgroundImage(ctx, project, plan, prompt, { ...opts, format: "svg" });
|
|
501
|
+
const scene = opts.scene ? plan.scenes.find((s) => s.id === opts.scene) : undefined;
|
|
502
|
+
if (opts.scene && (!scene || scene.treatment !== "illustration")) return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene, so it does not take a graphic. Leave out --scene to save the file and place it yourself.` };
|
|
503
|
+
const have = scene ? project.readJsonOr<Record<string, string>>(FILES.images, {})[scene.id] : undefined;
|
|
504
|
+
if (scene && have && project.exists(have) && !opts.redo) return { ok: true, data: { path: have, skipped: true }, summary: `Scene ${scene.id} already has an image at ${have}. Use --redo to generate a new one.` };
|
|
505
|
+
// Shared for review only when the plan says the scene is generic and nothing of the user's is in it; --private always keeps it here.
|
|
506
|
+
const shareable = Boolean(scene?.shareable) && !opts.private && !isPrivateProject(project) && !project.footage() && (scene?.userAssetIds.length ?? 1) === 0;
|
|
507
|
+
ctx.log("Drawing the graphic. It takes about 45 seconds.");
|
|
508
|
+
const img = await client(ctx)("images", { prompt, aspect: plan.aspect, shareable, tags: scene?.imageTags ?? [], format: "svg" });
|
|
509
|
+
if (img.ext !== "svg") return { ok: false, summary: "The server sent a picture, not a vector graphic. Nothing was saved; run the command again." };
|
|
510
|
+
const path = scene ? `assets/img-${scene.id}.svg` : `assets/svg-${randomUUID().slice(0, 8)}.svg`;
|
|
511
|
+
const bad = await saveSvg(img.url, project.path(path));
|
|
512
|
+
if (bad) return { ok: false, summary: bad };
|
|
513
|
+
if (scene) setSceneImage(project, scene.id, path);
|
|
514
|
+
const { note } = project.exists(FILES.plan) ? tryManifest(project, plan) : { note: undefined };
|
|
515
|
+
return {
|
|
516
|
+
ok: true, data: { path, libraryId: img.libraryId ?? null },
|
|
517
|
+
summary: [`Generated the graphic at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}. Use it as urls["${path}"] in an <Img>. It counted as one image.`, ...(note ? [note] : [])].join("\n"),
|
|
518
|
+
};
|
|
519
|
+
}
|
|
392
520
|
|
|
393
521
|
const CLIP_POLL_MS = 5_000;
|
|
394
522
|
const CLIP_PROGRESS_MS = 30_000;
|
|
@@ -398,17 +526,22 @@ const CLIP_TIMEOUT_MS = 10 * 60_000;
|
|
|
398
526
|
// The id is the only handle on a paid job, so every way out after the start says it and how to resume.
|
|
399
527
|
export async function assetsGenClip(
|
|
400
528
|
ctx: Ctx, prompt: string | undefined,
|
|
401
|
-
opts: { scene
|
|
529
|
+
opts: { scene?: string; green?: boolean; cutout?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string; background?: boolean },
|
|
402
530
|
deps: ClipDeps = {},
|
|
403
531
|
): Promise<Result> {
|
|
404
532
|
const project = openProject(ctx.cwd);
|
|
405
533
|
const plan = loadPlan(project);
|
|
534
|
+
if (opts.background) return genBackgroundClip(ctx, project, plan, prompt, opts, deps);
|
|
535
|
+
if (!opts.scene) return { ok: false, summary: "Say which scene the clip is for with --scene <id>, or use --background for a clip behind the whole film." };
|
|
406
536
|
const scene = plan.scenes.find((s) => s.id === opts.scene);
|
|
407
537
|
if (!scene || scene.treatment !== "clip" || !scene.clipPrompt) return { ok: false, summary: `Scene ${opts.scene} is not a clip scene, so it does not take a clip. Set its treatment to "clip" and give it a clipPrompt in plan.json.` };
|
|
408
538
|
if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green asks for a green background and keys it on this machine for free, --cutout removes any background on the server." };
|
|
409
539
|
const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
|
|
410
540
|
if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
|
|
411
541
|
if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
|
|
542
|
+
if (opts.share && isPrivateProject(project)) {
|
|
543
|
+
return { ok: false, summary: "This project was made with `reelkit init --private`, so nothing in it goes to the library and --share is refused. Run again without --share." };
|
|
544
|
+
}
|
|
412
545
|
if (opts.share && (!scene.shareable || scene.userAssetIds.length > 0 || project.footage())) {
|
|
413
546
|
return { ok: false, summary: `Scene ${scene.id} is private (it is not marked shareable in plan.json, or it shows your own files), so --share is refused. Run again without --share.` };
|
|
414
547
|
}
|
|
@@ -561,3 +694,154 @@ async function keyScene(project: Project, sceneId: string, record: ClipRecord, k
|
|
|
561
694
|
setSceneClip(project, sceneId, { ...record, greenScreen: true, keyedKey });
|
|
562
695
|
return { ok: true, summary: "" };
|
|
563
696
|
}
|
|
697
|
+
|
|
698
|
+
// A clip meant to sit behind everything: a generic ambient loop, never shared. It uses the same clip quota and job as any clip, and `--resume` works.
|
|
699
|
+
async function genBackgroundClip(ctx: Ctx, project: Project, plan: ScenePlan, prompt: string | undefined, opts: { scene?: string; green?: boolean; cutout?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string }, deps: ClipDeps): Promise<Result> {
|
|
700
|
+
if (opts.scene) return { ok: false, summary: "A background clip is for the whole film, so leave out --scene. A scene's own ground is a picture: `reelkit assets gen image --background --scene <id>`." };
|
|
701
|
+
if (opts.green || opts.cutout) return { ok: false, summary: "--green and --cutout are for a subject, not a background. Leave them out with --background." };
|
|
702
|
+
const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
|
|
703
|
+
if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
|
|
704
|
+
if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
|
|
705
|
+
const have = readBackground(project).film;
|
|
706
|
+
if (have && project.exists(have.key) && !opts.redo && !opts.resume) return { ok: true, data: { path: have.key, skipped: true }, summary: `The film already has a background at ${have.key}. Use --redo to generate a new one.` };
|
|
707
|
+
const now = deps.now ?? Date.now, api = client(ctx), started = now();
|
|
708
|
+
const sent = `${prompt ?? `${lookFromNotes(plan.scenes[0]?.notes)}.`} ${BACKGROUND_CLIP_SENTENCE}`;
|
|
709
|
+
let id = opts.resume;
|
|
710
|
+
if (!id) {
|
|
711
|
+
id = (await api("clipStart", { prompt: sent, aspect: plan.aspect, greenScreen: false, durationSec: seconds, shareable: false, tags: [] })).id;
|
|
712
|
+
ctx.log(`Started background clip ${id}. It usually takes a few minutes.`);
|
|
713
|
+
} else ctx.log(`Waiting for background clip ${id}.`);
|
|
714
|
+
const resume = `reelkit assets gen clip --background --resume ${id}`;
|
|
715
|
+
try {
|
|
716
|
+
let lastLog = started;
|
|
717
|
+
for (;;) {
|
|
718
|
+
const st = await api("clipStatus", { id });
|
|
719
|
+
if (st.status === "failed") return { ok: false, data: { id, status: "failed" }, summary: `The background clip failed: ${st.message ?? "the video provider gave no reason."} You were not charged. Change the prompt and run the command again.` };
|
|
720
|
+
if (st.status === "done") {
|
|
721
|
+
const path = `assets/bg-film.${st.ext}`;
|
|
722
|
+
await download(st.url!, project.path(path));
|
|
723
|
+
const durationSec = st.durationSec ?? seconds;
|
|
724
|
+
setBackground(project, { key: path, title: "generated background", durationSec });
|
|
725
|
+
const { note } = tryManifest(project, plan);
|
|
726
|
+
return { ok: true, data: { id, path, durationSec, prompt: sent }, summary: [`Generated the film's background clip at ${path} (${durationSec}s). The prompt sent was: ${sent} Use it with <BgVideo src={urls["${path}"]} durationSec={manifest.background?.durationSec} />.`, ...(note ? [note] : [])].join("\n") };
|
|
727
|
+
}
|
|
728
|
+
const elapsed = now() - started;
|
|
729
|
+
if (elapsed >= CLIP_TIMEOUT_MS) return { ok: false, data: { id, status: "pending" }, summary: `The background clip (id ${id}) is not ready after ${CLIP_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${resume}\` to keep waiting for it; that does not charge again.` };
|
|
730
|
+
if (now() - lastLog >= CLIP_PROGRESS_MS) { lastLog = now(); ctx.log(`Still making the background clip (${Math.round(elapsed / 1000)}s so far)...`); }
|
|
731
|
+
await ctx.sleep(CLIP_POLL_MS);
|
|
732
|
+
}
|
|
733
|
+
} catch (e) {
|
|
734
|
+
if (e instanceof ApiFailure && (e.code === "not_found" || e.code === "unauthenticated")) throw e;
|
|
735
|
+
return { ok: false, data: { id }, summary: `Lost contact while the background clip (id ${id}) was being made: ${e instanceof Error ? e.message : String(e)} The clip is not lost: run \`${resume}\`; that does not charge again.` };
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
// ---- Layers: a still picture in layers, with the words as a layer of it ----
|
|
741
|
+
|
|
742
|
+
// Measures where words can sit in a picture (free, on this machine) and records it beside the picture's layers. With `fresh` the record starts again, so that
|
|
743
|
+
// a new picture never keeps the subject of the one it replaces. A picture ffmpeg cannot read is recorded as nothing: the layers are an addition, never a failure.
|
|
744
|
+
async function recordZone(project: Project, key: string, fresh = false): Promise<LayerEntry | undefined> {
|
|
745
|
+
try {
|
|
746
|
+
const all = project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {});
|
|
747
|
+
const old = fresh ? undefined : all[key];
|
|
748
|
+
const textZone = await measureTextZone(project.path(key), old?.subjectBox);
|
|
749
|
+
const entry: LayerEntry = { ...(old ?? {}), textZone };
|
|
750
|
+
project.writeJson(FILES.layers, { ...all, [key]: entry });
|
|
751
|
+
return entry;
|
|
752
|
+
} catch { return undefined; }
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
function saveLayerEntry(project: Project, key: string, entry: LayerEntry) {
|
|
756
|
+
project.writeJson(FILES.layers, { ...project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {}), [key]: entry });
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
const subjectPathOf = (key: string) => key.replace(/\.[^./]+$/, "") + ".subject.png";
|
|
760
|
+
|
|
761
|
+
// Makes the picture a scene's image is made of into layers: the subject cut out of it (a PNG with alpha beside it, `<name>.subject.png`), its box, and where words can
|
|
762
|
+
// sit. It reuses the service that removes the background of a video: the still is sent as a one-second video, so it uses about 1 second of the cutout quota, and one
|
|
763
|
+
// frame of the transparent video that comes back is kept. A picture with no clear subject (the cut-out covers under 3% or over 85% of it) stays flat and the command
|
|
764
|
+
// still succeeds. `file` is a picture in the project; `scene` is a scene's image.
|
|
765
|
+
export async function assetsLayers(ctx: Ctx, file: string | undefined, opts: { scene?: string; redo?: boolean; resume?: string }, deps: CutoutDeps = {}): Promise<Result> {
|
|
766
|
+
const project = openProject(ctx.cwd);
|
|
767
|
+
if (!file && !opts.scene) return { ok: false, summary: "Say which picture: `reelkit assets layers --scene <id>` for a scene's image, or `reelkit assets layers <file>` for a picture in the project." };
|
|
768
|
+
if (file && opts.scene) return { ok: false, summary: "Give a file or --scene, not both." };
|
|
769
|
+
if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a cutout id. Copy it from the earlier message.` };
|
|
770
|
+
let key: string;
|
|
771
|
+
if (opts.scene) {
|
|
772
|
+
const have = project.readJsonOr<Record<string, string>>(FILES.images, {})[opts.scene];
|
|
773
|
+
if (!have || !project.exists(have)) return { ok: false, summary: `Scene ${opts.scene} has no image yet. Make one with \`reelkit assets gen image --scene ${opts.scene}\` or pull one with \`reelkit assets pull <id> --scene ${opts.scene}\`.` };
|
|
774
|
+
if (/\.(svg|mp4|webm|mov)$/i.test(have)) return { ok: false, summary: `Scene ${opts.scene}'s picture ${have} is not a photo or a raster picture, so it cannot be split into layers.` };
|
|
775
|
+
key = have;
|
|
776
|
+
} else {
|
|
777
|
+
const abs = resolve(ctx.cwd, file!);
|
|
778
|
+
const rel = relative(project.dir, abs);
|
|
779
|
+
if (rel.startsWith("..") || !project.exists(rel)) return { ok: false, summary: `${file} is not a file in this project. Add it with \`reelkit assets upload ${file}\` and give the path it is stored at.` };
|
|
780
|
+
let probe: Awaited<ReturnType<typeof probeFile>>;
|
|
781
|
+
try { probe = await probeFile(abs); } catch (e) { return { ok: false, summary: e instanceof Error ? e.message : String(e) }; }
|
|
782
|
+
if (probe.kind !== "image") return { ok: false, summary: `${file} is ${probe.kind}, not a picture, so it has no layers.` };
|
|
783
|
+
key = rel;
|
|
784
|
+
}
|
|
785
|
+
const tag = opts.scene ? ` for scene ${opts.scene}` : "";
|
|
786
|
+
const retry = `reelkit assets layers ${opts.scene ? `--scene ${opts.scene}` : shellArg(key)}`;
|
|
787
|
+
const entry0 = project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {})[key];
|
|
788
|
+
const subjectKey = subjectPathOf(key);
|
|
789
|
+
const refreshManifest = () => { if (project.exists(FILES.plan)) tryManifest(project, loadPlan(project)); };
|
|
790
|
+
// Already done: nothing is sent and nothing is charged.
|
|
791
|
+
if (!opts.redo && !opts.resume && entry0 && (entry0.noSubject || (entry0.subject && project.exists(entry0.subject)))) {
|
|
792
|
+
return { ok: true, data: { key, ...entry0, skipped: true }, summary: entry0.noSubject ? `The picture ${key} has no clear subject, so it stays flat; the words go in its ${entry0.textZone.region}. Use --redo to try again.` : `The picture ${key} already has its subject at ${entry0.subject}. Use --redo to make it again.` };
|
|
793
|
+
}
|
|
794
|
+
const zoneEntry = (await recordZone(project, key, opts.redo)) ?? { textZone: { region: "bottom" as const, luminance: 0.3, busy: 0.5 } };
|
|
795
|
+
const flat = (why: string): Result => ({ ok: true, data: { key, ...zoneEntry, noSubject: true }, summary: `${why} The picture ${key} stays one flat image; the words go in its ${zoneEntry.textZone.region} (${zoneEntry.textZone.luminance > 0.55 ? "light" : "dark"} there: use dark or light ink to match, or TextOnImage does).` });
|
|
796
|
+
const sceneNote = ` In the composition: <ImageLayers layers={s.imageLayers}><TextOnImage layers={s.imageLayers}>...</TextOnImage></ImageLayers>.`;
|
|
797
|
+
|
|
798
|
+
let id = opts.resume ?? (opts.redo ? undefined : entry0?.cutoutId);
|
|
799
|
+
const work = tempDir();
|
|
800
|
+
try {
|
|
801
|
+
if (!id) {
|
|
802
|
+
const video = `${work.dir}/still.mp4`;
|
|
803
|
+
try { await stillToVideo(project.path(key), video); } catch (e) { return flat(`The picture could not be prepared for a cut-out: ${e instanceof Error ? e.message : String(e)}`); }
|
|
804
|
+
try {
|
|
805
|
+
id = await startCutout(ctx, { source: video, filename: `${basename(key).replace(/\.[^.]+$/, "")}.mp4`.replace(/[^A-Za-z0-9._-]/g, "_"), contentType: "video/mp4", bytes: statSync(video).size, durationSec: 1 });
|
|
806
|
+
} catch (e) {
|
|
807
|
+
return { ok: false, data: { key, ...zoneEntry }, summary: `The picture ${key} stays flat: the cut-out could not be started: ${e instanceof Error ? e.message : String(e)} Nothing was charged. Run \`${retry}\` to try again.` };
|
|
808
|
+
}
|
|
809
|
+
saveLayerEntry(project, key, { ...zoneEntry, cutoutId: id });
|
|
810
|
+
} else ctx.log(`Waiting for cutout ${id} of ${key}.`);
|
|
811
|
+
const webm = `${work.dir}/subject.webm`;
|
|
812
|
+
let outcome: CutoutOutcome;
|
|
813
|
+
try { outcome = await pollCutout(ctx, id, webm, deps); } catch (e) {
|
|
814
|
+
if (!(e instanceof ApiFailure && e.code === "not_found")) throw e;
|
|
815
|
+
saveLayerEntry(project, key, { ...zoneEntry, cutoutId: undefined });
|
|
816
|
+
return { ok: false, data: { key }, summary: `The picture ${key} stays flat: the server no longer has cutout ${id}, so it was forgotten. Run \`${retry} --redo\` to start a new one.` };
|
|
817
|
+
}
|
|
818
|
+
if (outcome.kind !== "done") {
|
|
819
|
+
const resume = `${retry} --resume ${id}`;
|
|
820
|
+
if (outcome.kind === "failed") { saveLayerEntry(project, key, { ...zoneEntry, cutoutId: undefined }); return { ok: false, data: { key, status: "failed" }, summary: `The picture ${key} stays flat: the cut-out failed: ${outcome.message} You were not charged. Try \`${retry} --redo\` or a different picture. ${SENT_NOTE}` }; }
|
|
821
|
+
return { ok: false, data: { key, status: "pending", id }, summary: outcome.kind === "timeout" ? `The picture ${key} stays flat for now: the cut-out (id ${id}) is not ready after ${CUTOUT_TIMEOUT_MS / 60_000} minutes. It is not lost: run \`${resume}\`; that does not upload or charge again.` : `The picture ${key} stays flat for now: contact was lost while the cut-out (id ${id}) was being made: ${outcome.message} It is not lost: run \`${resume}\`; that does not charge again.` };
|
|
822
|
+
}
|
|
823
|
+
let cut: Awaited<ReturnType<typeof frameWithAlpha>>;
|
|
824
|
+
try { cut = await frameWithAlpha(webm, project.path(subjectKey)); } catch (e) {
|
|
825
|
+
rmSync(project.path(subjectKey), { force: true });
|
|
826
|
+
saveLayerEntry(project, key, { ...zoneEntry, cutoutId: undefined });
|
|
827
|
+
return flat(`The cut-out was made (about 1 second of your monthly cutout quota) but its subject layer could not be used: ${e instanceof Error ? e.message : String(e)}`);
|
|
828
|
+
}
|
|
829
|
+
const quota = "It used about 1 second of your monthly cutout quota.";
|
|
830
|
+
const verdict = coverageVerdict(cut.coverage);
|
|
831
|
+
if (!verdict.ok) {
|
|
832
|
+
rmSync(project.path(subjectKey), { force: true });
|
|
833
|
+
saveLayerEntry(project, key, { ...zoneEntry, noSubject: true, coverage: cut.coverage });
|
|
834
|
+
refreshManifest();
|
|
835
|
+
return { ok: true, data: { key, noSubject: true, coverage: cut.coverage, textZone: zoneEntry.textZone }, summary: `${verdict.note} ${quota} ${SENT_NOTE}` };
|
|
836
|
+
}
|
|
837
|
+
// The words' calm zone is measured again now that the subject is known, so that it is never offered behind it.
|
|
838
|
+
const textZone = await measureTextZone(project.path(key), cut.box).catch(() => zoneEntry.textZone);
|
|
839
|
+
const entry: LayerEntry = { subject: subjectKey, ...(cut.box ? { subjectBox: cut.box } : {}), textZone, coverage: Math.round(cut.coverage * 1000) / 1000 };
|
|
840
|
+
saveLayerEntry(project, key, entry);
|
|
841
|
+
refreshManifest();
|
|
842
|
+
return {
|
|
843
|
+
ok: true, data: { key, ...entry },
|
|
844
|
+
summary: `Layers${tag}: the subject is cut out into ${subjectKey} (it covers ${Math.round(cut.coverage * 100)}% of the picture) and the words have a calm ${textZone.region} (${textZone.luminance > 0.55 ? "light" : "dark"}). ${quota} ${SENT_NOTE}${opts.scene ? sceneNote : ` Use urls["${key}"] as the back and urls["${subjectKey}"] as the subject.`}`,
|
|
845
|
+
};
|
|
846
|
+
} finally { work.done(); }
|
|
847
|
+
}
|