reelkit-cli 0.5.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/package.json +7 -2
- package/skill/SKILL.md +22 -11
- package/skill/THIRD_PARTY.md +102 -0
- package/skill/commands/launch-film.md +7 -0
- package/skill/reference/asset-reuse.md +13 -2
- package/skill/reference/backgrounds.md +63 -0
- package/skill/reference/beat-sync.md +32 -17
- package/skill/reference/captions.md +11 -5
- package/skill/reference/component-authoring.md +12 -1
- package/skill/reference/continuity.md +21 -2
- package/skill/reference/kit.md +140 -11
- package/skill/reference/launch-film.md +190 -0
- package/skill/reference/remotion-composition.md +4 -3
- package/skill/reference/scene-treatments.md +20 -0
- package/skill/reference/scriptwriting.md +4 -1
- package/skill/reference/three-d.md +134 -0
- package/skill/reference/voice-sync.md +108 -0
- package/src/agents.ts +23 -12
- package/src/api/client.ts +4 -1
- package/src/cli.ts +25 -8
- package/src/commands/assets.ts +334 -36
- package/src/commands/build.ts +172 -36
- package/src/commands/components.ts +220 -0
- package/src/commands/init.ts +9 -3
- package/src/commands/install.ts +1 -1
- package/src/commands/plan.ts +8 -5
- package/src/commands/ref.ts +5 -2
- package/src/contract/index.ts +27 -5
- package/src/pipeline/beatsnap.ts +72 -0
- package/src/pipeline/review.ts +67 -7
- package/src/pipeline/schema.ts +51 -4
- package/src/pipeline/timing.ts +27 -1
- package/src/project/background.ts +33 -0
- package/src/project/layers.ts +60 -0
- package/src/project/loudness.ts +68 -0
- package/src/project/manifest.ts +68 -14
- package/src/project/music.ts +19 -5
- package/src/project/project.ts +7 -2
- package/src/project/refmeasure.ts +1 -1
- package/src/project/soundreport.ts +347 -0
- package/src/project/svgcheck.ts +21 -0
- package/src/remotion/kit/Assemble3D.tsx +92 -0
- package/src/remotion/kit/BrowserFrame.tsx +83 -0
- package/src/remotion/kit/Camera.tsx +6 -4
- package/src/remotion/kit/Captions.tsx +33 -17
- package/src/remotion/kit/Card3D.tsx +211 -0
- package/src/remotion/kit/ChapterFrame.tsx +68 -0
- package/src/remotion/kit/CounterRoll.tsx +75 -0
- package/src/remotion/kit/GlassPanel.tsx +43 -0
- package/src/remotion/kit/Grounds.tsx +177 -0
- package/src/remotion/kit/Headline.tsx +97 -0
- package/src/remotion/kit/Hero3D.tsx +197 -0
- package/src/remotion/kit/HudOverlay.tsx +52 -0
- package/src/remotion/kit/ImageLayers.tsx +48 -0
- package/src/remotion/kit/Music.tsx +4 -4
- package/src/remotion/kit/NamedCursor.tsx +54 -0
- package/src/remotion/kit/Orbit3D.tsx +49 -0
- package/src/remotion/kit/Particles3D.tsx +74 -0
- package/src/remotion/kit/Place.tsx +12 -0
- package/src/remotion/kit/PromptBox.tsx +84 -0
- package/src/remotion/kit/Scene3D.tsx +70 -0
- package/src/remotion/kit/SceneFrame.tsx +88 -11
- package/src/remotion/kit/Sfx.tsx +12 -6
- package/src/remotion/kit/SoundCues.tsx +22 -0
- package/src/remotion/kit/TerminalLog.tsx +98 -0
- package/src/remotion/kit/Text3D.tsx +78 -0
- package/src/remotion/kit/TextOnImage.tsx +41 -0
- package/src/remotion/kit/Warp3D.tsx +59 -0
- package/src/remotion/kit/bg-math.ts +179 -0
- package/src/remotion/kit/caption-groups.ts +7 -3
- package/src/remotion/kit/caption-style.ts +45 -0
- package/src/remotion/kit/docs.ts +133 -11
- package/src/remotion/kit/image-layers-math.ts +115 -0
- package/src/remotion/kit/index.ts +43 -1
- package/src/remotion/kit/inter-bold-typeface.ts +3 -0
- package/src/remotion/kit/media.ts +5 -3
- package/src/remotion/kit/motion-math.ts +36 -2
- package/src/remotion/kit/music-math.ts +27 -10
- package/src/remotion/kit/quiet-three.ts +11 -0
- package/src/remotion/kit/sample-text.ts +55 -0
- package/src/remotion/kit/scene3d-context.ts +5 -0
- package/src/remotion/kit/seeded.ts +13 -0
- package/src/remotion/kit/sound-cues.ts +89 -0
- package/src/remotion/kit/sound-kinds.ts +122 -0
- package/src/remotion/kit/theme.ts +2 -0
- package/src/remotion/kit/three-fx-math.ts +192 -0
- package/src/remotion/kit/three-math.ts +145 -0
- package/src/remotion/kit/transition-math.ts +116 -0
- package/src/remotion/kit/ui-math.ts +145 -0
- package/src/remotion/kit/ui-theme.ts +25 -0
- package/src/remotion/kit/word-anchor.ts +107 -0
- package/src/render/contact-sheet.ts +39 -0
- package/src/render/continuity.ts +14 -4
- package/src/render/deps.ts +15 -3
- package/src/render/master.ts +31 -0
- package/src/render/render.ts +15 -8
- package/src/render/sound-notes.ts +106 -0
- package/src/render/static-check.ts +156 -0
- package/src/render/validate.ts +3 -150
- package/src/render/word-check.ts +181 -0
- package/src/testing/conformance.ts +61 -1
- package/src/testing/fake-api.ts +11 -5
- package/src/testing/fixtures.ts +3 -0
package/src/commands/assets.ts
CHANGED
|
@@ -1,19 +1,26 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
3
|
-
import { basename, dirname, resolve } from "node:path";
|
|
2
|
+
import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { basename, dirname, relative, resolve } from "node:path";
|
|
4
4
|
import { ApiFailure, download, uploadTo } from "../api/client";
|
|
5
5
|
import { client, type Ctx, type Result } from "../context";
|
|
6
6
|
import { LibraryKindSchema, MAX_CUTOUT_SECONDS, MAX_UPLOAD_BYTES, UploadKindSchema, type CutoutType } from "../contract";
|
|
7
|
-
import { PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
|
|
7
|
+
import { gapSec, isVoiceless, LAST_TAIL_SEC, PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
|
|
8
8
|
import { GreenTooDullError, keyGreen } from "../project/chromakey";
|
|
9
|
-
import { buildManifest, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
|
|
9
|
+
import { buildManifest, sentenceGaps, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
|
|
10
10
|
import { loadPlan } from "./plan";
|
|
11
11
|
import { FILE_NAME } from "../render/validate";
|
|
12
|
-
import {
|
|
12
|
+
import { levelOf } from "../project/loudness";
|
|
13
|
+
import { beatReport, measureMusic, type MusicRecord } from "../project/music";
|
|
14
|
+
import { audioDuration } from "../project/refmeasure";
|
|
15
|
+
import { BACKGROUND_CLIP_SENTENCE, BACKGROUND_IMAGE_SENTENCE, isVideoPath, lookFromNotes, readBackground, setBackground } from "../project/background";
|
|
13
16
|
import { probeFile } from "../project/probe";
|
|
14
|
-
import {
|
|
17
|
+
import { frameWithAlpha, measureTextZone, stillToVideo, tempDir, type LayerEntry } from "../project/layers";
|
|
18
|
+
import { coverageVerdict } from "../remotion/kit/image-layers-math";
|
|
19
|
+
import { svgProblem } from "../project/svgcheck";
|
|
20
|
+
import { FILES, isPrivateProject, openProject, type Project } from "../project/project";
|
|
15
21
|
|
|
16
|
-
|
|
22
|
+
// gainDb is the levelling of a sound (sfx or music), in dB, measured when it was pulled: see src/project/loudness.ts.
|
|
23
|
+
export type LibraryEntry = { path: string; kind: string; title: string; meta: Record<string, unknown>; gainDb?: number };
|
|
17
24
|
|
|
18
25
|
// A failure to lay out the timeline after a paid call must not hide that the call succeeded and was saved.
|
|
19
26
|
function tryManifest(project: Project, plan: ScenePlan): { manifest?: AssetManifest; note?: string } {
|
|
@@ -47,7 +54,7 @@ const shellArg = (s: string) => (/^[\w./~@%+=:,-]+$/.test(s) ? s : JSON.stringif
|
|
|
47
54
|
|
|
48
55
|
// Sends a video to the server and starts its cutout. The seconds are reserved when the job is run, so this is the point where it becomes
|
|
49
56
|
// paid for: nothing is charged by a failure before it, and the id it returns is the only handle on the job after it.
|
|
50
|
-
async function startCutout(ctx: Ctx, input: CutoutInput): Promise<string> {
|
|
57
|
+
export async function startCutout(ctx: Ctx, input: CutoutInput): Promise<string> {
|
|
51
58
|
const api = client(ctx);
|
|
52
59
|
ctx.log(`Sending ${input.filename} to the Reelkit server to cut the subject out.`);
|
|
53
60
|
const up = await api("cutoutStart", { filename: input.filename.slice(-200), contentType: input.contentType, bytes: input.bytes, durationSec: input.durationSec });
|
|
@@ -61,7 +68,7 @@ type CutoutOutcome = { kind: "done" } | { kind: "failed"; message: string } | {
|
|
|
61
68
|
|
|
62
69
|
// Asks every few seconds until the job ends (at most ten minutes), then saves the result at `dest`. A job that is not found or a login that
|
|
63
70
|
// is wrong has nothing to resume and is thrown; any other trouble may be a hiccup on a job that is paid for, so it is an outcome.
|
|
64
|
-
async function pollCutout(ctx: Ctx, id: string, dest: string, deps: CutoutDeps): Promise<CutoutOutcome> {
|
|
71
|
+
export async function pollCutout(ctx: Ctx, id: string, dest: string, deps: CutoutDeps): Promise<CutoutOutcome> {
|
|
65
72
|
const api = client(ctx);
|
|
66
73
|
const now = deps.now ?? Date.now;
|
|
67
74
|
const started = now();
|
|
@@ -99,14 +106,14 @@ function cutoutRefusal(file: string, probe: Awaited<ReturnType<typeof probeFile>
|
|
|
99
106
|
|
|
100
107
|
// Registers one of the user's own files. It stays on this machine unless --share is given.
|
|
101
108
|
export async function assetsUpload(
|
|
102
|
-
ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; green?: boolean; cutout?: boolean; resume?: string; share?: boolean; kind?: string; tags?: string },
|
|
109
|
+
ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; green?: boolean; cutout?: boolean; resume?: string; share?: boolean; kind?: string; tags?: string; background?: boolean },
|
|
103
110
|
deps: { key?: typeof keyGreen } & CutoutDeps = {},
|
|
104
111
|
): Promise<Result> {
|
|
105
112
|
const project = openProject(ctx.cwd);
|
|
106
113
|
if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green keys a green background on this machine for free, --cutout removes any background on the server." };
|
|
107
114
|
if (opts.resume && !opts.cutout) return { ok: false, summary: "--resume goes with --cutout: it keeps waiting for a cutout that was already started." };
|
|
108
115
|
if (opts.resume) return resumeUploadCutout(ctx, project, file, opts.resume, deps);
|
|
109
|
-
if (opts.share && opts.kind === "component") return { ok: false, summary: "Components
|
|
116
|
+
if (opts.share && opts.kind === "component") return { ok: false, summary: "Components are shared with `reelkit components share`, not with assets upload. Share images, overlays, sound effects, music or clips here." };
|
|
110
117
|
const source = resolve(ctx.cwd, file);
|
|
111
118
|
let probe: Awaited<ReturnType<typeof probeFile>>;
|
|
112
119
|
try {
|
|
@@ -114,6 +121,8 @@ export async function assetsUpload(
|
|
|
114
121
|
} catch (e) {
|
|
115
122
|
return { ok: false, summary: e instanceof Error ? e.message : String(e) };
|
|
116
123
|
}
|
|
124
|
+
if (opts.background && probe.kind === "audio") return { ok: false, summary: `${file} is audio, so it cannot be the film's background. Use a video or an image.` };
|
|
125
|
+
if (opts.background && (opts.footage || opts.green || opts.cutout)) return { ok: false, summary: "--background makes the file the ground behind everything; it does not go with --footage, --green or --cutout." };
|
|
117
126
|
if (opts.footage && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so it cannot be the footage.` };
|
|
118
127
|
if (opts.green && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so there is no green screen to key out.` };
|
|
119
128
|
const refusal = opts.cutout ? cutoutRefusal(file, probe) : undefined;
|
|
@@ -141,7 +150,12 @@ export async function assetsUpload(
|
|
|
141
150
|
}
|
|
142
151
|
const record: AssetRecord = { id, filename, key, ...probe, ...(opts.describe ? { description: opts.describe } : {}), ...(keyedKey ? { keyedKey } : {}), ...(cutoutId ? { cutoutId } : {}), createdAt: new Date().toISOString() };
|
|
143
152
|
project.writeJson(FILES.assetIndex, [...project.assets(), record]);
|
|
153
|
+
if (probe.kind === "image" && !/\.svg$/i.test(filename)) await recordZone(project, key, true);
|
|
144
154
|
if (opts.footage) project.writeJson(FILES.config, { ...project.config(), footage: id });
|
|
155
|
+
if (opts.background) {
|
|
156
|
+
setBackground(project, { key, id, title: filename, ...(probe.kind === "video" && probe.durationSec ? { durationSec: probe.durationSec } : {}) });
|
|
157
|
+
if (project.exists(FILES.plan)) tryManifest(project, loadPlan(project));
|
|
158
|
+
}
|
|
145
159
|
|
|
146
160
|
let libraryId: string | undefined;
|
|
147
161
|
if (opts.share && kind?.success) {
|
|
@@ -161,7 +175,7 @@ export async function assetsUpload(
|
|
|
161
175
|
}
|
|
162
176
|
return {
|
|
163
177
|
ok: true, data: { ...record, ...(libraryId ? { libraryId } : {}) },
|
|
164
|
-
summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.${keyedKey ? ` The green is keyed out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"].` : ""}`,
|
|
178
|
+
summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${opts.background ? `, set as the film's background: use manifest.background (urls["${key}"])` : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.${keyedKey ? ` The green is keyed out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"].` : ""}`,
|
|
165
179
|
};
|
|
166
180
|
}
|
|
167
181
|
|
|
@@ -201,14 +215,36 @@ async function resumeUploadCutout(ctx: Ctx, project: Project, file: string, id:
|
|
|
201
215
|
return { ...finished, summary: `Resumed ${record.filename} (${record.id}). ${finished.summary}` };
|
|
202
216
|
}
|
|
203
217
|
|
|
218
|
+
export type SearchRecord = { q: string; kind: string | null; at: string; top: { id: string; match: number }[] };
|
|
219
|
+
const MAX_SEARCHES = 50;
|
|
220
|
+
|
|
221
|
+
// What was searched for in this project, so `reelkit check` can say whether the library was looked at. A search outside a project is not recorded.
|
|
222
|
+
function recordSearch(ctx: Ctx, q: string, kind: string | null, items: { id: string; match: number }[]) {
|
|
223
|
+
try {
|
|
224
|
+
const project = openProject(ctx.cwd);
|
|
225
|
+
const all = project.readJsonOr<SearchRecord[]>(FILES.searches, []);
|
|
226
|
+
project.writeJson(FILES.searches, [...all, { q, kind, at: new Date().toISOString(), top: items.slice(0, 3).map((i) => ({ id: i.id, match: i.match })) }].slice(-MAX_SEARCHES));
|
|
227
|
+
} catch { /* not a project, or the file is unreadable: the search itself still works */ }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// The length and, for music, the tempo of a result, at the end of its line, when the library has them. Sounds are only useful once their length is known:
|
|
231
|
+
// a riser must be as long as the build it scores, and a track must have the tempo the cuts will follow.
|
|
232
|
+
export function lengthOf(item: { kind: string; meta: Record<string, unknown> }): string {
|
|
233
|
+
if (item.kind !== "music" && item.kind !== "sfx") return "";
|
|
234
|
+
const d = item.meta.durationSec, bpm = item.meta.bpm;
|
|
235
|
+
const parts = [typeof d === "number" && d > 0 ? `${Math.round(d * 10) / 10}s` : "", item.kind === "music" && typeof bpm === "number" && bpm > 0 ? `${Math.round(bpm)} BPM` : ""].filter(Boolean);
|
|
236
|
+
return parts.length ? ` (${parts.join(", ")})` : "";
|
|
237
|
+
}
|
|
238
|
+
|
|
204
239
|
export async function assetsSearch(ctx: Ctx, query: string, opts: { kind?: string; limit?: string }): Promise<Result> {
|
|
205
240
|
const kind = opts.kind === undefined ? undefined : LibraryKindSchema.safeParse(opts.kind);
|
|
206
241
|
if (kind && !kind.success) return { ok: false, summary: `Unknown kind "${opts.kind}". Use one of: ${LibraryKindSchema.options.join(", ")}.` };
|
|
207
242
|
const { items } = await client(ctx)("librarySearch", { q: query, ...(kind?.success ? { kind: kind.data } : {}), ...(opts.limit ? { limit: Number(opts.limit) } : {}) });
|
|
243
|
+
recordSearch(ctx, query, kind?.success ? kind.data : null, items);
|
|
208
244
|
return {
|
|
209
245
|
ok: true, data: { items },
|
|
210
246
|
summary: items.length
|
|
211
|
-
? items.map((i) => `${`${Math.round(i.match * 100)}%`.padStart(4)} ${i.id} ${i.kind} ${i.title}: ${i.description} [${i.tags.join(", ")}]`).join("\n")
|
|
247
|
+
? items.map((i) => `${`${Math.round(i.match * 100)}%`.padStart(4)} ${i.id} ${i.kind} ${i.title}: ${i.description} [${i.tags.join(", ")}]${lengthOf(i)}`).join("\n")
|
|
212
248
|
: "No matches in the library. Generate it instead.",
|
|
213
249
|
};
|
|
214
250
|
}
|
|
@@ -229,10 +265,13 @@ function setSceneClip(project: Project, sceneId: string, record: ClipRecord) {
|
|
|
229
265
|
project.writeJson(FILES.clips, { ...project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {}), [sceneId]: record });
|
|
230
266
|
}
|
|
231
267
|
|
|
232
|
-
export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean; music?: boolean }, deps: ClipDeps = {}): Promise<Result> {
|
|
268
|
+
export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean; music?: boolean; background?: boolean; layers?: boolean }, deps: ClipDeps = {}): Promise<Result> {
|
|
269
|
+
if (opts.layers && (!opts.scene || opts.background || opts.music)) return { ok: false, summary: "--layers goes with --scene: it splits the scene's image into layers right after it is pulled." };
|
|
233
270
|
const project = openProject(ctx.cwd);
|
|
234
271
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id)) return { ok: false, summary: `"${id}" is not a library id. Copy the id from \`reelkit assets search\`.` };
|
|
235
|
-
if (opts.
|
|
272
|
+
if (opts.background && opts.music) return { ok: false, summary: "--background makes the item the ground behind the picture and --music makes it the track. Use one." };
|
|
273
|
+
if (opts.background && opts.scene && !loadPlan(project).scenes.some((s) => s.id === opts.scene)) return { ok: false, summary: `Scene ${opts.scene} is not in plan.json.` };
|
|
274
|
+
if (opts.scene && !opts.background) {
|
|
236
275
|
// Checked before anything is downloaded.
|
|
237
276
|
const scenes = loadPlan(project).scenes;
|
|
238
277
|
const illustrations = scenes.filter((s) => s.treatment === "illustration").map((s) => s.id);
|
|
@@ -241,11 +280,13 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
|
|
|
241
280
|
return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}${clipScenes.length ? ` The clip scenes are: ${clipScenes.join(", ")}.` : ""}` };
|
|
242
281
|
}
|
|
243
282
|
}
|
|
244
|
-
if (opts.music && opts.scene) return { ok: false, summary: "--music makes the item the video's track, not a scene's picture. Use --music or --scene, not both." };
|
|
283
|
+
if (opts.music && opts.scene && !opts.background) return { ok: false, summary: "--music makes the item the video's track, not a scene's picture. Use --music or --scene, not both." };
|
|
245
284
|
const pulled = await client(ctx)("libraryPull", { id });
|
|
246
285
|
if (opts.music && pulled.item.kind !== "music") return { ok: false, summary: `${id} is ${pulled.item.kind}, not music, so it cannot be the video's track. Search with --kind music.` };
|
|
247
286
|
if (unsafeName(pulled.filename, pulled.item.kind)) return { ok: false, summary: `The library returned an unsafe file name for ${id}, so nothing was written. Report this item.` };
|
|
248
|
-
if (opts.
|
|
287
|
+
if (opts.background && !["image", "clip", "overlay"].includes(pulled.item.kind)) return { ok: false, summary: `${id} is ${pulled.item.kind}, so it cannot be a background. Search with --kind clip, overlay or image.` };
|
|
288
|
+
if (opts.background && opts.scene && isVideoPath(pulled.filename)) return { ok: false, summary: `${id} is a video. A video is the ground of the whole film, so use --background without --scene; a scene's ground must be a picture.` };
|
|
289
|
+
if (opts.scene && !opts.background) {
|
|
249
290
|
const treatment = loadPlan(project).scenes.find((x) => x.id === opts.scene)?.treatment;
|
|
250
291
|
if (treatment === "clip" && pulled.item.kind !== "clip") return { ok: false, summary: `${id} is ${pulled.item.kind}, not a clip, so it cannot be scene ${opts.scene}'s clip.` };
|
|
251
292
|
if (treatment !== "clip" && pulled.item.kind !== "image") return { ok: false, summary: `${id} is ${pulled.item.kind}, not an image, so it cannot be a scene's image.` };
|
|
@@ -262,43 +303,59 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
|
|
|
262
303
|
}
|
|
263
304
|
const path = `assets/lib/${id}/${pulled.filename}`;
|
|
264
305
|
await download(pulled.url, project.path(path));
|
|
265
|
-
|
|
306
|
+
// A sound is levelled when it is pulled, so that the same volume number sounds equally loud for every file. A file that cannot be
|
|
307
|
+
// measured is used as it is.
|
|
308
|
+
const level = pulled.item.kind === "sfx" || pulled.item.kind === "music" ? await levelOf(project.path(path)) : undefined;
|
|
309
|
+
recordPull(project, id, { path, kind: pulled.item.kind, title: pulled.item.title, meta: pulled.item.meta, ...(level ? { gainDb: level.gainDb } : {}) });
|
|
310
|
+
if (opts.background) {
|
|
311
|
+
const video = isVideoPath(path);
|
|
312
|
+
let durationSec = typeof pulled.item.meta.durationSec === "number" ? pulled.item.meta.durationSec : undefined;
|
|
313
|
+
if (video && durationSec === undefined) { try { durationSec = Math.round((await audioDuration(project.path(path))) * 100) / 100; } catch { durationSec = undefined; } }
|
|
314
|
+
const previous = readBackground(project)[opts.scene ? "scenes" : "film"];
|
|
315
|
+
const had = opts.scene ? (previous as Record<string, { id?: string }>)[opts.scene] : (previous as { id?: string } | undefined);
|
|
316
|
+
setBackground(project, { key: path, id, title: pulled.item.title, ...(video && durationSec ? { durationSec } : {}) }, opts.scene);
|
|
317
|
+
const note = project.exists(FILES.plan) ? tryManifest(project, loadPlan(project)).note : undefined;
|
|
318
|
+
const where = opts.scene ? `scene ${opts.scene}'s background` : "the film's background";
|
|
319
|
+
return { ok: true, data: { path, kind: pulled.item.kind, background: true, ...(opts.scene ? { scene: opts.scene } : {}) }, summary: [`Pulled ${id} as ${where}${had?.id && had.id !== id ? ` (it replaces ${had.id})` : ""}. In the composition use ${opts.scene ? "the scene's `background.key`" : "`manifest.background`"}: <${video ? "BgVideo" : "BgImage"} src={urls["${path}"]} />. Read reference/backgrounds.md.`, ...(note ? [note] : [])].join("\n") };
|
|
320
|
+
}
|
|
266
321
|
// A clip filmed on green also comes keyed, so it can be laid over a background.
|
|
267
322
|
const green = pulled.item.kind === "clip" && pulled.item.meta.greenScreen === true;
|
|
268
323
|
const keyedPath = green ? keyedPathOf(path) : undefined;
|
|
269
324
|
if (keyedPath) await (deps.key ?? keyGreen)(project.path(path), project.path(keyedPath));
|
|
270
325
|
const sceneIsClip = pulled.item.kind === "clip";
|
|
271
326
|
let note: string | undefined;
|
|
327
|
+
let layered: Result | undefined;
|
|
272
328
|
if (opts.scene) {
|
|
273
329
|
if (sceneIsClip) {
|
|
274
330
|
const durationSec = typeof pulled.item.meta.durationSec === "number" ? pulled.item.meta.durationSec : 5;
|
|
275
331
|
setSceneClip(project, opts.scene, { key: path, greenScreen: green, durationSec, ...(keyedPath ? { keyedKey: keyedPath } : {}) });
|
|
276
|
-
} else setSceneImage(project, opts.scene, path);
|
|
332
|
+
} else { setSceneImage(project, opts.scene, path); await recordZone(project, path, true); if (opts.layers) layered = await assetsLayers(ctx, undefined, { scene: opts.scene, redo: true }, deps); }
|
|
277
333
|
if (project.exists(FILES.plan)) note = tryManifest(project, loadPlan(project)).note;
|
|
278
334
|
}
|
|
279
|
-
if (pulled.item.kind === "music" && opts.music) return pullMusic(project, id, pulled.item.title, path);
|
|
335
|
+
if (pulled.item.kind === "music" && opts.music) return pullMusic(project, id, pulled.item.title, path, level?.gainDb ?? 0);
|
|
280
336
|
const musicHint = pulled.item.kind === "music" ? ` To make it the video's track, with scene changes on its beat, run \`reelkit assets pull ${id} --music\`.` : "";
|
|
281
337
|
const refs = keyedPath ? ` The original is urls["${path}"]; the keyed copy with a transparent background is urls["${keyedPath}"].` : ` Reference it in the composition as urls["${path}"].`;
|
|
282
338
|
return {
|
|
283
|
-
ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}) },
|
|
339
|
+
ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}), ...(layered ? { layers: layered.data } : {}) },
|
|
284
340
|
summary: [
|
|
285
341
|
opts.scene ? `Pulled ${id} as the ${sceneIsClip ? "clip" : "image"} for scene ${opts.scene}.${keyedPath ? refs : ""}` : `Pulled ${id} to ${path}.${refs}${musicHint}`,
|
|
342
|
+
...(layered ? [layered.summary] : []),
|
|
286
343
|
...(note ? [note] : []),
|
|
287
344
|
].join("\n"),
|
|
288
345
|
};
|
|
289
346
|
}
|
|
290
347
|
|
|
291
348
|
// Makes a pulled track the video's one music track: measured here, saved in assets/music.json, and the timeline is laid out again around it.
|
|
292
|
-
async function pullMusic(project: Project, id: string, title: string, path: string): Promise<Result> {
|
|
349
|
+
async function pullMusic(project: Project, id: string, title: string, path: string, gainDb: number): Promise<Result> {
|
|
293
350
|
let measured: Awaited<ReturnType<typeof measureMusic>>;
|
|
294
351
|
try { measured = await measureMusic(project.path(path)); }
|
|
295
352
|
catch (e) { return { ok: false, summary: `${id} was downloaded to ${path} but could not be read as audio (${e instanceof Error ? e.message : String(e)}). Pull a different track.` }; }
|
|
296
353
|
const previous = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
|
|
297
|
-
const record: MusicRecord = { key: path, id, title, ...measured, gainDb
|
|
354
|
+
const record: MusicRecord = { key: path, id, title, ...measured, gainDb };
|
|
298
355
|
project.writeJson(FILES.music, record);
|
|
299
356
|
const note = project.exists(FILES.plan) ? tryManifest(project, loadPlan(project)).note : undefined;
|
|
300
357
|
const beat = record.bpm
|
|
301
|
-
? `about ${Math.round(record.bpm)} BPM; scene changes will land on its beat.`
|
|
358
|
+
? `about ${Math.round(record.bpm)} BPM; scene changes will land on its beat${project.exists(FILES.plan) && !isVoiceless(loadPlan(project)) ? " where that is within 4 frames of where the voice puts them (the others follow the voice)" : ""}.`
|
|
302
359
|
: "no clear tempo was found, so scene changes stay where the narration puts them.";
|
|
303
360
|
return {
|
|
304
361
|
ok: true, data: { path, kind: "music", music: record },
|
|
@@ -306,11 +363,18 @@ async function pullMusic(project: Project, id: string, title: string, path: stri
|
|
|
306
363
|
};
|
|
307
364
|
}
|
|
308
365
|
|
|
366
|
+
// Female voices first, then neutral and unmarked, then male, each group keeping the server's order: the first one listed is the default offer.
|
|
367
|
+
export function orderVoices<T extends { gender?: string }>(voices: readonly T[]): T[] {
|
|
368
|
+
const rank = (v: T) => (/^f/i.test(v.gender ?? "") ? 0 : /^m/i.test(v.gender ?? "") ? 2 : 1);
|
|
369
|
+
return voices.map((v, i) => ({ v, i })).sort((a, b) => rank(a.v) - rank(b.v) || a.i - b.i).map((x) => x.v);
|
|
370
|
+
}
|
|
371
|
+
|
|
309
372
|
export async function assetsVoices(ctx: Ctx): Promise<Result> {
|
|
310
|
-
const
|
|
373
|
+
const listed = await client(ctx)("voices", {});
|
|
374
|
+
const voices = orderVoices(listed.voices);
|
|
311
375
|
return {
|
|
312
376
|
ok: true, data: { voices },
|
|
313
|
-
summary: voices.map((v) => `${v.id} ${v.name} (${[v.gender, v.accent, v.language].filter(Boolean).join(", ")}): ${v.description}`).join("\n"),
|
|
377
|
+
summary: [...voices.map((v) => `${v.id} ${v.name} (${[v.gender, v.accent, v.language].filter(Boolean).join(", ")}): ${v.description}`), "The plan needs a voiceId from this list (`reelkit plan check` asks for one); the first voice listed is the default offer to the user."].join("\n"),
|
|
314
378
|
};
|
|
315
379
|
}
|
|
316
380
|
|
|
@@ -318,6 +382,8 @@ export async function assetsVoices(ctx: Ctx): Promise<Result> {
|
|
|
318
382
|
export async function assetsVoiceover(ctx: Ctx, opts: { scene?: string; all?: boolean; redo?: boolean }): Promise<Result> {
|
|
319
383
|
const project = openProject(ctx.cwd);
|
|
320
384
|
const plan = loadPlan(project);
|
|
385
|
+
// Said as success so that an agent following the older steps carries on instead of stopping.
|
|
386
|
+
if (isVoiceless(plan)) return { ok: true, data: { recorded: [], voice: "none" }, summary: "This video has no voice (voice is \"none\" in plan.json), so there is nothing to record. Go on to the music and the composition." };
|
|
321
387
|
if (!opts.all && !opts.scene) return { ok: false, summary: "Say which scene with --scene <id>, or use --all." };
|
|
322
388
|
const wanted = opts.all ? plan.scenes : plan.scenes.filter((s) => s.id === opts.scene);
|
|
323
389
|
if (!wanted.length) return { ok: false, summary: `No scene with id ${opts.scene}. The scenes are: ${plan.scenes.map((s) => s.id).join(", ")}.` };
|
|
@@ -338,39 +404,115 @@ export async function assetsVoiceover(ctx: Ctx, opts: { scene?: string; all?: bo
|
|
|
338
404
|
}
|
|
339
405
|
const { manifest, note } = tryManifest(project, plan);
|
|
340
406
|
const left = plan.scenes.filter((s) => !done[s.id] || voiceoverStale(done[s.id], s, plan)).map((s) => s.id);
|
|
407
|
+
const gaps = manifest ? sentenceGaps(manifest) : [];
|
|
408
|
+
const beat = manifest ? beatReport(manifest) : undefined;
|
|
341
409
|
const recorded = `Recorded ${todo.length} scene(s)${rerecorded ? ` (${rerecorded} re-recorded because the script or voice changed)` : ""}.`;
|
|
342
410
|
return {
|
|
343
|
-
ok: true, data: { recorded: todo.map((s) => s.id), rerecorded, voiceovers: done, totalSeconds: manifest ? manifest.totalFrames / manifest.fps : null },
|
|
411
|
+
ok: true, data: { recorded: todo.map((s) => s.id), rerecorded, voiceovers: done, totalSeconds: manifest ? manifest.totalFrames / manifest.fps : null, ...(manifest ? { sentenceGaps: gaps, ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) } : {}) },
|
|
344
412
|
summary: [
|
|
345
413
|
manifest
|
|
346
|
-
? `${recorded} The video runs ${(manifest.totalFrames / manifest.fps).toFixed(1)}s. Voiceover files are in assets/.${left.length ? ` Still out of date: ${left.join(", ")}. Run \`reelkit assets voiceover --all\`.` : ""}`
|
|
414
|
+
? `${recorded} The video runs ${(manifest.totalFrames / manifest.fps).toFixed(1)}s (the last scene keeps ${LAST_TAIL_SEC}s after its last word). Voiceover files are in assets/.${left.length ? ` Still out of date: ${left.join(", ")}. Run \`reelkit assets voiceover --all\`.` : ""}\nSilence between sentences (plan gap "${plan.gap ?? "normal"}", ${gapSec(plan)}s): ${gaps.map((g) => `${g.toFixed(2)}s`).join(", ") || "none"}.${beat ? `\nBeat: ${beat.line}.` : ""}`
|
|
347
415
|
: `${recorded}${left.length ? ` Still to record: ${left.join(", ")}.` : ""}`,
|
|
348
416
|
...(note ? [note] : []),
|
|
349
417
|
].join("\n"),
|
|
350
418
|
};
|
|
351
419
|
}
|
|
352
420
|
|
|
353
|
-
|
|
421
|
+
// Said at the end of every image prompt that does not ask for text or a logo: a generated picture with letters in it is almost always wrong.
|
|
422
|
+
// Added to the prompt of a scene that has on-screen text, so that the picture has a clear single subject and calm room for the words.
|
|
423
|
+
export const TEXT_ROOM_SENTENCE = "One clear subject, with calm empty space on one side of the picture where words can sit.";
|
|
424
|
+
export const NO_TEXT_SENTENCE = "No text, letters, numbers, logos or brand marks anywhere in the picture.";
|
|
425
|
+
// What a prompt asks for when it wants writing or a logo in the picture. A clause that forbids them ("no text, no logos") does not count.
|
|
426
|
+
const WANTS_TEXT = /\b(text|texts|letters?|lettering|typography|words?|caption|captions|title|headline|slogan|quote|says|saying|reads|reading|written|writing|sign|signage|label|logo|logos|wordmark|brand ?mark|brand name)\b/i;
|
|
427
|
+
const FORBIDS = /\b(?:no|without|never|avoid|free of|not any|zero)\b[^.;\n]*/gi;
|
|
428
|
+
export const asksForText = (prompt: string): boolean => WANTS_TEXT.test(prompt.replace(FORBIDS, " "));
|
|
429
|
+
// The prompt as sent: with NO_TEXT_SENTENCE after it, unless the prompt itself asks for text or a logo.
|
|
430
|
+
export const withNoText = (prompt: string): string => (asksForText(prompt) ? prompt : `${prompt.trim()}${/[.!?]$/.test(prompt.trim()) ? "" : "."} ${NO_TEXT_SENTENCE}`);
|
|
431
|
+
|
|
432
|
+
export async function assetsGenImage(ctx: Ctx, prompt: string | undefined, opts: { scene?: string; redo?: boolean; background?: boolean; layers?: boolean }, deps: CutoutDeps = {}): Promise<Result> {
|
|
354
433
|
const project = openProject(ctx.cwd);
|
|
355
434
|
const plan = loadPlan(project);
|
|
435
|
+
if (opts.background) return genBackgroundImage(ctx, project, plan, prompt, { ...opts, format: "png" });
|
|
436
|
+
if (!opts.scene) return { ok: false, summary: "Say which scene the image is for with --scene <id>, or use --background for the film's ground." };
|
|
356
437
|
const scene = plan.scenes.find((s) => s.id === opts.scene);
|
|
357
438
|
if (!scene || scene.treatment !== "illustration" || !scene.imagePrompt) return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene, so it does not take an image.` };
|
|
358
439
|
// A scene that already has its image is not charged for again, whatever prompt is given; only --redo asks for a new one.
|
|
359
440
|
const have = project.readJsonOr<Record<string, string>>(FILES.images, {})[scene.id];
|
|
360
|
-
if (have && project.exists(have) && !opts.redo)
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
441
|
+
if (have && project.exists(have) && !opts.redo) {
|
|
442
|
+
const layered = opts.layers ? await assetsLayers(ctx, undefined, { scene: scene.id }, deps) : undefined;
|
|
443
|
+
return { ok: true, data: { path: have, skipped: true }, summary: `Scene ${scene.id} already has an image at ${have}. Use --redo to generate a new one.${layered ? ` ${layered.summary}` : ""}` };
|
|
444
|
+
}
|
|
445
|
+
// Anything tied to the user's footage or uploads stays private, whatever the plan says. So does a prompt changed by hand, and every image of a
|
|
446
|
+
// project made with `reelkit init --private`.
|
|
447
|
+
const priv = isPrivateProject(project);
|
|
448
|
+
const shareable = scene.shareable && !priv && !project.footage() && scene.userAssetIds.length === 0 && !prompt;
|
|
449
|
+
const base = prompt ?? scene.imagePrompt;
|
|
450
|
+
const img = await client(ctx)("images", { prompt: withNoText(scene.onScreenText.length && !prompt ? `${base.trim()}${/[.!?]$/.test(base.trim()) ? "" : "."} ${TEXT_ROOM_SENTENCE}` : base), aspect: plan.aspect, shareable, tags: scene.imageTags, format: "png" });
|
|
364
451
|
const path = `assets/img-${scene.id}.${img.ext}`;
|
|
365
452
|
await download(img.url, project.path(path));
|
|
366
453
|
setSceneImage(project, scene.id, path);
|
|
454
|
+
await recordZone(project, path, true);
|
|
455
|
+
const layered = opts.layers ? await assetsLayers(ctx, undefined, { scene: scene.id, redo: true }, deps) : undefined;
|
|
367
456
|
const { note } = tryManifest(project, plan);
|
|
368
457
|
return {
|
|
369
|
-
ok: true, data: { path, libraryId: img.libraryId ?? null },
|
|
370
|
-
summary: [`Generated the image for scene ${scene.id} at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}.`, ...(note ? [note] : [])].join("\n"),
|
|
458
|
+
ok: true, data: { path, libraryId: img.libraryId ?? null, ...(layered ? { layers: layered.data } : {}) },
|
|
459
|
+
summary: [`Generated the image for scene ${scene.id} at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}.`, ...(layered ? [layered.summary] : []), ...(note ? [note] : [])].join("\n"),
|
|
371
460
|
};
|
|
372
461
|
}
|
|
373
462
|
|
|
463
|
+
// Downloads a generated vector graphic into memory first, checks it, and only then writes it, so a file that fails the check never reaches the project.
|
|
464
|
+
async function saveSvg(url: string, dest: string): Promise<string | undefined> {
|
|
465
|
+
const res = await fetch(url);
|
|
466
|
+
if (!res.ok) return "The generated graphic could not be downloaded. Run the command again.";
|
|
467
|
+
const text = await res.text();
|
|
468
|
+
const problem = svgProblem(text);
|
|
469
|
+
if (problem) return problem;
|
|
470
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
471
|
+
writeFileSync(dest, text);
|
|
472
|
+
return undefined;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// A background made for the film (or for one scene, as a picture): a generic ambient image, never shared, saved and recorded as the ground.
|
|
476
|
+
async function genBackgroundImage(ctx: Ctx, project: Project, plan: ScenePlan, prompt: string | undefined, opts: { scene?: string; redo?: boolean; format: "png" | "svg" }): Promise<Result> {
|
|
477
|
+
if (opts.scene && !plan.scenes.some((s) => s.id === opts.scene)) return { ok: false, summary: `Scene ${opts.scene} is not in plan.json.` };
|
|
478
|
+
const have = readBackground(project)[opts.scene ? "scenes" : "film"];
|
|
479
|
+
const existing = opts.scene ? (have as Record<string, { key: string }>)[opts.scene] : (have as { key: string } | undefined);
|
|
480
|
+
if (existing && project.exists(existing.key) && !opts.redo) return { ok: true, data: { path: existing.key, skipped: true }, summary: `${opts.scene ? `Scene ${opts.scene}` : "The film"} already has a background at ${existing.key}. Use --redo to generate a new one.` };
|
|
481
|
+
const sent = `${prompt ?? `${lookFromNotes(plan.scenes[0]?.notes)}.`} ${BACKGROUND_IMAGE_SENTENCE}`;
|
|
482
|
+
const img = await client(ctx)("images", { prompt: sent, aspect: plan.aspect, shareable: false, tags: [], format: opts.format });
|
|
483
|
+
const path = `assets/bg-${opts.scene ?? "film"}.${img.ext}`;
|
|
484
|
+
if (img.ext === "svg") { const bad = await saveSvg(img.url, project.path(path)); if (bad) return { ok: false, summary: bad }; }
|
|
485
|
+
else await download(img.url, project.path(path));
|
|
486
|
+
setBackground(project, { key: path, title: "generated background" }, opts.scene);
|
|
487
|
+
const { note } = tryManifest(project, plan);
|
|
488
|
+
return { ok: true, data: { path, prompt: sent }, summary: [`Generated ${opts.scene ? `scene ${opts.scene}'s` : "the film's"} background at ${path}. The prompt sent was: ${sent}`, ...(note ? [note] : [])].join("\n") };
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// A vector graphic: icons, logo-like marks, simple illustrations, diagrams. It is one image against the image quota and takes about 45 seconds.
|
|
492
|
+
export async function assetsGenSvg(ctx: Ctx, prompt: string, opts: { scene?: string; redo?: boolean; private?: boolean; background?: boolean }): Promise<Result> {
|
|
493
|
+
const project = openProject(ctx.cwd);
|
|
494
|
+
const plan = loadPlan(project);
|
|
495
|
+
if (!prompt.trim()) return { ok: false, summary: "Say what the graphic shows: `reelkit assets gen svg \"a red rocket icon, flat, transparent background\"`." };
|
|
496
|
+
if (opts.background) return genBackgroundImage(ctx, project, plan, prompt, { ...opts, format: "svg" });
|
|
497
|
+
const scene = opts.scene ? plan.scenes.find((s) => s.id === opts.scene) : undefined;
|
|
498
|
+
if (opts.scene && (!scene || scene.treatment !== "illustration")) return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene, so it does not take a graphic. Leave out --scene to save the file and place it yourself.` };
|
|
499
|
+
const have = scene ? project.readJsonOr<Record<string, string>>(FILES.images, {})[scene.id] : undefined;
|
|
500
|
+
if (scene && have && project.exists(have) && !opts.redo) return { ok: true, data: { path: have, skipped: true }, summary: `Scene ${scene.id} already has an image at ${have}. Use --redo to generate a new one.` };
|
|
501
|
+
// Shared for review only when the plan says the scene is generic and nothing of the user's is in it; --private always keeps it here.
|
|
502
|
+
const shareable = Boolean(scene?.shareable) && !opts.private && !isPrivateProject(project) && !project.footage() && (scene?.userAssetIds.length ?? 1) === 0;
|
|
503
|
+
ctx.log("Drawing the graphic. It takes about 45 seconds.");
|
|
504
|
+
const img = await client(ctx)("images", { prompt, aspect: plan.aspect, shareable, tags: scene?.imageTags ?? [], format: "svg" });
|
|
505
|
+
if (img.ext !== "svg") return { ok: false, summary: "The server sent a picture, not a vector graphic. Nothing was saved; run the command again." };
|
|
506
|
+
const path = scene ? `assets/img-${scene.id}.svg` : `assets/svg-${randomUUID().slice(0, 8)}.svg`;
|
|
507
|
+
const bad = await saveSvg(img.url, project.path(path));
|
|
508
|
+
if (bad) return { ok: false, summary: bad };
|
|
509
|
+
if (scene) setSceneImage(project, scene.id, path);
|
|
510
|
+
const { note } = project.exists(FILES.plan) ? tryManifest(project, plan) : { note: undefined };
|
|
511
|
+
return {
|
|
512
|
+
ok: true, data: { path, libraryId: img.libraryId ?? null },
|
|
513
|
+
summary: [`Generated the graphic at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}. Use it as urls["${path}"] in an <Img>. It counted as one image.`, ...(note ? [note] : [])].join("\n"),
|
|
514
|
+
};
|
|
515
|
+
}
|
|
374
516
|
|
|
375
517
|
const CLIP_POLL_MS = 5_000;
|
|
376
518
|
const CLIP_PROGRESS_MS = 30_000;
|
|
@@ -380,17 +522,22 @@ const CLIP_TIMEOUT_MS = 10 * 60_000;
|
|
|
380
522
|
// The id is the only handle on a paid job, so every way out after the start says it and how to resume.
|
|
381
523
|
export async function assetsGenClip(
|
|
382
524
|
ctx: Ctx, prompt: string | undefined,
|
|
383
|
-
opts: { scene
|
|
525
|
+
opts: { scene?: string; green?: boolean; cutout?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string; background?: boolean },
|
|
384
526
|
deps: ClipDeps = {},
|
|
385
527
|
): Promise<Result> {
|
|
386
528
|
const project = openProject(ctx.cwd);
|
|
387
529
|
const plan = loadPlan(project);
|
|
530
|
+
if (opts.background) return genBackgroundClip(ctx, project, plan, prompt, opts, deps);
|
|
531
|
+
if (!opts.scene) return { ok: false, summary: "Say which scene the clip is for with --scene <id>, or use --background for a clip behind the whole film." };
|
|
388
532
|
const scene = plan.scenes.find((s) => s.id === opts.scene);
|
|
389
533
|
if (!scene || scene.treatment !== "clip" || !scene.clipPrompt) return { ok: false, summary: `Scene ${opts.scene} is not a clip scene, so it does not take a clip. Set its treatment to "clip" and give it a clipPrompt in plan.json.` };
|
|
390
534
|
if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green asks for a green background and keys it on this machine for free, --cutout removes any background on the server." };
|
|
391
535
|
const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
|
|
392
536
|
if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
|
|
393
537
|
if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
|
|
538
|
+
if (opts.share && isPrivateProject(project)) {
|
|
539
|
+
return { ok: false, summary: "This project was made with `reelkit init --private`, so nothing in it goes to the library and --share is refused. Run again without --share." };
|
|
540
|
+
}
|
|
394
541
|
if (opts.share && (!scene.shareable || scene.userAssetIds.length > 0 || project.footage())) {
|
|
395
542
|
return { ok: false, summary: `Scene ${scene.id} is private (it is not marked shareable in plan.json, or it shows your own files), so --share is refused. Run again without --share.` };
|
|
396
543
|
}
|
|
@@ -543,3 +690,154 @@ async function keyScene(project: Project, sceneId: string, record: ClipRecord, k
|
|
|
543
690
|
setSceneClip(project, sceneId, { ...record, greenScreen: true, keyedKey });
|
|
544
691
|
return { ok: true, summary: "" };
|
|
545
692
|
}
|
|
693
|
+
|
|
694
|
+
// A clip meant to sit behind everything: a generic ambient loop, never shared. It uses the same clip quota and job as any clip, and `--resume` works.
|
|
695
|
+
async function genBackgroundClip(ctx: Ctx, project: Project, plan: ScenePlan, prompt: string | undefined, opts: { scene?: string; green?: boolean; cutout?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string }, deps: ClipDeps): Promise<Result> {
|
|
696
|
+
if (opts.scene) return { ok: false, summary: "A background clip is for the whole film, so leave out --scene. A scene's own ground is a picture: `reelkit assets gen image --background --scene <id>`." };
|
|
697
|
+
if (opts.green || opts.cutout) return { ok: false, summary: "--green and --cutout are for a subject, not a background. Leave them out with --background." };
|
|
698
|
+
const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
|
|
699
|
+
if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
|
|
700
|
+
if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
|
|
701
|
+
const have = readBackground(project).film;
|
|
702
|
+
if (have && project.exists(have.key) && !opts.redo && !opts.resume) return { ok: true, data: { path: have.key, skipped: true }, summary: `The film already has a background at ${have.key}. Use --redo to generate a new one.` };
|
|
703
|
+
const now = deps.now ?? Date.now, api = client(ctx), started = now();
|
|
704
|
+
const sent = `${prompt ?? `${lookFromNotes(plan.scenes[0]?.notes)}.`} ${BACKGROUND_CLIP_SENTENCE}`;
|
|
705
|
+
let id = opts.resume;
|
|
706
|
+
if (!id) {
|
|
707
|
+
id = (await api("clipStart", { prompt: sent, aspect: plan.aspect, greenScreen: false, durationSec: seconds, shareable: false, tags: [] })).id;
|
|
708
|
+
ctx.log(`Started background clip ${id}. It usually takes a few minutes.`);
|
|
709
|
+
} else ctx.log(`Waiting for background clip ${id}.`);
|
|
710
|
+
const resume = `reelkit assets gen clip --background --resume ${id}`;
|
|
711
|
+
try {
|
|
712
|
+
let lastLog = started;
|
|
713
|
+
for (;;) {
|
|
714
|
+
const st = await api("clipStatus", { id });
|
|
715
|
+
if (st.status === "failed") return { ok: false, data: { id, status: "failed" }, summary: `The background clip failed: ${st.message ?? "the video provider gave no reason."} You were not charged. Change the prompt and run the command again.` };
|
|
716
|
+
if (st.status === "done") {
|
|
717
|
+
const path = `assets/bg-film.${st.ext}`;
|
|
718
|
+
await download(st.url!, project.path(path));
|
|
719
|
+
const durationSec = st.durationSec ?? seconds;
|
|
720
|
+
setBackground(project, { key: path, title: "generated background", durationSec });
|
|
721
|
+
const { note } = tryManifest(project, plan);
|
|
722
|
+
return { ok: true, data: { id, path, durationSec, prompt: sent }, summary: [`Generated the film's background clip at ${path} (${durationSec}s). The prompt sent was: ${sent} Use it with <BgVideo src={urls["${path}"]} durationSec={manifest.background?.durationSec} />.`, ...(note ? [note] : [])].join("\n") };
|
|
723
|
+
}
|
|
724
|
+
const elapsed = now() - started;
|
|
725
|
+
if (elapsed >= CLIP_TIMEOUT_MS) return { ok: false, data: { id, status: "pending" }, summary: `The background clip (id ${id}) is not ready after ${CLIP_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${resume}\` to keep waiting for it; that does not charge again.` };
|
|
726
|
+
if (now() - lastLog >= CLIP_PROGRESS_MS) { lastLog = now(); ctx.log(`Still making the background clip (${Math.round(elapsed / 1000)}s so far)...`); }
|
|
727
|
+
await ctx.sleep(CLIP_POLL_MS);
|
|
728
|
+
}
|
|
729
|
+
} catch (e) {
|
|
730
|
+
if (e instanceof ApiFailure && (e.code === "not_found" || e.code === "unauthenticated")) throw e;
|
|
731
|
+
return { ok: false, data: { id }, summary: `Lost contact while the background clip (id ${id}) was being made: ${e instanceof Error ? e.message : String(e)} The clip is not lost: run \`${resume}\`; that does not charge again.` };
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
// ---- Layers: a still picture in layers, with the words as a layer of it ----
|
|
737
|
+
|
|
738
|
+
// Measures where words can sit in a picture (free, on this machine) and records it beside the picture's layers. With `fresh` the record starts again, so that
|
|
739
|
+
// a new picture never keeps the subject of the one it replaces. A picture ffmpeg cannot read is recorded as nothing: the layers are an addition, never a failure.
|
|
740
|
+
async function recordZone(project: Project, key: string, fresh = false): Promise<LayerEntry | undefined> {
|
|
741
|
+
try {
|
|
742
|
+
const all = project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {});
|
|
743
|
+
const old = fresh ? undefined : all[key];
|
|
744
|
+
const textZone = await measureTextZone(project.path(key), old?.subjectBox);
|
|
745
|
+
const entry: LayerEntry = { ...(old ?? {}), textZone };
|
|
746
|
+
project.writeJson(FILES.layers, { ...all, [key]: entry });
|
|
747
|
+
return entry;
|
|
748
|
+
} catch { return undefined; }
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
function saveLayerEntry(project: Project, key: string, entry: LayerEntry) {
|
|
752
|
+
project.writeJson(FILES.layers, { ...project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {}), [key]: entry });
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
const subjectPathOf = (key: string) => key.replace(/\.[^./]+$/, "") + ".subject.png";
|
|
756
|
+
|
|
757
|
+
// Makes the picture a scene's image is made of into layers: the subject cut out of it (a PNG with alpha beside it, `<name>.subject.png`), its box, and where words can
|
|
758
|
+
// sit. It reuses the service that removes the background of a video: the still is sent as a one-second video, so it uses about 1 second of the cutout quota, and one
|
|
759
|
+
// frame of the transparent video that comes back is kept. A picture with no clear subject (the cut-out covers under 3% or over 85% of it) stays flat and the command
|
|
760
|
+
// still succeeds. `file` is a picture in the project; `scene` is a scene's image.
|
|
761
|
+
export async function assetsLayers(ctx: Ctx, file: string | undefined, opts: { scene?: string; redo?: boolean; resume?: string }, deps: CutoutDeps = {}): Promise<Result> {
|
|
762
|
+
const project = openProject(ctx.cwd);
|
|
763
|
+
if (!file && !opts.scene) return { ok: false, summary: "Say which picture: `reelkit assets layers --scene <id>` for a scene's image, or `reelkit assets layers <file>` for a picture in the project." };
|
|
764
|
+
if (file && opts.scene) return { ok: false, summary: "Give a file or --scene, not both." };
|
|
765
|
+
if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a cutout id. Copy it from the earlier message.` };
|
|
766
|
+
let key: string;
|
|
767
|
+
if (opts.scene) {
|
|
768
|
+
const have = project.readJsonOr<Record<string, string>>(FILES.images, {})[opts.scene];
|
|
769
|
+
if (!have || !project.exists(have)) return { ok: false, summary: `Scene ${opts.scene} has no image yet. Make one with \`reelkit assets gen image --scene ${opts.scene}\` or pull one with \`reelkit assets pull <id> --scene ${opts.scene}\`.` };
|
|
770
|
+
if (/\.(svg|mp4|webm|mov)$/i.test(have)) return { ok: false, summary: `Scene ${opts.scene}'s picture ${have} is not a photo or a raster picture, so it cannot be split into layers.` };
|
|
771
|
+
key = have;
|
|
772
|
+
} else {
|
|
773
|
+
const abs = resolve(ctx.cwd, file!);
|
|
774
|
+
const rel = relative(project.dir, abs);
|
|
775
|
+
if (rel.startsWith("..") || !project.exists(rel)) return { ok: false, summary: `${file} is not a file in this project. Add it with \`reelkit assets upload ${file}\` and give the path it is stored at.` };
|
|
776
|
+
let probe: Awaited<ReturnType<typeof probeFile>>;
|
|
777
|
+
try { probe = await probeFile(abs); } catch (e) { return { ok: false, summary: e instanceof Error ? e.message : String(e) }; }
|
|
778
|
+
if (probe.kind !== "image") return { ok: false, summary: `${file} is ${probe.kind}, not a picture, so it has no layers.` };
|
|
779
|
+
key = rel;
|
|
780
|
+
}
|
|
781
|
+
const tag = opts.scene ? ` for scene ${opts.scene}` : "";
|
|
782
|
+
const retry = `reelkit assets layers ${opts.scene ? `--scene ${opts.scene}` : shellArg(key)}`;
|
|
783
|
+
const entry0 = project.readJsonOr<Record<string, LayerEntry>>(FILES.layers, {})[key];
|
|
784
|
+
const subjectKey = subjectPathOf(key);
|
|
785
|
+
const refreshManifest = () => { if (project.exists(FILES.plan)) tryManifest(project, loadPlan(project)); };
|
|
786
|
+
// Already done: nothing is sent and nothing is charged.
|
|
787
|
+
if (!opts.redo && !opts.resume && entry0 && (entry0.noSubject || (entry0.subject && project.exists(entry0.subject)))) {
|
|
788
|
+
return { ok: true, data: { key, ...entry0, skipped: true }, summary: entry0.noSubject ? `The picture ${key} has no clear subject, so it stays flat; the words go in its ${entry0.textZone.region}. Use --redo to try again.` : `The picture ${key} already has its subject at ${entry0.subject}. Use --redo to make it again.` };
|
|
789
|
+
}
|
|
790
|
+
const zoneEntry = (await recordZone(project, key, opts.redo)) ?? { textZone: { region: "bottom" as const, luminance: 0.3, busy: 0.5 } };
|
|
791
|
+
const flat = (why: string): Result => ({ ok: true, data: { key, ...zoneEntry, noSubject: true }, summary: `${why} The picture ${key} stays one flat image; the words go in its ${zoneEntry.textZone.region} (${zoneEntry.textZone.luminance > 0.55 ? "light" : "dark"} there: use dark or light ink to match, or TextOnImage does).` });
|
|
792
|
+
const sceneNote = ` In the composition: <ImageLayers layers={s.imageLayers}><TextOnImage layers={s.imageLayers}>...</TextOnImage></ImageLayers>.`;
|
|
793
|
+
|
|
794
|
+
let id = opts.resume ?? (opts.redo ? undefined : entry0?.cutoutId);
|
|
795
|
+
const work = tempDir();
|
|
796
|
+
try {
|
|
797
|
+
if (!id) {
|
|
798
|
+
const video = `${work.dir}/still.mp4`;
|
|
799
|
+
try { await stillToVideo(project.path(key), video); } catch (e) { return flat(`The picture could not be prepared for a cut-out: ${e instanceof Error ? e.message : String(e)}`); }
|
|
800
|
+
try {
|
|
801
|
+
id = await startCutout(ctx, { source: video, filename: `${basename(key).replace(/\.[^.]+$/, "")}.mp4`.replace(/[^A-Za-z0-9._-]/g, "_"), contentType: "video/mp4", bytes: statSync(video).size, durationSec: 1 });
|
|
802
|
+
} catch (e) {
|
|
803
|
+
return { ok: false, data: { key, ...zoneEntry }, summary: `The picture ${key} stays flat: the cut-out could not be started: ${e instanceof Error ? e.message : String(e)} Nothing was charged. Run \`${retry}\` to try again.` };
|
|
804
|
+
}
|
|
805
|
+
saveLayerEntry(project, key, { ...zoneEntry, cutoutId: id });
|
|
806
|
+
} else ctx.log(`Waiting for cutout ${id} of ${key}.`);
|
|
807
|
+
const webm = `${work.dir}/subject.webm`;
|
|
808
|
+
let outcome: CutoutOutcome;
|
|
809
|
+
try { outcome = await pollCutout(ctx, id, webm, deps); } catch (e) {
|
|
810
|
+
if (!(e instanceof ApiFailure && e.code === "not_found")) throw e;
|
|
811
|
+
saveLayerEntry(project, key, { ...zoneEntry, cutoutId: undefined });
|
|
812
|
+
return { ok: false, data: { key }, summary: `The picture ${key} stays flat: the server no longer has cutout ${id}, so it was forgotten. Run \`${retry} --redo\` to start a new one.` };
|
|
813
|
+
}
|
|
814
|
+
if (outcome.kind !== "done") {
|
|
815
|
+
const resume = `${retry} --resume ${id}`;
|
|
816
|
+
if (outcome.kind === "failed") { saveLayerEntry(project, key, { ...zoneEntry, cutoutId: undefined }); return { ok: false, data: { key, status: "failed" }, summary: `The picture ${key} stays flat: the cut-out failed: ${outcome.message} You were not charged. Try \`${retry} --redo\` or a different picture. ${SENT_NOTE}` }; }
|
|
817
|
+
return { ok: false, data: { key, status: "pending", id }, summary: outcome.kind === "timeout" ? `The picture ${key} stays flat for now: the cut-out (id ${id}) is not ready after ${CUTOUT_TIMEOUT_MS / 60_000} minutes. It is not lost: run \`${resume}\`; that does not upload or charge again.` : `The picture ${key} stays flat for now: contact was lost while the cut-out (id ${id}) was being made: ${outcome.message} It is not lost: run \`${resume}\`; that does not charge again.` };
|
|
818
|
+
}
|
|
819
|
+
let cut: Awaited<ReturnType<typeof frameWithAlpha>>;
|
|
820
|
+
try { cut = await frameWithAlpha(webm, project.path(subjectKey)); } catch (e) {
|
|
821
|
+
rmSync(project.path(subjectKey), { force: true });
|
|
822
|
+
saveLayerEntry(project, key, { ...zoneEntry, cutoutId: undefined });
|
|
823
|
+
return flat(`The cut-out was made (about 1 second of your monthly cutout quota) but its subject layer could not be used: ${e instanceof Error ? e.message : String(e)}`);
|
|
824
|
+
}
|
|
825
|
+
const quota = "It used about 1 second of your monthly cutout quota.";
|
|
826
|
+
const verdict = coverageVerdict(cut.coverage);
|
|
827
|
+
if (!verdict.ok) {
|
|
828
|
+
rmSync(project.path(subjectKey), { force: true });
|
|
829
|
+
saveLayerEntry(project, key, { ...zoneEntry, noSubject: true, coverage: cut.coverage });
|
|
830
|
+
refreshManifest();
|
|
831
|
+
return { ok: true, data: { key, noSubject: true, coverage: cut.coverage, textZone: zoneEntry.textZone }, summary: `${verdict.note} ${quota} ${SENT_NOTE}` };
|
|
832
|
+
}
|
|
833
|
+
// The words' calm zone is measured again now that the subject is known, so that it is never offered behind it.
|
|
834
|
+
const textZone = await measureTextZone(project.path(key), cut.box).catch(() => zoneEntry.textZone);
|
|
835
|
+
const entry: LayerEntry = { subject: subjectKey, ...(cut.box ? { subjectBox: cut.box } : {}), textZone, coverage: Math.round(cut.coverage * 1000) / 1000 };
|
|
836
|
+
saveLayerEntry(project, key, entry);
|
|
837
|
+
refreshManifest();
|
|
838
|
+
return {
|
|
839
|
+
ok: true, data: { key, ...entry },
|
|
840
|
+
summary: `Layers${tag}: the subject is cut out into ${subjectKey} (it covers ${Math.round(cut.coverage * 100)}% of the picture) and the words have a calm ${textZone.region} (${textZone.luminance > 0.55 ? "light" : "dark"}). ${quota} ${SENT_NOTE}${opts.scene ? sceneNote : ` Use urls["${key}"] as the back and urls["${subjectKey}"] as the subject.`}`,
|
|
841
|
+
};
|
|
842
|
+
} finally { work.done(); }
|
|
843
|
+
}
|