reelkit-cli 0.1.3 → 0.3.0

package/src/commands/assets.ts

@@ -1,11 +1,12 @@
 import { randomUUID } from "node:crypto";
-import { copyFileSync, mkdirSync, readFileSync } from "node:fs";
+import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
 import { basename, dirname, resolve } from "node:path";
-import { download, uploadTo } from "../api/client";
+import { ApiFailure, download, uploadTo } from "../api/client";
 import { client, type Ctx, type Result } from "../context";
-import { LibraryKindSchema, UploadKindSchema } from "../contract";
+import { LibraryKindSchema, MAX_CUTOUT_SECONDS, MAX_UPLOAD_BYTES, UploadKindSchema, type CutoutType } from "../contract";
 import { PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
-import { buildManifest, voiceoverStale, type Voiceover } from "../project/manifest";
+import { keyGreen } from "../project/chromakey";
+import { buildManifest, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
 import { loadPlan } from "./plan";
 import { FILE_NAME } from "../render/validate";
 import { probeFile } from "../project/probe";
@@ -30,9 +31,80 @@ export function setSceneImage(project: Project, sceneId: string, path: string) {
   project.writeJson(FILES.images, { ...project.readJsonOr<Record<string, string>>(FILES.images, {}), [sceneId]: path });
 }
 
+const CUTOUT_POLL_MS = 5_000;
+const CUTOUT_PROGRESS_MS = 30_000;
+const CUTOUT_TIMEOUT_MS = 10 * 60_000;
+const SENT_NOTE = "The video was sent to the Reelkit server to be processed there; the server deletes its copy of it when the job ends.";
+
+// What a test may replace for a cutout: the clock the wait is timed by.
+export type CutoutDeps = { now?: () => number };
+
+type CutoutInput = { source: string; filename: string; contentType: CutoutType; bytes: number; durationSec: number };
+
+// A path as it is typed in a command: quoted only when it has to be.
+const shellArg = (s: string) => (/^[\w./~@%+=:,-]+$/.test(s) ? s : JSON.stringify(s));
+
+// Sends a video to the server and starts its cutout. The seconds are reserved when the job is run, so this is the point where it becomes
+// paid for: nothing is charged by a failure before it, and the id it returns is the only handle on the job after it.
+async function startCutout(ctx: Ctx, input: CutoutInput): Promise<string> {
+  const api = client(ctx);
+  ctx.log(`Sending ${input.filename} to the Reelkit server to cut the subject out.`);
+  const up = await api("cutoutStart", { filename: input.filename.slice(-200), contentType: input.contentType, bytes: input.bytes, durationSec: input.durationSec });
+  await uploadTo(up.uploadUrl, new Uint8Array(readFileSync(input.source)), input.contentType);
+  await api("cutoutRun", { id: up.id });
+  ctx.log(`Started cutout ${up.id}. It usually takes about a minute.`);
+  return up.id;
+}
+
+type CutoutOutcome = { kind: "done" } | { kind: "failed"; message: string } | { kind: "timeout" } | { kind: "lost"; message: string };
+
+// Asks every few seconds until the job ends (at most ten minutes), then saves the result at `dest`. A job that is not found or a login that
+// is wrong has nothing to resume and is thrown; any other trouble may be a hiccup on a job that is paid for, so it is an outcome.
+async function pollCutout(ctx: Ctx, id: string, dest: string, deps: CutoutDeps): Promise<CutoutOutcome> {
+  const api = client(ctx);
+  const now = deps.now ?? Date.now;
+  const started = now();
+  let lastLog = started;
+  try {
+    for (;;) {
+      const st = await api("cutoutStatus", { id });
+      if (st.status === "failed") return { kind: "failed", message: st.message ?? "The server gave no reason." };
+      if (st.status === "done") {
+        await download(st.url!, dest);
+        return { kind: "done" };
+      }
+      const elapsed = now() - started;
+      if (elapsed >= CUTOUT_TIMEOUT_MS) return { kind: "timeout" };
+      if (now() - lastLog >= CUTOUT_PROGRESS_MS) {
+        lastLog = now();
+        ctx.log(`Still cutting out the subject (${Math.round(elapsed / 1000)}s so far)...`);
+      }
+      await ctx.sleep(CUTOUT_POLL_MS);
+    }
+  } catch (e) {
+    if (e instanceof ApiFailure && (e.code === "not_found" || e.code === "unauthenticated")) throw e;
+    return { kind: "lost", message: e instanceof Error ? e.message : String(e) };
+  }
+}
+
+// The one line a person reads when a video cannot be cut out, or undefined when it can.
+function cutoutRefusal(file: string, probe: Awaited<ReturnType<typeof probeFile>>): string | undefined {
+  if (probe.kind !== "video") return `${file} is ${probe.kind}, not a video, so there is nothing to cut out.`;
+  if (!probe.durationSec || probe.durationSec <= 0) return `${file} has no length that could be read, so it cannot be cut out. Export it again.`;
+  if (probe.durationSec > MAX_CUTOUT_SECONDS) return `${file} is ${Math.ceil(probe.durationSec)} seconds long and a cutout takes at most ${MAX_CUTOUT_SECONDS}. Trim it to ${MAX_CUTOUT_SECONDS} seconds or less and run the command again.`;
+  if (probe.bytes > MAX_UPLOAD_BYTES) return `${file} is larger than 200 MB, which is the most a cutout takes. Trim or compress it and run the command again.`;
+  return undefined;
+}
+
 // Registers one of the user's own files. It stays on this machine unless --share is given.
-export async function assetsUpload(ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; share?: boolean; kind?: string; tags?: string }): Promise<Result> {
+export async function assetsUpload(
+  ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; green?: boolean; cutout?: boolean; resume?: string; share?: boolean; kind?: string; tags?: string },
+  deps: { key?: typeof keyGreen } & CutoutDeps = {},
+): Promise<Result> {
   const project = openProject(ctx.cwd);
+  if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green keys a green background on this machine for free, --cutout removes any background on the server." };
+  if (opts.resume && !opts.cutout) return { ok: false, summary: "--resume goes with --cutout: it keeps waiting for a cutout that was already started." };
+  if (opts.resume) return resumeUploadCutout(ctx, project, file, opts.resume, deps);
   if (opts.share && opts.kind === "component") return { ok: false, summary: "Components cannot be shared from the CLI yet. Share images, overlays, sound effects, music or clips." };
   const source = resolve(ctx.cwd, file);
   let probe: Awaited<ReturnType<typeof probeFile>>;
@@ -42,15 +114,31 @@ export async function assetsUpload(ctx: Ctx, file: string, opts: { describe?: st
     return { ok: false, summary: e instanceof Error ? e.message : String(e) };
   }
   if (opts.footage && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so it cannot be the footage.` };
+  if (opts.green && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so there is no green screen to key out.` };
+  const refusal = opts.cutout ? cutoutRefusal(file, probe) : undefined;
+  if (refusal) return { ok: false, summary: refusal };
   const kind = opts.share ? UploadKindSchema.safeParse(opts.kind) : undefined;
   if (opts.share && (!kind?.success || !opts.describe)) return { ok: false, summary: "Sharing needs --kind (image, overlay, sfx, music or clip) and --describe \"what it is\"." };
 
   const id = `a-${randomUUID().slice(0, 8)}`;
   const filename = basename(source).replace(/[^A-Za-z0-9._-]/g, "_");
   const key = `assets/user/${id}-${filename}`;
+  // A cutout is made on the server and paid for when it is run, so it is started before anything is registered: if the server refuses
+  // (quota used up, no GPU service) or the upload fails, nothing is left behind.
+  const cutoutId = opts.cutout ? await startCutout(ctx, { source, filename, contentType: probe.contentType as CutoutType, bytes: probe.bytes, durationSec: probe.durationSec! }) : undefined;
   mkdirSync(dirname(project.path(key)), { recursive: true });
   copyFileSync(source, project.path(key));
-  const record: AssetRecord = { id, filename, key, ...probe, ...(opts.describe ? { description: opts.describe } : {}), createdAt: new Date().toISOString() };
+  // A subject filmed on green gets a second copy with the green made transparent, sound kept. Nothing is registered if it cannot be keyed.
+  let keyedKey: string | undefined;
+  if (opts.green) {
+    keyedKey = key.replace(/\.[^./]+$/, "") + ".webm";
+    try { await (deps.key ?? keyGreen)(project.path(key), project.path(keyedKey), { keepAudio: true, requireGreen: true }); }
+    catch (e) {
+      rmSync(project.path(key), { force: true });
+      return { ok: false, summary: e instanceof Error ? e.message : String(e) };
+    }
+  }
+  const record: AssetRecord = { id, filename, key, ...probe, ...(opts.describe ? { description: opts.describe } : {}), ...(keyedKey ? { keyedKey } : {}), ...(cutoutId ? { cutoutId } : {}), createdAt: new Date().toISOString() };
   project.writeJson(FILES.assetIndex, [...project.assets(), record]);
   if (opts.footage) project.writeJson(FILES.config, { ...project.config(), footage: id });
 
@@ -66,12 +154,52 @@ export async function assetsUpload(ctx: Ctx, file: string, opts: { describe?: st
     await uploadTo(up.uploadUrl, new Uint8Array(readFileSync(source)), probe.contentType);
     libraryId = (await api("libraryCommit", { id: up.id })).item.id;
   }
+  if (cutoutId) {
+    const finished = await finishUploadCutout(ctx, project, record, file, deps);
+    return { ...finished, data: { ...(finished.data as object), ...(libraryId ? { libraryId } : {}) }, summary: `Added ${filename} as ${id}${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ""}. ${finished.summary}` };
+  }
   return {
     ok: true, data: { ...record, ...(libraryId ? { libraryId } : {}) },
-    summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.`,
+    summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.${keyedKey ? ` The green is keyed out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"].` : ""}`,
   };
 }
 
+// Waits for the cutout of a registered video and saves it beside the original. The asset stays registered whatever happens.
+async function finishUploadCutout(ctx: Ctx, project: Project, record: AssetRecord, file: string, deps: CutoutDeps): Promise<Result> {
+  const cutoutId = record.cutoutId!;
+  const keyedKey = record.key.replace(/\.[^./]+$/, "") + ".webm";
+  const save = (r: AssetRecord) => project.writeJson(FILES.assetIndex, project.assets().map((a) => (a.id === r.id ? r : a)));
+  const outcome = await pollCutout(ctx, cutoutId, project.path(keyedKey), deps);
+  const resume = `reelkit assets upload ${shellArg(file)} --cutout --resume ${cutoutId}`;
+  const { cutoutId: _pending, ...settled } = record;
+  if (outcome.kind === "done") {
+    const done: AssetRecord = { ...settled, keyedKey };
+    save(done);
+    const seconds = Math.ceil(record.durationSec ?? 0);
+    return { ok: true, data: done, summary: `The subject is cut out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"]. It used ${seconds} second${seconds === 1 ? "" : "s"} of your monthly cutout quota. ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "failed") {
+    save(settled);
+    return { ok: false, data: { ...settled, status: "failed" }, summary: `The cutout failed: ${outcome.message} You were not charged. The video stays added without a cutout; try a different video. ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "timeout") {
+    return { ok: false, data: { ...record, status: "pending" }, summary: `The video stays added without a cutout: the cutout (id ${cutoutId}) is not ready after ${CUTOUT_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${resume}\` to keep waiting for it; that does not upload or charge again.` };
+  }
+  return { ok: false, data: { ...record, status: "pending" }, summary: `The video stays added without a cutout: contact was lost while it was being made (${outcome.message}) The cutout (id ${cutoutId}) is not lost: run \`${resume}\`; that does not upload or charge again.` };
+}
+
+// Picks up a cutout that was started earlier: finds the asset waiting for it and only polls.
+async function resumeUploadCutout(ctx: Ctx, project: Project, file: string, id: string, deps: CutoutDeps): Promise<Result> {
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(id)) return { ok: false, summary: `"${id}" is not a cutout id. Copy it from the earlier message.` };
+  const record = project.assets().find((a) => a.cutoutId === id);
+  if (!record) return { ok: false, summary: `No asset in this project is waiting for cutout ${id}. Copy the id from the earlier message.` };
+  const name = basename(file).replace(/[^A-Za-z0-9._-]/g, "_");
+  if (name !== record.filename) return { ok: false, summary: `Cutout ${id} is for ${record.filename}, not ${file}. Give the file it was started with.` };
+  ctx.log(`Waiting for cutout ${id} of ${record.filename}.`);
+  const finished = await finishUploadCutout(ctx, project, record, file, deps);
+  return { ...finished, summary: `Resumed ${record.filename} (${record.id}). ${finished.summary}` };
+}
+
 export async function assetsSearch(ctx: Ctx, query: string, opts: { kind?: string; limit?: string }): Promise<Result> {
   const kind = opts.kind === undefined ? undefined : LibraryKindSchema.safeParse(opts.kind);
   if (kind && !kind.success) return { ok: false, summary: `Unknown kind "${opts.kind}". Use one of: ${LibraryKindSchema.options.join(", ")}.` };
@@ -90,19 +218,35 @@ export function unsafeName(name: string, kind: string): boolean {
   return kind === "component" && (!FILE_NAME.test(name) || name === "Video.tsx");
 }
 
-export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean }): Promise<Result> {
+// What a test may replace: the ffmpeg call that keys green out, and the clock the clip wait is timed by.
+export type ClipDeps = { key?: (input: string, output: string) => Promise<void>; now?: () => number };
+
+// The keyed copy sits beside its clip: clip.mp4 becomes clip.webm.
+const keyedPathOf = (path: string) => path.replace(/\.[^./]+$/, "") + ".webm";
+
+function setSceneClip(project: Project, sceneId: string, record: ClipRecord) {
+  project.writeJson(FILES.clips, { ...project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {}), [sceneId]: record });
+}
+
+export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean }, deps: ClipDeps = {}): Promise<Result> {
   const project = openProject(ctx.cwd);
   if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id)) return { ok: false, summary: `"${id}" is not a library id. Copy the id from \`reelkit assets search\`.` };
   if (opts.scene) {
     // Checked before anything is downloaded.
-    const illustrations = loadPlan(project).scenes.filter((s) => s.treatment === "illustration").map((s) => s.id);
-    if (!illustrations.includes(opts.scene)) {
-      return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}` };
+    const scenes = loadPlan(project).scenes;
+    const illustrations = scenes.filter((s) => s.treatment === "illustration").map((s) => s.id);
+    const clipScenes = scenes.filter((s) => s.treatment === "clip").map((s) => s.id);
+    if (!illustrations.includes(opts.scene) && !clipScenes.includes(opts.scene)) {
+      return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}${clipScenes.length ? ` The clip scenes are: ${clipScenes.join(", ")}.` : ""}` };
     }
   }
   const pulled = await client(ctx)("libraryPull", { id });
   if (unsafeName(pulled.filename, pulled.item.kind)) return { ok: false, summary: `The library returned an unsafe file name for ${id}, so nothing was written. Report this item.` };
-  if (opts.scene && pulled.item.kind !== "image") return { ok: false, summary: `${id} is ${pulled.item.kind}, not an image, so it cannot be a scene's image.` };
+  if (opts.scene) {
+    const treatment = loadPlan(project).scenes.find((x) => x.id === opts.scene)?.treatment;
+    if (treatment === "clip" && pulled.item.kind !== "clip") return { ok: false, summary: `${id} is ${pulled.item.kind}, not a clip, so it cannot be scene ${opts.scene}'s clip.` };
+    if (treatment !== "clip" && pulled.item.kind !== "image") return { ok: false, summary: `${id} is ${pulled.item.kind}, not an image, so it cannot be a scene's image.` };
+  }
   // A component is code, not media: it goes beside Video.tsx, which may only import files in src/.
   if (pulled.item.kind === "component") {
     const path = `src/${pulled.filename}`;
@@ -116,15 +260,24 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
   const path = `assets/lib/${id}/${pulled.filename}`;
   await download(pulled.url, project.path(path));
   recordPull(project, id, { path, kind: pulled.item.kind, title: pulled.item.title, meta: pulled.item.meta });
+  // A clip filmed on green also comes keyed, so it can be laid over a background.
+  const green = pulled.item.kind === "clip" && pulled.item.meta.greenScreen === true;
+  const keyedPath = green ? keyedPathOf(path) : undefined;
+  if (keyedPath) await (deps.key ?? keyGreen)(project.path(path), project.path(keyedPath));
+  const sceneIsClip = pulled.item.kind === "clip";
   let note: string | undefined;
   if (opts.scene) {
-    setSceneImage(project, opts.scene, path);
+    if (sceneIsClip) {
+      const durationSec = typeof pulled.item.meta.durationSec === "number" ? pulled.item.meta.durationSec : 5;
+      setSceneClip(project, opts.scene, { key: path, greenScreen: green, durationSec, ...(keyedPath ? { keyedKey: keyedPath } : {}) });
+    } else setSceneImage(project, opts.scene, path);
     if (project.exists(FILES.plan)) note = tryManifest(project, loadPlan(project)).note;
   }
+  const refs = keyedPath ? ` The original is urls["${path}"]; the keyed copy with a transparent background is urls["${keyedPath}"].` : ` Reference it in the composition as urls["${path}"].`;
   return {
-    ok: true, data: { path, kind: pulled.item.kind },
+    ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}) },
     summary: [
-      opts.scene ? `Pulled ${id} as the image for scene ${opts.scene}.` : `Pulled ${id} to ${path}. Reference it in the composition as urls["${path}"].`,
+      opts.scene ? `Pulled ${id} as the ${sceneIsClip ? "clip" : "image"} for scene ${opts.scene}.${keyedPath ? refs : ""}` : `Pulled ${id} to ${path}.${refs}`,
       ...(note ? [note] : []),
     ].join("\n"),
   };
@@ -194,3 +347,169 @@ export async function assetsGenImage(ctx: Ctx, prompt: string | undefined, opts:
     summary: [`Generated the image for scene ${scene.id} at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}.`, ...(note ? [note] : [])].join("\n"),
   };
 }
+
+
+const CLIP_POLL_MS = 5_000;
+const CLIP_PROGRESS_MS = 30_000;
+const CLIP_TIMEOUT_MS = 10 * 60_000;
+
+// A clip takes minutes, so it is a job: start it (which reserves the clip from the quota), then ask every few seconds until it ends.
+// The id is the only handle on a paid job, so every way out after the start says it and how to resume.
+export async function assetsGenClip(
+  ctx: Ctx, prompt: string | undefined,
+  opts: { scene: string; green?: boolean; cutout?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string },
+  deps: ClipDeps = {},
+): Promise<Result> {
+  const project = openProject(ctx.cwd);
+  const plan = loadPlan(project);
+  const scene = plan.scenes.find((s) => s.id === opts.scene);
+  if (!scene || scene.treatment !== "clip" || !scene.clipPrompt) return { ok: false, summary: `Scene ${opts.scene} is not a clip scene, so it does not take a clip. Set its treatment to "clip" and give it a clipPrompt in plan.json.` };
+  if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green asks for a green background and keys it on this machine for free, --cutout removes any background on the server." };
+  const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
+  if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
+  if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
+  if (opts.share && (!scene.shareable || scene.userAssetIds.length > 0 || project.footage())) {
+    return { ok: false, summary: `Scene ${scene.id} is private (it is not marked shareable in plan.json, or it shows your own files), so --share is refused. Run again without --share.` };
+  }
+
+  const key = deps.key ?? keyGreen;
+  const now = deps.now ?? Date.now;
+  const keyedKey = `assets/clip-${scene.id}.webm`;
+  const records = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
+  const have = records[scene.id];
+  // A scene that already has its clip is not charged for again, whatever prompt is given; only --redo asks for a new one.
+  if (have && project.exists(have.key) && !opts.redo && !opts.resume) {
+    if (opts.cutout) {
+      if (have.keyedKey && project.exists(have.keyedKey)) return { ok: true, data: { path: have.key, keyedPath: have.keyedKey, skipped: true }, summary: `Scene ${scene.id} already has a clip at ${have.key} and its cutout at ${have.keyedKey}. Use --redo to generate a new clip.` };
+      // The clip was paid for but its cutout was not made (or is still being made): only the cutout is done, and the clip is not charged again.
+      const cut = await cutoutScene(ctx, project, scene.id, have, now);
+      return { ...cut, data: { path: have.key, ...(cut.ok ? { keyedPath: keyedKey } : {}) }, summary: cut.ok ? `Cut out scene ${scene.id}'s clip. ${cut.summary}` : cut.summary };
+    }
+    if (opts.green && have.greenScreen && !have.keyedKey) {
+      // The clip was paid for but not keyed (ffmpeg failed or --green came later): keying is free, so do it now.
+      const keyed = await keyScene(project, scene.id, have, key);
+      if (!keyed.ok) return keyed;
+      return { ok: true, data: { path: have.key, keyedPath: keyedKey }, summary: `Keyed the green out of scene ${scene.id}'s clip into ${keyedKey}.` };
+    }
+    return { ok: true, data: { path: have.key, skipped: true }, summary: `Scene ${scene.id} already has a clip at ${have.key}. Use --redo to generate a new one.` };
+  }
+
+  const api = client(ctx);
+  const started = now();
+  let id = opts.resume;
+  if (!id) {
+    const job = await api("clipStart", {
+      prompt: prompt ?? scene.clipPrompt, aspect: plan.aspect, greenScreen: Boolean(opts.green), durationSec: seconds,
+      shareable: Boolean(opts.share), tags: scene.imageTags,
+    });
+    id = job.id;
+    ctx.log(`Started clip ${id} for scene ${scene.id}. It usually takes a few minutes.`);
+  } else ctx.log(`Waiting for clip ${id} for scene ${scene.id}.`);
+  const resume = `reelkit assets gen clip --resume ${id} --scene ${scene.id}${opts.cutout ? " --cutout" : ""}`;
+
+  try {
+    let lastLog = started;
+    for (;;) {
+      const st = await api("clipStatus", { id });
+      if (st.status === "failed") {
+        return { ok: false, data: { id, status: "failed" }, summary: `The clip for scene ${scene.id} failed: ${st.message ?? "the video provider gave no reason."} You were not charged. Change the prompt and run the command again.` };
+      }
+      if (st.status === "done") {
+        const path = `assets/clip-${scene.id}.${st.ext}`;
+        await download(st.url!, project.path(path));
+        const green = Boolean(st.greenScreen ?? opts.green);
+        const record: ClipRecord = { key: path, greenScreen: green, durationSec: st.durationSec ?? seconds };
+        setSceneClip(project, scene.id, record);
+        let keyedPath: string | undefined;
+        if (green || opts.green) {
+          // The paid clip is already saved above; if keying fails, running the command again keys it without a new charge.
+          const keyed = await keyScene(project, scene.id, { ...record, greenScreen: true }, key);
+          if (!keyed.ok) return keyed;
+          keyedPath = keyedKey;
+        }
+        let cutoutNote: string | undefined;
+        if (opts.cutout) {
+          // The paid clip is already saved above; if the cutout fails, running the command again does only the cutout.
+          const cut = await cutoutScene(ctx, project, scene.id, record, now);
+          const shared = st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : "";
+          if (!cut.ok) return { ...cut, data: { id, path, durationSec: record.durationSec, libraryId: st.libraryId ?? null }, summary: `Generated the clip for scene ${scene.id}${shared}. ${cut.summary}` };
+          keyedPath = keyedKey;
+          cutoutNote = cut.summary;
+        }
+        const { note } = tryManifest(project, plan);
+        return {
+          ok: true, data: { id, path, durationSec: record.durationSec, ...(keyedPath ? { keyedPath } : {}), libraryId: st.libraryId ?? null },
+          summary: [
+            `Generated the clip for scene ${scene.id} at ${path}${keyedPath && !cutoutNote ? `, keyed to ${keyedPath}` : ""}${st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : ""}.${cutoutNote ? ` ${cutoutNote}` : ""}`,
+            ...(note ? [note] : []),
+          ].join("\n"),
+        };
+      }
+      const elapsed = now() - started;
+      if (elapsed >= CLIP_TIMEOUT_MS) {
+        return { ok: false, data: { id, status: "pending" }, summary: `The clip for scene ${scene.id} (id ${id}) is not ready after ${CLIP_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${resume}\` to keep waiting for it; that does not charge again.` };
+      }
+      if (now() - lastLog >= CLIP_PROGRESS_MS) {
+        lastLog = now();
+        ctx.log(`Still making the clip for scene ${scene.id} (${Math.round(elapsed / 1000)}s so far)...`);
+      }
+      await ctx.sleep(CLIP_POLL_MS);
+    }
+  } catch (e) {
+    // A clip that is not found or a login that is wrong has nothing to resume; anything else may be a hiccup on a clip that is paid for.
+    if (e instanceof ApiFailure && (e.code === "not_found" || e.code === "unauthenticated")) throw e;
+    return { ok: false, data: { id }, summary: `Lost contact while the clip for scene ${scene.id} (id ${id}) was being made: ${e instanceof Error ? e.message : String(e)} The clip is not lost: run \`${resume}\`; that does not charge again.` };
+  }
+}
+
+// Cuts the subject out of a scene's saved clip on the server and records the transparent copy as the clip's keyed copy. The id of a job
+// that is still being made is kept in the record, so running the command again waits for it instead of paying again. The clip stays
+// whatever happens here, and every way out says so and how to try the cutout again.
+async function cutoutScene(ctx: Ctx, project: Project, sceneId: string, record: ClipRecord, now: () => number): Promise<Result> {
+  const keyedKey = `assets/clip-${sceneId}.webm`;
+  const retry = `reelkit assets gen clip --scene ${sceneId} --cutout`;
+  const saved = `The clip is saved at ${record.key}`;
+  let id = record.cutoutId;
+  if (!id) {
+    try {
+      id = await startCutout(ctx, { source: project.path(record.key), filename: basename(record.key), contentType: "video/mp4", bytes: statSync(project.path(record.key)).size, durationSec: record.durationSec });
+    } catch (e) {
+      return { ok: false, summary: `${saved}, but the cutout could not be started: ${e instanceof Error ? e.message : String(e)} Run \`${retry}\` to try the cutout again; the clip is not charged again.` };
+    }
+    setSceneClip(project, sceneId, { ...record, cutoutId: id });
+  } else ctx.log(`Waiting for cutout ${id} of scene ${sceneId}'s clip.`);
+  const { cutoutId: _pending, ...settled } = record;
+  let outcome: CutoutOutcome;
+  try {
+    outcome = await pollCutout(ctx, id, project.path(keyedKey), { now });
+  } catch (e) {
+    if (!(e instanceof ApiFailure && e.code === "not_found")) throw e;
+    setSceneClip(project, sceneId, settled);
+    return { ok: false, summary: `${saved}, but the server no longer has cutout ${id}, so it was forgotten. Run \`${retry}\` to start a new one; the clip is not charged again.` };
+  }
+  if (outcome.kind === "done") {
+    setSceneClip(project, sceneId, { ...settled, keyedKey });
+    const seconds = Math.ceil(record.durationSec);
+    return { ok: true, summary: `The subject is cut out into ${keyedKey} (${seconds} second${seconds === 1 ? "" : "s"} of your monthly cutout quota). ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "failed") {
+    setSceneClip(project, sceneId, settled);
+    return { ok: false, summary: `${saved}, but the cutout failed: ${outcome.message} You were not charged for the cutout. Run \`${retry}\` to try the cutout again; the clip is not charged again. ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "timeout") {
+    return { ok: false, summary: `${saved}, but the cutout (id ${id}) is not ready after ${CUTOUT_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${retry}\` to keep waiting for it; that does not upload or charge again. ${SENT_NOTE}` };
+  }
+  return { ok: false, summary: `${saved}, but contact was lost while the cutout (id ${id}) was being made: ${outcome.message} It is not lost: run \`${retry}\`; that does not upload or charge again.` };
+}
+
+// Keys a scene's saved clip and records the keyed copy. A failure leaves the record as it was.
+async function keyScene(project: Project, sceneId: string, record: ClipRecord, key: NonNullable<ClipDeps["key"]>): Promise<Result> {
+  const keyedKey = `assets/clip-${sceneId}.webm`;
+  try {
+    await key(project.path(record.key), project.path(keyedKey));
+  } catch (e) {
+    return { ok: false, data: { path: record.key }, summary: `The clip is saved at ${record.key}, but the green could not be keyed out: ${e instanceof Error ? e.message : String(e)}` };
+  }
+  setSceneClip(project, sceneId, { ...record, greenScreen: true, keyedKey });
+  return { ok: true, summary: "" };
+}

package/src/commands/auth.ts

@@ -113,6 +113,6 @@ export async function whoami(ctx: Ctx): Promise<Result> {
   const q = me.quota;
   return {
     ok: true, data: me,
-    summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
+    summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nClips: ${q.clips.used}/${q.clips.limit}\nCutouts: ${q.cutoutSeconds.used}/${q.cutoutSeconds.limit} seconds\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
   };
 }

package/src/contract/index.ts

@@ -39,9 +39,34 @@
 //   a file URL serves one file, an upload URL accepts uploads for one object only.
 // - The public listing's `previewUrl` is a preview that needs no login. An item with a preview of its own gets that one; otherwise an `image`
 //   or an `sfx` gets a URL of its own file (a picture and a short sound need no login to look at or hear), and every other kind (music,
-//   overlay, clip, component), whose full file is long media or code, gets no `previewUrl`.
+//   overlay, clip, component), whose full file is long media or code, gets no `previewUrl`. The one exception is a clip of ten seconds
+//   or less (`meta.durationSec`): short generated clips are made to be shared, so such a clip is previewed by its own file.
 // - Every returned URL needs no headers and stays valid for at least five minutes.
 // - The generation routes (`voiceover`, `images`) are synchronous: they return the URL of the finished file.
+// - A video clip takes minutes, so it is a job with two calls. `clipStart` checks the request, reserves one clip from the caller's monthly clip
+//   quota and answers at once with an id and `pending`; the client then asks `clipStatus` for that id until it is `done` or `failed`.
+//   `url`, `ext` and `contentType` are present exactly when the status is `done`, and `message` (one plain line saying what went wrong)
+//   exactly when it is `failed`. A failed clip is not charged: the reservation is released. A clip id belongs to its caller: another user's id,
+//   or an unknown one, is `not_found`.
+// - When the quota is used up, `clipStart` is 429 `quota_exceeded` with the reset date, as for images. When the server has no video provider
+//   configured, `clipStart` is 400 `invalid_request` with the message "Clip generation is not available on this server yet."
+// - A green-screen clip (`greenScreen: true`) is a subject filmed on a flat pure green background, meant to be keyed out by the client. The server
+//   stores and returns the original, green and all; it does not remove the background. A shareable clip also becomes a library item of kind
+//   `clip` in review, whose `meta` may carry `greenScreen: true` and `durationSec`, and its id is returned as `libraryId`.
+// - A cutout removes the background from a video, whatever it is, and answers with the subject on a transparent background. The video is sent to
+//   the server, a GPU model separates the subject, and a transparent `.webm` (VP9 with alpha, sound kept) comes back. It takes about a minute.
+//   It is a job with three calls: `cutoutStart` checks the request and answers with an id and an `uploadUrl` (charging nothing; the client sends
+//   the file there exactly as for a library upload); `cutoutRun` checks that the file arrived (else 400 `invalid_request`), reserves
+//   `ceil(durationSec)` seconds from the caller's monthly cutout quota and starts the job; the client then asks `cutoutStatus` until it is `done`
+//   or `failed`, with the same exactly-when rules as `clipStatus` (`url`, `ext` and `contentType` when done, `message` when failed).
+//   `cutoutStatus` for a job whose `cutoutRun` has not been called is 400 `invalid_request`.
+//   `cutoutRun` for a job already started answers its current pending state and does not charge again. A failed cutout is not charged.
+//   A cutout id belongs to its caller: another user's id, or an unknown one, is `not_found` for run and status.
+// - The video uploaded for a cutout is used only to make the cutout, and is deleted from the server when the job ends. The result stays in
+//   the user's private files and is never shared. A cutout is charged in whole seconds of video (`ceil(durationSec)`), and a video longer than
+//   20 seconds is refused by the request schema.
+// - When the cutout quota is used up, `cutoutRun` is 429 `quota_exceeded` with the reset date. When the server has no GPU service configured,
+//   `cutoutStart` and `cutoutRun` are 400 `invalid_request` with the message "Background removal is not available on this server yet."
 // - `durationSec` is the decoded audio length; `words` are in seconds from the start of that audio.
 // - Search returns published items only, and leaves out items that are not matches at all; the default `limit` is 8.
 import { z } from "zod";
@@ -71,6 +96,11 @@ const Meta = z.record(z.string(), z.unknown()).refine((m) => new TextEncoder().e
 
 // The most a single upload may be: 200 MB.
 export const MAX_UPLOAD_BYTES = 209_715_200;
+// The video types a cutout takes.
+export const CutoutTypeSchema = z.enum(["video/mp4", "video/quicktime", "video/webm"]);
+export type CutoutType = z.infer<typeof CutoutTypeSchema>;
+// The longest video a cutout takes, in seconds.
+export const MAX_CUTOUT_SECONDS = 20;
 
 export const ErrorCodeSchema = z.enum(["unauthenticated", "quota_exceeded", "not_found", "invalid_request", "server_error"]);
 export type ErrorCode = z.infer<typeof ErrorCodeSchema>;
@@ -89,7 +119,8 @@ export const LibraryItemSchema = z.object({
   title: z.string(),
   description: z.string(),
   tags: z.array(z.string()),
-  // width, height, durationSec, aspect, props, source, licence: whatever applies to the kind.
+  // width, height, durationSec, aspect, props, source, licence: whatever applies to the kind. A clip may carry `greenScreen: true` (filmed on a
+  // flat green background, to be keyed out) and `durationSec`.
   meta: z.record(z.string(), z.unknown()),
   visibility: z.enum(["private", "review", "published"]),
 });
@@ -105,7 +136,7 @@ export type Voice = z.infer<typeof VoiceSchema>;
 const Meter = z.object({ used: z.number(), limit: z.number() });
 export const MeSchema = z.object({
   userId: z.string(), handle: z.string(),
-  quota: z.object({ voiceoverChars: Meter, images: Meter, resetsAt: z.string() }),
+  quota: z.object({ voiceoverChars: Meter, images: Meter, clips: Meter, cutoutSeconds: Meter, resetsAt: z.string() }),
   // How many of the user's own items are published in the shared library. An item still in review, or sent back, is not counted.
   contributions: z.number(),
 });
@@ -148,6 +179,34 @@ export const routes = {
     z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12) }),
     // libraryId is set when the image was also added to the shared library.
     z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp"]), contentType: z.string(), libraryId: z.string().optional() })),
+  clipStart: route("POST", "/clips", true,
+    z.object({
+      prompt: text(2000).min(1), aspect: AspectSchema, greenScreen: z.boolean().default(false), durationSec: z.union([z.literal(5), z.literal(10)]).default(5),
+      shareable: z.boolean(), tags: z.array(text(40)).max(12),
+    }),
+    z.object({ id: z.string(), status: z.literal("pending") })),
+  clipStatus: route("POST", "/clips/status", true,
+    z.object({ id: ItemId }),
+    z.object({
+      id: z.string(), status: z.enum(["pending", "done", "failed"]), message: z.string().optional(),
+      url: z.string().optional(), ext: z.enum(["mp4"]).optional(), contentType: z.string().optional(), durationSec: z.number().positive().optional(),
+      greenScreen: z.boolean().optional(), libraryId: z.string().optional(),
+    }).refine((r) => (r.status === "done") === (r.url !== undefined && r.ext !== undefined && r.contentType !== undefined) && (r.status === "failed") === (r.message !== undefined),
+      "url, ext and contentType belong to done and message to failed")),
+  cutoutStart: route("POST", "/cutouts", true,
+    z.object({
+      filename: text(200), contentType: CutoutTypeSchema,
+      bytes: z.number().int().min(1).max(MAX_UPLOAD_BYTES), durationSec: z.number().positive().max(MAX_CUTOUT_SECONDS),
+    }),
+    z.object({ id: z.string(), uploadUrl: z.string() })),
+  cutoutRun: route("POST", "/cutouts/run", true, z.object({ id: ItemId }), z.object({ id: z.string(), status: z.literal("pending") })),
+  cutoutStatus: route("POST", "/cutouts/status", true,
+    z.object({ id: ItemId }),
+    z.object({
+      id: z.string(), status: z.enum(["pending", "done", "failed"]), message: z.string().optional(),
+      url: z.string().optional(), ext: z.enum(["webm"]).optional(), contentType: z.literal("video/webm").optional(),
+    }).refine((r) => (r.status === "done") === (r.url !== undefined && r.ext !== undefined && r.contentType !== undefined) && (r.status === "failed") === (r.message !== undefined),
+      "url, ext and contentType belong to done and message to failed")),
   publicLibrary: route("GET", "/public/library", false,
     z.object({ q: text(200).optional(), kind: LibraryKindSchema.optional(), page: z.coerce.number().int().min(1).max(10000).optional() }),
     // With `q` the items are the best matches by meaning, best first, each with `match` (0..1), in one page (`hasMore` false). Under heavy

package/src/pipeline/schema.ts

@@ -8,9 +8,12 @@ export type Aspect = z.infer<typeof AspectSchema>;
 export const SceneSchema = z.object({
   id: z.string().regex(/^[a-z0-9-]+$/).describe("short unique id, lowercase letters, digits, dashes"),
   narration: z.string().min(1).describe("the words spoken in this scene"),
-  treatment: z.enum(["motion-graphic", "illustration", "footage-overlay"]),
+  // clip: a generated or reused video clip is the scene's picture.
+  treatment: z.enum(["motion-graphic", "illustration", "footage-overlay", "clip"]),
   onScreenText: z.array(z.string()),
   imagePrompt: z.string().nullable().describe("required when treatment is illustration, otherwise null"),
+  // Optional only so plans stored before clips existed still load.
+  clipPrompt: z.string().nullable().optional().describe("required when treatment is clip, otherwise null: one subject, one action, a camera move, lighting, no text"),
   shareable: z.boolean().describe("true only when imagePrompt is fully generic: no brand, product, person or detail specific to this user. Such images are saved to a library shared by all users."),
   imageTags: z.array(z.string()).describe("3 to 6 short tags describing the illustration; empty when imagePrompt is null"),
   userAssetIds: z.array(z.string()),
@@ -43,6 +46,10 @@ export const AssetRecordSchema = z.object({
   durationSec: z.number().optional(),
   // What the file shows, as given at upload, so the plan can place it where it is relevant.
   description: z.string().optional(),
+  // A green-screen video also has a copy with the green made transparent, at this path.
+  keyedKey: z.string().optional(),
+  // The server cutout still being made for this video. Kept so that coming back to wait for it neither uploads nor charges again.
+  cutoutId: z.string().optional(),
   createdAt: z.string(),
 });
 export type AssetRecord = z.infer<typeof AssetRecordSchema>;
@@ -57,6 +64,9 @@ export const ManifestSceneSchema = z.object({
   voiceoverKey: z.string().optional(),
   words: z.array(WordTimingSchema),
   imageKey: z.string().optional(),
+  // The scene's video clip, and its transparent copy when it was made on a green background.
+  clipKey: z.string().optional(),
+  clipKeyedKey: z.string().optional(),
   userAssetKeys: z.array(z.string()),
 });
 export type ManifestScene = z.infer<typeof ManifestSceneSchema>;
@@ -103,6 +113,8 @@ export function validatePlan(
   for (const s of plan.scenes) {
     if (s.treatment === "illustration" && !s.imagePrompt)
       errors.push(`Scene ${s.id}: imagePrompt is required for an illustration.`);
+    if (s.treatment === "clip" && !s.clipPrompt)
+      errors.push(`Scene ${s.id}: clipPrompt is required for a clip.`);
     for (const id of s.userAssetIds)
       if (!allowed.has(id)) errors.push(`Scene ${s.id}: unknown user asset ${id}.`);
   }