reelkit-cli 0.2.0 → 0.3.1

package/README.md

@@ -53,13 +53,13 @@ reelkit render
 | `reelkit whoami` | Account, quota, contributions |
 | `reelkit install` | Install the skill into your coding agents (`--agent <id>`, `--force`) |
 | `reelkit init [name]` | Set up a video project, in a new folder named after it |
-| `reelkit assets upload <file>` | Add one of your own files (private unless `--share`) |
+| `reelkit assets upload <file>` | Add one of your own files (private unless `--share`); `--green` or `--cutout` also makes a transparent copy of a video (`--cutout` sends it to the server) |
 | `reelkit assets search "<query>"` | Search the shared library by meaning; each result shows a match percentage |
 | `reelkit assets pull <id>` | Download a library item (a component lands in `src/`) |
 | `reelkit assets voices` | List narration voices |
 | `reelkit assets voiceover` | Record narration from `plan.json` |
 | `reelkit assets gen image` | Generate a scene's illustration |
-| `reelkit assets gen clip` | Generate a scene's video clip, or a green-screen one keyed to a transparent video |
+| `reelkit assets gen clip` | Generate a scene's video clip, or a green-screen one keyed to a transparent video, or one with the subject cut out on the server |
 | `reelkit plan check` | Validate `plan.json` |
 | `reelkit check` | Check the composition without rendering |
 | `reelkit preview` | Two test frames per scene |
@@ -71,7 +71,7 @@ Your own files, your plan and your video stay on your machine. Illustrations tha
 
 A component you pull from the library is code, and it runs on your machine when you preview or render. The library only serves components published by Reelkit.
 
-The CLI talks to `https://reelkit-kohl.vercel.app/api/v1` by default. Set `REELKIT_API_URL` to point it at a different API. To use a local API while developing, set `REELKIT_API_URL=http://localhost:3000/api/v1`.
+The CLI talks to `https://reelkit.cc/api/v1` by default. Set `REELKIT_API_URL` to point it at a different API. To use a local API while developing, set `REELKIT_API_URL=http://localhost:3000/api/v1`.
 
 ## Development
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reelkit-cli",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "CLI and Claude skill for making short-form video with a shared asset library.",
   "license": "MIT",
   "author": "Daniel Livshin",
@@ -78,5 +78,5 @@
       "optional": true
     }
   },
-  "homepage": "https://reelkit-kohl.vercel.app"
+  "homepage": "https://reelkit.cc"
 }

package/skill/SKILL.md

@@ -23,7 +23,7 @@ Find out what the video is about and collect what the user has. Ask for nothing
 
 For each file the user gives, look at it, then register it with a description of what it shows:
 `reelkit assets upload ./logo.png --describe "Acme logo, white wordmark on blue"`
-Add `--footage` for a video the motion design should be laid over. User files stay on this machine.
+Add `--footage` for a video the motion design should be laid over. User files stay on this machine. A video of someone or something on a plain background can be made transparent: `--green` keys a green background locally for free, and `--cutout` removes any background on the server (the video is sent to Reelkit, so ask first; see `reference/clips.md`).
 
 ### 2. Look
 Read `reference/styles.md` and agree the video's look with the user: one of its looks, or their own. If they already described what they want, match it and confirm in one sentence. Ask anything still open in one message, each question with a default. If any on-screen text will be Hebrew, read `reference/hebrew-rtl.md` now: it changes how words may enter and how lines are written.

package/skill/reference/clips.md

@@ -52,6 +52,46 @@ import { Captions, KeyedClip, SceneFrame } from "reelkit/kit";
 - A green-screen scene still has a `clip` treatment and a `clipPrompt`, with the background and the type in the composition. A library clip with the same use comes keyed too: `reelkit assets pull <id> --scene <sceneId>` writes the `.webm` beside the `.mp4` when the clip is marked green screen.
 - It costs the same as any clip and is harder to get right: at most one or two green-screen clips per video.
 
+## Your own green-screen footage
+If the user filmed themselves or a product in front of a green background, register it with `reelkit assets upload ./me.mp4 --green --describe "what it shows"`. The original is kept and a copy with the green made transparent, sound included, is written beside it; the command prints the `urls[...]` path of that copy. The key colour is read from the corners of the frame, so the green does not have to be perfect, but it does have to fill the corners: a video whose corners are not green is refused. For a talking presenter, leave the copy's sound on (`muted={false}`) and do not also play the original.
+
+## No green screen: cut the subject out
+Use this when the footage or the clip is not on a green background but the subject has to be layered over something else.
+- If the background is green, use `--green` instead: it is free and runs on this machine. A cutout is the fallback.
+- `reelkit assets upload ./me.mp4 --cutout --describe "what it shows"` sends the video to the Reelkit server, where a GPU model separates the subject, and writes a transparent `.webm` (sound kept) beside the original; the command prints its `urls[...]` path. It takes about a minute. If it times out, the command prints the line that continues it (`--resume <id>`); that neither uploads nor charges again.
+- The video leaves the user's machine, so ask the user before cutting out their footage.
+- At most 20 seconds of video per cutout, counted in whole seconds against a monthly quota; `reelkit whoami` shows what is left.
+- Use the result exactly like a keyed clip: `KeyedClip` with the `.webm`, and the Layers section below applies.
+- Edges are best when the subject is well lit and clearly different from the background. Hair and fast motion can shimmer: check both preview frames.
+- A generated clip can be cut out in one step: `reelkit assets gen clip --scene <sceneId> --cutout` makes the clip as usual, then cuts it out and records it as `s.clipKeyedKey`. If only the cutout fails, run the same command again: it does the cutout without paying for a new clip.
+
+## Layers
+A keyed clip is what makes real depth possible: things can sit behind the subject as well as in front. Build every such scene bottom to top, in this order, and keep to it:
+
+1. **Background**: a mesh, a picture, the user's screenshot, a clip.
+2. **Behind the subject**: the big title, a chart, a logo. The subject will cover part of it, which is the point: a headline that a person stands in front of reads as being in the room.
+3. **The subject**: the keyed clip.
+4. **In front of the subject**: small labels, a lower third, a callout pointing at what the subject holds.
+5. **Captions**, always last.
+
+- Text behind the subject must stay readable with part of it hidden: make it large, keep it to one to three words, and place it so the subject covers the middle or one end, never the first letters of a word.
+- Give the subject a soft contact shadow or a slightly darker patch on the background where it stands, or it floats.
+- Light direction should agree: if the clip is lit from the left, put the brighter side of the background on the left.
+- Keep the subject's size steady across the scene. Move the layers around it, a slow push on the background and a smaller one on the subject, to get depth.
+- Check both preview frames for a green fringe on the edges and for the subject hiding something that must be read.
+
+```tsx
+import { AbsoluteFill } from "remotion";
+import { BgMesh, Captions, KeyedClip, SceneFrame, fonts } from "reelkit/kit";
+
+<SceneFrame from={s.startFrame} durationInFrames={s.durationFrames}>
+  <BgMesh bg="#101014" hero="#C6F135" />
+  <AbsoluteFill style={{ alignItems: "center", paddingTop: "18%", fontFamily: fonts.display, fontWeight: 900, fontSize: 260, color: "#ffffff" }}>LAUNCH</AbsoluteFill>
+  {s.clipKeyedKey ? <KeyedClip src={urls[s.clipKeyedKey]} x={0.5} y={1} scale={0.85} /> : null}
+  <Captions words={s.words} />
+</SceneFrame>
+```
+
 ## Rules
 - Search before generating, and generate once per scene. Do not regenerate to chase small improvements.
 - `--share` only for a generic clip with nothing specific to this user; the user's own files stay private.

package/src/cli.ts

@@ -67,10 +67,13 @@ program.command("whoami").description("Show your account, quota and contribution
 program.command("init [name]").description("Set up a video project (in a new folder named after it, or in this folder)").option("--aspect <aspect>", "9:16, 16:9 or 1:1").action(run(init));
 
 const assets = program.command("assets").description("Find, add and generate the files a video needs");
-assets.command("upload <file>").description("Add one of your own files to this project (private unless --share)")
-  .option("--describe <text>", "what the file shows").option("--footage", "use this video as the footage to overlay")
+assets.command("upload <file>").description("Add one of your own files to this project (private unless --share; with --cutout the video is sent to the Reelkit server to be processed)")
+  .option("--describe <text>", "what the file shows").option("--footage", "use this video as the footage to overlay").option("--green", "the video is a subject on a green background: also make a copy with the green transparent (free, on this machine)")
+  .option("--cutout", "remove any background: the video is sent to the Reelkit server to be processed and a copy with a transparent background comes back (up to 20 seconds, counts against a monthly quota)")
+  .option("--resume <id>", "with --cutout: keep waiting for a cutout already started, without uploading or paying again")
   .option("--share", "also send it to the shared library").option("--kind <kind>", "library kind when sharing").option("--tags <tags>", "comma-separated tags when sharing")
-  .action(run(assetsUpload));
+  // Called with the arguments named: commander adds its own command object last, which must not be taken for the function's test hooks.
+  .action(run((ctx, file: string, opts: Parameters<typeof assetsUpload>[2]) => assetsUpload(ctx, file, opts)));
 assets.command("search <query>").description("Search the shared library by meaning; each result shows how well it fits").option("--kind <kind>", "image, overlay, sfx, music, component or clip").option("--limit <n>", "how many results")
   .action(run(assetsSearch));
 assets.command("pull <id>").description("Download a library item into this project").option("--scene <sceneId>", "use it as this scene's image or clip").option("--force", "replace a component file that already exists")
@@ -82,7 +85,8 @@ const gen = assets.command("gen").description("Generate an asset");
 gen.command("image [prompt]").description("Generate a scene's illustration; the prompt defaults to the plan's")
   .requiredOption("--scene <sceneId>", "the scene it is for").option("--redo", "generate a new image even if the scene has one").action(run(assetsGenImage));
 gen.command("clip [prompt]").description("Generate a scene's video clip (takes minutes); the prompt defaults to the plan's clipPrompt")
-  .requiredOption("--scene <sceneId>", "the scene it is for").option("--green", "film it on green and key the green out into a transparent .webm").option("--seconds <n>", "5 or 10 (default 5)")
+  .requiredOption("--scene <sceneId>", "the scene it is for").option("--green", "film it on green and key the green out into a transparent .webm")
+  .option("--cutout", "make the clip normally, then cut the subject out on the server (the video is sent to the Reelkit server to be processed; counts against a monthly quota)").option("--seconds <n>", "5 or 10 (default 5)")
   .option("--share", "also add it to the shared library (only a generic clip)").option("--redo", "generate a new clip even if the scene has one").option("--resume <id>", "keep waiting for a clip already started, without paying again")
   .action(run((ctx, prompt, opts) => assetsGenClip(ctx, prompt, opts)));
 

package/src/commands/assets.ts

@@ -1,9 +1,9 @@
 import { randomUUID } from "node:crypto";
-import { copyFileSync, mkdirSync, readFileSync } from "node:fs";
+import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
 import { basename, dirname, resolve } from "node:path";
 import { ApiFailure, download, uploadTo } from "../api/client";
 import { client, type Ctx, type Result } from "../context";
-import { LibraryKindSchema, UploadKindSchema } from "../contract";
+import { LibraryKindSchema, MAX_CUTOUT_SECONDS, MAX_UPLOAD_BYTES, UploadKindSchema, type CutoutType } from "../contract";
 import { PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
 import { keyGreen } from "../project/chromakey";
 import { buildManifest, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
@@ -31,9 +31,80 @@ export function setSceneImage(project: Project, sceneId: string, path: string) {
   project.writeJson(FILES.images, { ...project.readJsonOr<Record<string, string>>(FILES.images, {}), [sceneId]: path });
 }
 
+const CUTOUT_POLL_MS = 5_000;
+const CUTOUT_PROGRESS_MS = 30_000;
+const CUTOUT_TIMEOUT_MS = 10 * 60_000;
+const SENT_NOTE = "The video was sent to the Reelkit server to be processed there; the server deletes its copy of it when the job ends.";
+
+// What a test may replace for a cutout: the clock the wait is timed by.
+export type CutoutDeps = { now?: () => number };
+
+type CutoutInput = { source: string; filename: string; contentType: CutoutType; bytes: number; durationSec: number };
+
+// A path as it is typed in a command: quoted only when it has to be.
+const shellArg = (s: string) => (/^[\w./~@%+=:,-]+$/.test(s) ? s : JSON.stringify(s));
+
+// Sends a video to the server and starts its cutout. The seconds are reserved when the job is run, so this is the point where it becomes
+// paid for: nothing is charged by a failure before it, and the id it returns is the only handle on the job after it.
+async function startCutout(ctx: Ctx, input: CutoutInput): Promise<string> {
+  const api = client(ctx);
+  ctx.log(`Sending ${input.filename} to the Reelkit server to cut the subject out.`);
+  const up = await api("cutoutStart", { filename: input.filename.slice(-200), contentType: input.contentType, bytes: input.bytes, durationSec: input.durationSec });
+  await uploadTo(up.uploadUrl, new Uint8Array(readFileSync(input.source)), input.contentType);
+  await api("cutoutRun", { id: up.id });
+  ctx.log(`Started cutout ${up.id}. It usually takes about a minute.`);
+  return up.id;
+}
+
+type CutoutOutcome = { kind: "done" } | { kind: "failed"; message: string } | { kind: "timeout" } | { kind: "lost"; message: string };
+
+// Asks every few seconds until the job ends (at most ten minutes), then saves the result at `dest`. A job that is not found or a login that
+// is wrong has nothing to resume and is thrown; any other trouble may be a hiccup on a job that is paid for, so it is an outcome.
+async function pollCutout(ctx: Ctx, id: string, dest: string, deps: CutoutDeps): Promise<CutoutOutcome> {
+  const api = client(ctx);
+  const now = deps.now ?? Date.now;
+  const started = now();
+  let lastLog = started;
+  try {
+    for (;;) {
+      const st = await api("cutoutStatus", { id });
+      if (st.status === "failed") return { kind: "failed", message: st.message ?? "The server gave no reason." };
+      if (st.status === "done") {
+        await download(st.url!, dest);
+        return { kind: "done" };
+      }
+      const elapsed = now() - started;
+      if (elapsed >= CUTOUT_TIMEOUT_MS) return { kind: "timeout" };
+      if (now() - lastLog >= CUTOUT_PROGRESS_MS) {
+        lastLog = now();
+        ctx.log(`Still cutting out the subject (${Math.round(elapsed / 1000)}s so far)...`);
+      }
+      await ctx.sleep(CUTOUT_POLL_MS);
+    }
+  } catch (e) {
+    if (e instanceof ApiFailure && (e.code === "not_found" || e.code === "unauthenticated")) throw e;
+    return { kind: "lost", message: e instanceof Error ? e.message : String(e) };
+  }
+}
+
+// The one line a person reads when a video cannot be cut out, or undefined when it can.
+function cutoutRefusal(file: string, probe: Awaited<ReturnType<typeof probeFile>>): string | undefined {
+  if (probe.kind !== "video") return `${file} is ${probe.kind}, not a video, so there is nothing to cut out.`;
+  if (!probe.durationSec || probe.durationSec <= 0) return `${file} has no length that could be read, so it cannot be cut out. Export it again.`;
+  if (probe.durationSec > MAX_CUTOUT_SECONDS) return `${file} is ${Math.ceil(probe.durationSec)} seconds long and a cutout takes at most ${MAX_CUTOUT_SECONDS}. Trim it to ${MAX_CUTOUT_SECONDS} seconds or less and run the command again.`;
+  if (probe.bytes > MAX_UPLOAD_BYTES) return `${file} is larger than 200 MB, which is the most a cutout takes. Trim or compress it and run the command again.`;
+  return undefined;
+}
+
 // Registers one of the user's own files. It stays on this machine unless --share is given.
-export async function assetsUpload(ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; share?: boolean; kind?: string; tags?: string }): Promise<Result> {
+export async function assetsUpload(
+  ctx: Ctx, file: string, opts: { describe?: string; footage?: boolean; green?: boolean; cutout?: boolean; resume?: string; share?: boolean; kind?: string; tags?: string },
+  deps: { key?: typeof keyGreen } & CutoutDeps = {},
+): Promise<Result> {
   const project = openProject(ctx.cwd);
+  if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green keys a green background on this machine for free, --cutout removes any background on the server." };
+  if (opts.resume && !opts.cutout) return { ok: false, summary: "--resume goes with --cutout: it keeps waiting for a cutout that was already started." };
+  if (opts.resume) return resumeUploadCutout(ctx, project, file, opts.resume, deps);
   if (opts.share && opts.kind === "component") return { ok: false, summary: "Components cannot be shared from the CLI yet. Share images, overlays, sound effects, music or clips." };
   const source = resolve(ctx.cwd, file);
   let probe: Awaited<ReturnType<typeof probeFile>>;
@@ -43,15 +114,31 @@ export async function assetsUpload(ctx: Ctx, file: string, opts: { describe?: st
     return { ok: false, summary: e instanceof Error ? e.message : String(e) };
   }
   if (opts.footage && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so it cannot be the footage.` };
+  if (opts.green && probe.kind !== "video") return { ok: false, summary: `${file} is ${probe.kind}, not a video, so there is no green screen to key out.` };
+  const refusal = opts.cutout ? cutoutRefusal(file, probe) : undefined;
+  if (refusal) return { ok: false, summary: refusal };
   const kind = opts.share ? UploadKindSchema.safeParse(opts.kind) : undefined;
   if (opts.share && (!kind?.success || !opts.describe)) return { ok: false, summary: "Sharing needs --kind (image, overlay, sfx, music or clip) and --describe \"what it is\"." };
 
   const id = `a-${randomUUID().slice(0, 8)}`;
   const filename = basename(source).replace(/[^A-Za-z0-9._-]/g, "_");
   const key = `assets/user/${id}-${filename}`;
+  // A cutout is made on the server and paid for when it is run, so it is started before anything is registered: if the server refuses
+  // (quota used up, no GPU service) or the upload fails, nothing is left behind.
+  const cutoutId = opts.cutout ? await startCutout(ctx, { source, filename, contentType: probe.contentType as CutoutType, bytes: probe.bytes, durationSec: probe.durationSec! }) : undefined;
   mkdirSync(dirname(project.path(key)), { recursive: true });
   copyFileSync(source, project.path(key));
-  const record: AssetRecord = { id, filename, key, ...probe, ...(opts.describe ? { description: opts.describe } : {}), createdAt: new Date().toISOString() };
+  // A subject filmed on green gets a second copy with the green made transparent, sound kept. Nothing is registered if it cannot be keyed.
+  let keyedKey: string | undefined;
+  if (opts.green) {
+    keyedKey = key.replace(/\.[^./]+$/, "") + ".webm";
+    try { await (deps.key ?? keyGreen)(project.path(key), project.path(keyedKey), { keepAudio: true, requireGreen: true }); }
+    catch (e) {
+      rmSync(project.path(key), { force: true });
+      return { ok: false, summary: e instanceof Error ? e.message : String(e) };
+    }
+  }
+  const record: AssetRecord = { id, filename, key, ...probe, ...(opts.describe ? { description: opts.describe } : {}), ...(keyedKey ? { keyedKey } : {}), ...(cutoutId ? { cutoutId } : {}), createdAt: new Date().toISOString() };
   project.writeJson(FILES.assetIndex, [...project.assets(), record]);
   if (opts.footage) project.writeJson(FILES.config, { ...project.config(), footage: id });
 
@@ -67,12 +154,52 @@ export async function assetsUpload(ctx: Ctx, file: string, opts: { describe?: st
     await uploadTo(up.uploadUrl, new Uint8Array(readFileSync(source)), probe.contentType);
     libraryId = (await api("libraryCommit", { id: up.id })).item.id;
   }
+  if (cutoutId) {
+    const finished = await finishUploadCutout(ctx, project, record, file, deps);
+    return { ...finished, data: { ...(finished.data as object), ...(libraryId ? { libraryId } : {}) }, summary: `Added ${filename} as ${id}${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ""}. ${finished.summary}` };
+  }
   return {
     ok: true, data: { ...record, ...(libraryId ? { libraryId } : {}) },
-    summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.`,
+    summary: `Added ${filename} as ${id} (${probe.kind})${opts.footage ? ", set as the footage" : ""}${libraryId ? `, shared to the library as ${libraryId} (awaiting review)` : ", private to this project"}.${keyedKey ? ` The green is keyed out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"].` : ""}`,
   };
 }
 
+// Waits for the cutout of a registered video and saves it beside the original. The asset stays registered whatever happens.
+async function finishUploadCutout(ctx: Ctx, project: Project, record: AssetRecord, file: string, deps: CutoutDeps): Promise<Result> {
+  const cutoutId = record.cutoutId!;
+  const keyedKey = record.key.replace(/\.[^./]+$/, "") + ".webm";
+  const save = (r: AssetRecord) => project.writeJson(FILES.assetIndex, project.assets().map((a) => (a.id === r.id ? r : a)));
+  const outcome = await pollCutout(ctx, cutoutId, project.path(keyedKey), deps);
+  const resume = `reelkit assets upload ${shellArg(file)} --cutout --resume ${cutoutId}`;
+  const { cutoutId: _pending, ...settled } = record;
+  if (outcome.kind === "done") {
+    const done: AssetRecord = { ...settled, keyedKey };
+    save(done);
+    const seconds = Math.ceil(record.durationSec ?? 0);
+    return { ok: true, data: done, summary: `The subject is cut out into ${keyedKey}: place it with KeyedClip as urls["${keyedKey}"]. It used ${seconds} second${seconds === 1 ? "" : "s"} of your monthly cutout quota. ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "failed") {
+    save(settled);
+    return { ok: false, data: { ...settled, status: "failed" }, summary: `The cutout failed: ${outcome.message} You were not charged. The video stays added without a cutout; try a different video. ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "timeout") {
+    return { ok: false, data: { ...record, status: "pending" }, summary: `The video stays added without a cutout: the cutout (id ${cutoutId}) is not ready after ${CUTOUT_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${resume}\` to keep waiting for it; that does not upload or charge again.` };
+  }
+  return { ok: false, data: { ...record, status: "pending" }, summary: `The video stays added without a cutout: contact was lost while it was being made (${outcome.message}) The cutout (id ${cutoutId}) is not lost: run \`${resume}\`; that does not upload or charge again.` };
+}
+
+// Picks up a cutout that was started earlier: finds the asset waiting for it and only polls.
+async function resumeUploadCutout(ctx: Ctx, project: Project, file: string, id: string, deps: CutoutDeps): Promise<Result> {
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(id)) return { ok: false, summary: `"${id}" is not a cutout id. Copy it from the earlier message.` };
+  const record = project.assets().find((a) => a.cutoutId === id);
+  if (!record) return { ok: false, summary: `No asset in this project is waiting for cutout ${id}. Copy the id from the earlier message.` };
+  const name = basename(file).replace(/[^A-Za-z0-9._-]/g, "_");
+  if (name !== record.filename) return { ok: false, summary: `Cutout ${id} is for ${record.filename}, not ${file}. Give the file it was started with.` };
+  ctx.log(`Waiting for cutout ${id} of ${record.filename}.`);
+  const finished = await finishUploadCutout(ctx, project, record, file, deps);
+  return { ...finished, summary: `Resumed ${record.filename} (${record.id}). ${finished.summary}` };
+}
+
 export async function assetsSearch(ctx: Ctx, query: string, opts: { kind?: string; limit?: string }): Promise<Result> {
   const kind = opts.kind === undefined ? undefined : LibraryKindSchema.safeParse(opts.kind);
   if (kind && !kind.success) return { ok: false, summary: `Unknown kind "${opts.kind}". Use one of: ${LibraryKindSchema.options.join(", ")}.` };
@@ -230,13 +357,14 @@ const CLIP_TIMEOUT_MS = 10 * 60_000;
 // The id is the only handle on a paid job, so every way out after the start says it and how to resume.
 export async function assetsGenClip(
   ctx: Ctx, prompt: string | undefined,
-  opts: { scene: string; green?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string },
+  opts: { scene: string; green?: boolean; cutout?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string },
   deps: ClipDeps = {},
 ): Promise<Result> {
   const project = openProject(ctx.cwd);
   const plan = loadPlan(project);
   const scene = plan.scenes.find((s) => s.id === opts.scene);
   if (!scene || scene.treatment !== "clip" || !scene.clipPrompt) return { ok: false, summary: `Scene ${opts.scene} is not a clip scene, so it does not take a clip. Set its treatment to "clip" and give it a clipPrompt in plan.json.` };
+  if (opts.green && opts.cutout) return { ok: false, summary: "Use --green or --cutout, not both: --green asks for a green background and keys it on this machine for free, --cutout removes any background on the server." };
   const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
   if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
   if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
@@ -251,6 +379,12 @@ export async function assetsGenClip(
   const have = records[scene.id];
   // A scene that already has its clip is not charged for again, whatever prompt is given; only --redo asks for a new one.
   if (have && project.exists(have.key) && !opts.redo && !opts.resume) {
+    if (opts.cutout) {
+      if (have.keyedKey && project.exists(have.keyedKey)) return { ok: true, data: { path: have.key, keyedPath: have.keyedKey, skipped: true }, summary: `Scene ${scene.id} already has a clip at ${have.key} and its cutout at ${have.keyedKey}. Use --redo to generate a new clip.` };
+      // The clip was paid for but its cutout was not made (or is still being made): only the cutout is done, and the clip is not charged again.
+      const cut = await cutoutScene(ctx, project, scene.id, have, now);
+      return { ...cut, data: { path: have.key, ...(cut.ok ? { keyedPath: keyedKey } : {}) }, summary: cut.ok ? `Cut out scene ${scene.id}'s clip. ${cut.summary}` : cut.summary };
+    }
     if (opts.green && have.greenScreen && !have.keyedKey) {
       // The clip was paid for but not keyed (ffmpeg failed or --green came later): keying is free, so do it now.
       const keyed = await keyScene(project, scene.id, have, key);
@@ -271,7 +405,7 @@ export async function assetsGenClip(
     id = job.id;
     ctx.log(`Started clip ${id} for scene ${scene.id}. It usually takes a few minutes.`);
   } else ctx.log(`Waiting for clip ${id} for scene ${scene.id}.`);
-  const resume = `reelkit assets gen clip --resume ${id} --scene ${scene.id}`;
+  const resume = `reelkit assets gen clip --resume ${id} --scene ${scene.id}${opts.cutout ? " --cutout" : ""}`;
 
   try {
     let lastLog = started;
@@ -293,11 +427,20 @@ export async function assetsGenClip(
           if (!keyed.ok) return keyed;
           keyedPath = keyedKey;
         }
+        let cutoutNote: string | undefined;
+        if (opts.cutout) {
+          // The paid clip is already saved above; if the cutout fails, running the command again does only the cutout.
+          const cut = await cutoutScene(ctx, project, scene.id, record, now);
+          const shared = st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : "";
+          if (!cut.ok) return { ...cut, data: { id, path, durationSec: record.durationSec, libraryId: st.libraryId ?? null }, summary: `Generated the clip for scene ${scene.id}${shared}. ${cut.summary}` };
+          keyedPath = keyedKey;
+          cutoutNote = cut.summary;
+        }
         const { note } = tryManifest(project, plan);
         return {
           ok: true, data: { id, path, durationSec: record.durationSec, ...(keyedPath ? { keyedPath } : {}), libraryId: st.libraryId ?? null },
           summary: [
-            `Generated the clip for scene ${scene.id} at ${path}${keyedPath ? `, keyed to ${keyedPath}` : ""}${st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : ""}.`,
+            `Generated the clip for scene ${scene.id} at ${path}${keyedPath && !cutoutNote ? `, keyed to ${keyedPath}` : ""}${st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : ""}.${cutoutNote ? ` ${cutoutNote}` : ""}`,
             ...(note ? [note] : []),
           ].join("\n"),
         };
@@ -319,6 +462,46 @@ export async function assetsGenClip(
   }
 }
 
+// Cuts the subject out of a scene's saved clip on the server and records the transparent copy as the clip's keyed copy. The id of a job
+// that is still being made is kept in the record, so running the command again waits for it instead of paying again. The clip stays
+// whatever happens here, and every way out says so and how to try the cutout again.
+async function cutoutScene(ctx: Ctx, project: Project, sceneId: string, record: ClipRecord, now: () => number): Promise<Result> {
+  const keyedKey = `assets/clip-${sceneId}.webm`;
+  const retry = `reelkit assets gen clip --scene ${sceneId} --cutout`;
+  const saved = `The clip is saved at ${record.key}`;
+  let id = record.cutoutId;
+  if (!id) {
+    try {
+      id = await startCutout(ctx, { source: project.path(record.key), filename: basename(record.key), contentType: "video/mp4", bytes: statSync(project.path(record.key)).size, durationSec: record.durationSec });
+    } catch (e) {
+      return { ok: false, summary: `${saved}, but the cutout could not be started: ${e instanceof Error ? e.message : String(e)} Run \`${retry}\` to try the cutout again; the clip is not charged again.` };
+    }
+    setSceneClip(project, sceneId, { ...record, cutoutId: id });
+  } else ctx.log(`Waiting for cutout ${id} of scene ${sceneId}'s clip.`);
+  const { cutoutId: _pending, ...settled } = record;
+  let outcome: CutoutOutcome;
+  try {
+    outcome = await pollCutout(ctx, id, project.path(keyedKey), { now });
+  } catch (e) {
+    if (!(e instanceof ApiFailure && e.code === "not_found")) throw e;
+    setSceneClip(project, sceneId, settled);
+    return { ok: false, summary: `${saved}, but the server no longer has cutout ${id}, so it was forgotten. Run \`${retry}\` to start a new one; the clip is not charged again.` };
+  }
+  if (outcome.kind === "done") {
+    setSceneClip(project, sceneId, { ...settled, keyedKey });
+    const seconds = Math.ceil(record.durationSec);
+    return { ok: true, summary: `The subject is cut out into ${keyedKey} (${seconds} second${seconds === 1 ? "" : "s"} of your monthly cutout quota). ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "failed") {
+    setSceneClip(project, sceneId, settled);
+    return { ok: false, summary: `${saved}, but the cutout failed: ${outcome.message} You were not charged for the cutout. Run \`${retry}\` to try the cutout again; the clip is not charged again. ${SENT_NOTE}` };
+  }
+  if (outcome.kind === "timeout") {
+    return { ok: false, summary: `${saved}, but the cutout (id ${id}) is not ready after ${CUTOUT_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${retry}\` to keep waiting for it; that does not upload or charge again. ${SENT_NOTE}` };
+  }
+  return { ok: false, summary: `${saved}, but contact was lost while the cutout (id ${id}) was being made: ${outcome.message} It is not lost: run \`${retry}\`; that does not upload or charge again.` };
+}
+
 // Keys a scene's saved clip and records the keyed copy. A failure leaves the record as it was.
 async function keyScene(project: Project, sceneId: string, record: ClipRecord, key: NonNullable<ClipDeps["key"]>): Promise<Result> {
   const keyedKey = `assets/clip-${sceneId}.webm`;

package/src/commands/auth.ts

@@ -113,6 +113,6 @@ export async function whoami(ctx: Ctx): Promise<Result> {
   const q = me.quota;
   return {
     ok: true, data: me,
-    summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nClips: ${q.clips.used}/${q.clips.limit}\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
+    summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nClips: ${q.clips.used}/${q.clips.limit}\nCutouts: ${q.cutoutSeconds.used}/${q.cutoutSeconds.limit} seconds\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
   };
 }

package/src/contract/index.ts

@@ -53,6 +53,20 @@
 // - A green-screen clip (`greenScreen: true`) is a subject filmed on a flat pure green background, meant to be keyed out by the client. The server
 //   stores and returns the original, green and all; it does not remove the background. A shareable clip also becomes a library item of kind
 //   `clip` in review, whose `meta` may carry `greenScreen: true` and `durationSec`, and its id is returned as `libraryId`.
+// - A cutout removes the background from a video, whatever it is, and answers with the subject on a transparent background. The video is sent to
+//   the server, a GPU model separates the subject, and a transparent `.webm` (VP9 with alpha, sound kept) comes back. It takes about a minute.
+//   It is a job with three calls: `cutoutStart` checks the request and answers with an id and an `uploadUrl` (charging nothing; the client sends
+//   the file there exactly as for a library upload); `cutoutRun` checks that the file arrived (else 400 `invalid_request`), reserves
+//   `ceil(durationSec)` seconds from the caller's monthly cutout quota and starts the job; the client then asks `cutoutStatus` until it is `done`
+//   or `failed`, with the same exactly-when rules as `clipStatus` (`url`, `ext` and `contentType` when done, `message` when failed).
+//   `cutoutStatus` for a job whose `cutoutRun` has not been called is 400 `invalid_request`.
+//   `cutoutRun` for a job already started answers its current pending state and does not charge again. A failed cutout is not charged.
+//   A cutout id belongs to its caller: another user's id, or an unknown one, is `not_found` for run and status.
+// - The video uploaded for a cutout is used only to make the cutout, and is deleted from the server when the job ends. The result stays in
+//   the user's private files and is never shared. A cutout is charged in whole seconds of video (`ceil(durationSec)`), and a video longer than
+//   20 seconds is refused by the request schema.
+// - When the cutout quota is used up, `cutoutRun` is 429 `quota_exceeded` with the reset date. When the server has no GPU service configured,
+//   `cutoutStart` and `cutoutRun` are 400 `invalid_request` with the message "Background removal is not available on this server yet."
 // - `durationSec` is the decoded audio length; `words` are in seconds from the start of that audio.
 // - Search returns published items only, and leaves out items that are not matches at all; the default `limit` is 8.
 import { z } from "zod";
@@ -82,6 +96,11 @@ const Meta = z.record(z.string(), z.unknown()).refine((m) => new TextEncoder().e
 
 // The most a single upload may be: 200 MB.
 export const MAX_UPLOAD_BYTES = 209_715_200;
+// The video types a cutout takes.
+export const CutoutTypeSchema = z.enum(["video/mp4", "video/quicktime", "video/webm"]);
+export type CutoutType = z.infer<typeof CutoutTypeSchema>;
+// The longest video a cutout takes, in seconds.
+export const MAX_CUTOUT_SECONDS = 20;
 
 export const ErrorCodeSchema = z.enum(["unauthenticated", "quota_exceeded", "not_found", "invalid_request", "server_error"]);
 export type ErrorCode = z.infer<typeof ErrorCodeSchema>;
@@ -117,7 +136,7 @@ export type Voice = z.infer<typeof VoiceSchema>;
 const Meter = z.object({ used: z.number(), limit: z.number() });
 export const MeSchema = z.object({
   userId: z.string(), handle: z.string(),
-  quota: z.object({ voiceoverChars: Meter, images: Meter, clips: Meter, resetsAt: z.string() }),
+  quota: z.object({ voiceoverChars: Meter, images: Meter, clips: Meter, cutoutSeconds: Meter, resetsAt: z.string() }),
   // How many of the user's own items are published in the shared library. An item still in review, or sent back, is not counted.
   contributions: z.number(),
 });
@@ -174,6 +193,20 @@ export const routes = {
       greenScreen: z.boolean().optional(), libraryId: z.string().optional(),
     }).refine((r) => (r.status === "done") === (r.url !== undefined && r.ext !== undefined && r.contentType !== undefined) && (r.status === "failed") === (r.message !== undefined),
       "url, ext and contentType belong to done and message to failed")),
+  cutoutStart: route("POST", "/cutouts", true,
+    z.object({
+      filename: text(200), contentType: CutoutTypeSchema,
+      bytes: z.number().int().min(1).max(MAX_UPLOAD_BYTES), durationSec: z.number().positive().max(MAX_CUTOUT_SECONDS),
+    }),
+    z.object({ id: z.string(), uploadUrl: z.string() })),
+  cutoutRun: route("POST", "/cutouts/run", true, z.object({ id: ItemId }), z.object({ id: z.string(), status: z.literal("pending") })),
+  cutoutStatus: route("POST", "/cutouts/status", true,
+    z.object({ id: ItemId }),
+    z.object({
+      id: z.string(), status: z.enum(["pending", "done", "failed"]), message: z.string().optional(),
+      url: z.string().optional(), ext: z.enum(["webm"]).optional(), contentType: z.literal("video/webm").optional(),
+    }).refine((r) => (r.status === "done") === (r.url !== undefined && r.ext !== undefined && r.contentType !== undefined) && (r.status === "failed") === (r.message !== undefined),
+      "url, ext and contentType belong to done and message to failed")),
   publicLibrary: route("GET", "/public/library", false,
     z.object({ q: text(200).optional(), kind: LibraryKindSchema.optional(), page: z.coerce.number().int().min(1).max(10000).optional() }),
     // With `q` the items are the best matches by meaning, best first, each with `match` (0..1), in one page (`hasMore` false). Under heavy

package/src/credentials.ts

@@ -3,7 +3,7 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 
 type Env = Record<string, string | undefined>;
-export const DEFAULT_API_URL = "https://reelkit-kohl.vercel.app/api/v1";
+export const DEFAULT_API_URL = "https://reelkit.cc/api/v1";
 
 export const configDir = (env: Env) => env.REELKIT_CONFIG_DIR || join(homedir(), ".config", "reelkit");
 export const credentialsPath = (env: Env) => join(configDir(env), "credentials.json");

package/src/pipeline/schema.ts

@@ -46,6 +46,10 @@ export const AssetRecordSchema = z.object({
   durationSec: z.number().optional(),
   // What the file shows, as given at upload, so the plan can place it where it is relevant.
   description: z.string().optional(),
+  // A green-screen video also has a copy with the green made transparent, at this path.
+  keyedKey: z.string().optional(),
+  // The server cutout still being made for this video. Kept so that coming back to wait for it neither uploads nor charges again.
+  cutoutId: z.string().optional(),
   createdAt: z.string(),
 });
 export type AssetRecord = z.infer<typeof AssetRecordSchema>;

package/src/project/chromakey.ts

@@ -1,16 +1,50 @@
 import { execFile } from "node:child_process";
+import { rmSync } from "node:fs";
 import { promisify } from "node:util";
 
 const run = promisify(execFile);
+const PURE_GREEN = "0x00FF00";
 
-// Turns the flat pure green of a green-screen clip into transparency: VP9 with an alpha channel, which Remotion plays with `transparent`.
-// The key is on pure green with a little give for compression noise; the despill pulls the green glow off the subject's edges.
-// The original is left alone.
-export async function keyGreen(input: string, output: string): Promise<void> {
-  const filter = "chromakey=color=0x00FF00:similarity=0.2:blend=0.1,despill=type=green:mix=0.5:expand=0,format=yuva420p";
+export type KeyOptions = {
+  // Keep the clip's sound in the transparent copy (a person talking on green). Generated clips have none worth keeping.
+  keepAudio?: boolean;
+  // Fail when the frame's corners are not green, instead of keying on pure green anyway. For footage a person supplied.
+  requireGreen?: boolean;
+};
+
+const NOT_GREEN = "This video does not have a green background in its corners, so there is nothing to key out. Film the subject in front of an evenly lit green background that fills the frame.";
+
+// The colour to key on, read from the first frame's four corners. A real green screen, filmed or generated, is never exactly
+// #00FF00: it is whatever green the light and the encoder made of it, and keying on the wrong green leaves a haze. Undefined when
+// the corners are not green.
+async function cornerGreen(input: string): Promise<string | undefined> {
+  const corner = (x: string, y: string) => `crop=24:24:${x}:${y},scale=1:1:flags=area`;
+  const picks = ["0:0", "iw-24:0", "0:ih-24", "iw-24:ih-24"].map((c) => { const [x, y] = c.split(":") as [string, string]; return corner(x, y); });
+  const colours: [number, number, number][] = [];
+  for (const vf of picks) {
+    const { stdout } = await run("ffmpeg", ["-v", "error", "-i", input, "-frames:v", "1", "-vf", vf, "-pix_fmt", "rgb24", "-f", "rawvideo", "-"], { encoding: "buffer", maxBuffer: 1 << 16 });
+    if (stdout.length >= 3) colours.push([stdout[0]!, stdout[1]!, stdout[2]!]);
+  }
+  const green = colours.filter(([r, g, b]) => g > 60 && g > r * 1.3 && g > b * 1.3);
+  // Most corners must agree: a subject may reach one corner, but not three.
+  if (green.length < 3) return undefined;
+  const avg = (i: 0 | 1 | 2) => Math.round(green.reduce((sum, c) => sum + c[i], 0) / green.length);
+  return `0x${[avg(0), avg(1), avg(2)].map((n) => n.toString(16).padStart(2, "0")).join("")}`;
+}
+
+// Turns the green of a green-screen clip into transparency: VP9 with an alpha channel, which Remotion plays with `transparent`.
+// The key is on the green found in the frame's corners, with some give for noise and uneven light; the despill pulls the green glow
+// off the subject's edges. The original is left alone.
+export async function keyGreen(input: string, output: string, opts: KeyOptions = {}): Promise<void> {
   try {
-    await run("ffmpeg", ["-v", "error", "-y", "-i", input, "-vf", filter, "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-b:v", "0", "-crf", "30", "-an", output]);
+    const found = await cornerGreen(input);
+    if (!found && opts.requireGreen) throw new Error(NOT_GREEN);
+    const filter = `chromakey=color=${found ?? PURE_GREEN}:similarity=0.16:blend=0.08,despill=type=green:mix=0.5:expand=0,format=yuva420p`;
+    const audio = opts.keepAudio ? ["-c:a", "libopus", "-b:a", "96k"] : ["-an"];
+    await run("ffmpeg", ["-v", "error", "-y", "-i", input, "-vf", filter, "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-b:v", "0", "-crf", "30", ...audio, output]);
   } catch (e) {
+    rmSync(output, { force: true });
+    if (e instanceof Error && e.message === NOT_GREEN) throw e;
     if ((e as NodeJS.ErrnoException)?.code === "ENOENT") throw new Error("ffmpeg was not found. Install ffmpeg (macOS: `brew install ffmpeg`) and run the command again.");
     throw new Error("ffmpeg could not key the green out of the clip. The original clip is saved; run the command again to retry.");
   }

package/src/project/manifest.ts

@@ -5,8 +5,9 @@ import { FILES, type Project } from "./project";
 // What a scene was recorded from is stored with it, so a changed script or voice is noticed.
 export type Voiceover = { key: string; durationSec: number; words: WordTiming[]; text?: string; voiceId?: string; speed?: number };
 
-// A scene's video clip: the original, and the transparent copy when it was filmed on green and keyed out. Stored in assets/clips.json by scene id.
-export type ClipRecord = { key: string; greenScreen: boolean; durationSec: number; keyedKey?: string };
+// A scene's video clip: the original, and the transparent copy when it was filmed on green and keyed out, or cut out on the server (`cutoutId` is
+// that cutout while it is still being made). Stored in assets/clips.json by scene id.
+export type ClipRecord = { key: string; greenScreen: boolean; durationSec: number; keyedKey?: string; cutoutId?: string };
 
 // True when the stored recording no longer matches what the plan asks for. An entry from before this was tracked counts as different.
 export function voiceoverStale(vo: Voiceover, scene: ScenePlan["scenes"][number], plan: ScenePlan): boolean {

package/src/testing/conformance.ts

@@ -20,8 +20,11 @@ export type Harness = {
   clipFailPrompt?: string;
   // How long to wait between two asks for a clip's status, in milliseconds: a server whose fake provider needs a moment sets it. Default 20.
   clipPollMs?: number;
+  // Optional, for the cutout tests. A file name that makes a cutout job fail (the fake fails any name containing "cutout-fail"); without
+  // it the failing-job case is skipped. The cutout tests wait between status asks for `clipPollMs` too.
+  cutoutFailFilename?: string;
 };
-export type StartOptions = { voiceoverChars?: number; images?: number; clips?: number };
+export type StartOptions = { voiceoverChars?: number; images?: number; clips?: number; cutoutSeconds?: number };
 
 const PNG = new Uint8Array(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"));
 const DATE = /\d{4}-\d{2}-\d{2}/;
@@ -82,6 +85,8 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
         expect(me.quota.images.used).toBe(0);
         expect(me.quota.clips.used).toBe(0);
         expect(me.quota.clips.limit).toBeGreaterThan(0);
+        expect(me.quota.cutoutSeconds.used).toBe(0);
+        expect(me.quota.cutoutSeconds.limit).toBeGreaterThan(0);
         expect(me.quota.voiceoverChars.limit).toBeGreaterThan(0);
         expect(me.contributions).toBe(0);
         const resets = new Date(me.quota.resetsAt);
@@ -398,6 +403,125 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
       });
     });
 
+    describe("cutouts", () => {
+      const VIDEO = new Uint8Array([0, 0, 0, 24, 102, 116, 121, 112, 105, 115, 111, 109, ...new Array(2000).fill(7)]);
+      const meta = (over: Record<string, unknown> = {}) => ({ filename: "me.mp4", contentType: "video/mp4" as const, bytes: VIDEO.length, durationSec: 2.4, ...over });
+      const settleCutout = async (api: Api, id: string) => {
+        for (let i = 0; i < 20; i++) {
+          const st = await api("cutoutStatus", { id });
+          if (st.status !== "pending") return st;
+          await new Promise((r) => setTimeout(r, h.clipPollMs ?? 20));
+        }
+        throw new Error(`cutout ${id} was still pending after 20 asks`);
+      };
+      // Starts a job and sends its file, as a client does before asking for the run.
+      const uploaded = async (api: Api, over: Record<string, unknown> = {}) => {
+        const started = await api("cutoutStart", meta(over) as never);
+        await put(started.uploadUrl, VIDEO, (over.contentType as string | undefined) ?? "video/mp4");
+        return started;
+      };
+
+      it("start, upload, run: pending at once, then done with a transparent webm that downloads; charged in whole seconds to this user only", async () => {
+        const api = await as("user-cut000001"), other = await as("user-cut000002");
+        const started = await api("cutoutStart", meta());
+        expect(started.id).toBeTruthy();
+        expect(started.uploadUrl).toBeTruthy();
+        // Starting charges nothing.
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(0);
+        expect((await put(started.uploadUrl, VIDEO, "video/mp4")).ok).toBe(true);
+        expect(await api("cutoutRun", { id: started.id })).toEqual({ id: started.id, status: "pending" });
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(3);
+        const done = await settleCutout(api, started.id);
+        expect(done).toMatchObject({ id: started.id, status: "done", ext: "webm", contentType: "video/webm" });
+        expect(done.url).toBeTruthy();
+        expect(done.message).toBeUndefined();
+        await download(done.url!, join(dest, "cutout.webm"));
+        expect(readFileSync(join(dest, "cutout.webm")).length).toBeGreaterThan(0);
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(3);
+        expect((await other("me", {})).quota.cutoutSeconds.used).toBe(0);
+      });
+
+      it("a run before the file arrived is invalid_request and charges nothing", async () => {
+        const api = await as("user-cut000003");
+        const started = await api("cutoutStart", meta());
+        const e = await failure(api("cutoutRun", { id: started.id }));
+        expect(e.code).toBe("invalid_request");
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(0);
+        // The upload can still be sent and the job run afterwards.
+        await put(started.uploadUrl, VIDEO, "video/mp4");
+        expect((await api("cutoutRun", { id: started.id })).status).toBe("pending");
+      });
+
+      it("running a job twice answers its pending state again and charges once", async () => {
+        const api = await as("user-cut000004");
+        const started = await uploaded(api);
+        await api("cutoutRun", { id: started.id });
+        expect(await api("cutoutRun", { id: started.id })).toEqual({ id: started.id, status: "pending" });
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(3);
+        expect((await settleCutout(api, started.id)).status).toBe("done");
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(3);
+      });
+
+      it("a cutout id belongs to its caller: another user, or an unknown id, is not_found for run and status", async () => {
+        const api = await as("user-cut000005"), other = await as("user-cut000006");
+        const started = await uploaded(api);
+        await api("cutoutRun", { id: started.id });
+        expect((await failure(other("cutoutRun", { id: started.id }))).code).toBe("not_found");
+        expect((await failure(other("cutoutStatus", { id: started.id }))).code).toBe("not_found");
+        expect((await failure(api("cutoutRun", { id: "cutout-nosuchjob" }))).code).toBe("not_found");
+        expect((await failure(api("cutoutStatus", { id: "cutout-nosuchjob" }))).code).toBe("not_found");
+        expect((await other("me", {})).quota.cutoutSeconds.used).toBe(0);
+      });
+
+      it("a failed job says why in one line and is not charged", async () => {
+        if (!h.cutoutFailFilename) return;
+        const api = await as("user-cut000007");
+        const started = await uploaded(api, { filename: h.cutoutFailFilename });
+        await api("cutoutRun", { id: started.id });
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(3);
+        const st = await settleCutout(api, started.id);
+        expect(st.status).toBe("failed");
+        expect(st.message?.length).toBeGreaterThan(0);
+        expect(st.message).not.toContain("\n");
+        expect(st.url).toBeUndefined();
+        expect((await api("me", {})).quota.cutoutSeconds.used).toBe(0);
+      });
+
+      it("at the limit a run is quota_exceeded with the reset date, charged nothing; other users are unaffected", async () => {
+        const t = await start({ cutoutSeconds: 3 });
+        try {
+          const api = createClient({ baseUrl: t.baseUrl, token: await t.login("user-cutq00001") });
+          const other = createClient({ baseUrl: t.baseUrl, token: await t.login("user-cutq00002") });
+          const send = async (c: Api) => { const s = await c("cutoutStart", meta({ durationSec: 2 })); await put(s.uploadUrl, VIDEO, "video/mp4"); return s; };
+          const first = await send(api);
+          await api("cutoutRun", { id: first.id });
+          const second = await send(api);
+          const e = await failure(api("cutoutRun", { id: second.id }));
+          expect(e.code).toBe("quota_exceeded");
+          expect(e.message).toMatch(DATE);
+          const me = await api("me", {});
+          expect(me.quota.cutoutSeconds).toEqual({ used: 2, limit: 3 });
+          expect(e.message).toContain(me.quota.resetsAt.slice(0, 10));
+          const third = await send(other);
+          expect((await other("cutoutRun", { id: third.id })).status).toBe("pending");
+        } finally { await t.close(); }
+      });
+
+      it("refuses a video over 20 seconds, a type that is not a video, a size out of range and a missing login", async () => {
+        const api = await as("user-cut000008");
+        for (const bad of [{ durationSec: 20.5 }, { durationSec: 0 }, { durationSec: -1 }, { contentType: "audio/mpeg" }, { contentType: "image/png" }, { bytes: 0 }, { bytes: MAX_UPLOAD_BYTES + 1 }, { filename: "a\u0000b.mp4" }]) {
+          expect((await failure(api("cutoutStart", meta(bad) as never))).code, JSON.stringify(bad)).toBe("invalid_request");
+        }
+        for (const ok of [{ durationSec: 20 }, { contentType: "video/quicktime" as const }, { contentType: "video/webm" as const }, { bytes: MAX_UPLOAD_BYTES }]) {
+          expect((await api("cutoutStart", meta(ok))).id).toBeTruthy();
+        }
+        expect((await failure(api("cutoutRun", { id: "../x" }))).code).toBe("invalid_request");
+        expect((await failure(anon("cutoutStart", meta()))).code).toBe("unauthenticated");
+        expect((await failure(anon("cutoutRun", { id: "cutout-x" }))).code).toBe("unauthenticated");
+        expect((await failure(anon("cutoutStatus", { id: "cutout-x" }))).code).toBe("unauthenticated");
+      });
+    });
+
     describe("quota", () => {
       it("passes exactly at the limit; one over is quota_exceeded with the reset date; other users are unaffected", async () => {
         const t = await start({ voiceoverChars: 10, images: 1 });

package/src/testing/fake-api.ts

@@ -62,10 +62,29 @@ function makeClip(greenScreen: boolean): Blob {
   } finally { rmSync(dir, { recursive: true, force: true }); }
 }
 
+// A cutout job: the file arrives at the upload URL, `run` reserves the seconds, and then it is pending for its first status call and done for
+// the second (or failed, when the file name says so). `seconds` is what it holds of the quota, 0 once given back.
+type CutoutJob = { owner: string; filename: string; contentType: string; bytes: number; durationSec: number; file?: Uint8Array; started: boolean; asks: number; seconds: number };
+const CUTOUT_FAIL = "cutout-fail";
+const NO_GPU = "Background removal is not available on this server yet.";
+
+// A real one-second WebM, made with ffmpeg: VP9 with an alpha channel, a transparent background and an opaque box.
+function makeCutout(): Blob {
+  const dir = mkdtempSync(join(tmpdir(), "rk-fakecut-"));
+  const out = join(dir, "cutout.webm");
+  try {
+    execFileSync("ffmpeg", ["-v", "error", "-f", "lavfi", "-i", "color=c=black@0:s=160x90:r=10:d=1,format=yuva420p,drawbox=x=50:y=20:w=60:h=50:color=red@1:t=fill:replace=1",
+      "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-b:v", "0", "-crf", "35", out], { stdio: "pipe" });
+    return { filename: "cutout.webm", contentType: "video/webm", bytes: new Uint8Array(readFileSync(out)) };
+  } catch (e) {
+    throw new Error(`The fake API could not make its test cutout: ffmpeg is needed (${e instanceof Error ? e.message : String(e)}).`);
+  } finally { rmSync(dir, { recursive: true, force: true }); }
+}
+
 export type FakeApi = Awaited<ReturnType<typeof startFakeApi>>;
 
 // An in-memory stand-in for the Reelkit API. It implements every route in the contract.
-export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLimit?: number; clipLimit?: number; noClipProvider?: boolean; autoApprove?: boolean } = {}) {
+export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLimit?: number; clipLimit?: number; noClipProvider?: boolean; cutoutSecondsLimit?: number; noCutoutService?: boolean; autoApprove?: boolean } = {}) {
   // token -> the user it belongs to
   const tokens = new Map<string, string>();
   const devices = new Map<string, { userCode: string; approved: boolean; userId: string }>();
@@ -74,18 +93,23 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
   const blobs = new Map<string, Blob>();
   // An unguessable token -> the one upload it accepts. `used` is set by the first PUT, accepted or not: the fake is stricter than a
   // real server needs to be, so a client that retries a PUT, or sends a second one, fails here instead of working by luck.
-  const uploads = new Map<string, { id: string; contentType: string; bytes: number; used: boolean }>();
+  // `sink` is where the accepted bytes go: a library item, or a cutout job.
+  const uploads = new Map<string, { contentType: string; bytes: number; used: boolean; sink: { has(): boolean; put(bytes: Uint8Array): void } | undefined }>();
   // Totals over every user (for tests of the CLI), and the same counts per user (what a quota is measured against).
-  const usage = { chars: 0, images: 0, clips: 0, pulls: 0, uploads: 0 };
-  const meters = new Map<string, { chars: number; images: number; clips: number; uploads: number }>();
+  const usage = { chars: 0, images: 0, clips: 0, cutoutSeconds: 0, pulls: 0, uploads: 0 };
+  const meters = new Map<string, { chars: number; images: number; clips: number; cutoutSeconds: number; uploads: number }>();
   const meter = (userId: string | undefined) => {
     const id = userId ?? "u-test";
-    if (!meters.has(id)) meters.set(id, { chars: 0, images: 0, clips: 0, uploads: 0 });
+    if (!meters.has(id)) meters.set(id, { chars: 0, images: 0, clips: 0, cutoutSeconds: 0, uploads: 0 });
     return meters.get(id)!;
   };
   const resetDate = () => nextMonthStart().toISOString();
-  const limits = { chars: opts.voiceoverCharLimit ?? 10_000, images: opts.imageLimit ?? 30, clips: opts.clipLimit ?? 5 };
+  const limits = { chars: opts.voiceoverCharLimit ?? 10_000, images: opts.imageLimit ?? 30, clips: opts.clipLimit ?? 5, cutoutSeconds: opts.cutoutSecondsLimit ?? 120 };
   const jobs = new Map<string, ClipJob>();
+  const cutouts = new Map<string, CutoutJob>();
+  let cutoutFile: Blob | undefined;
+  // While held, no cutout job finishes: every status call says pending. For a test that sees a client give up waiting and come back.
+  let cutoutsHeld = false;
   // Made on first use and kept for the life of this instance.
   const clipFiles: Partial<Record<"plain" | "green", Blob>> = {};
   const clipFile = (green: boolean) => (clipFiles[green ? "green" : "plain"] ??= makeClip(green));
@@ -117,7 +141,7 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
       const m = meter(userId);
       return {
         userId: userId ?? "u-test", handle: handleOf(userId ?? "u-test"),
-        quota: { voiceoverChars: { used: m.chars, limit: limits.chars }, images: { used: m.images, limit: limits.images }, clips: { used: m.clips, limit: limits.clips }, resetsAt: resetDate() },
+        quota: { voiceoverChars: { used: m.chars, limit: limits.chars }, images: { used: m.images, limit: limits.images }, clips: { used: m.clips, limit: limits.clips }, cutoutSeconds: { used: m.cutoutSeconds, limit: limits.cutoutSeconds }, resetsAt: resetDate() },
         // What the user gave the shared library that was accepted: their own items that are published. One waiting for review does not count.
         contributions: [...store.values()].filter((x) => x.owner === userId && x.committed && x.item.visibility === "published").length,
       };
@@ -151,7 +175,10 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
       const item: LibraryItem = { id, kind: input.kind, title: input.title, description: input.description, tags: input.tags, meta: input.meta, visibility: input.shareable ? "review" : "private" };
       store.set(id, { item, filename, contentType: input.contentType, owner: userId, committed: false });
       const token = secret();
-      uploads.set(token, { id, contentType: input.contentType, bytes: input.bytes, used: false });
+      uploads.set(token, {
+        contentType: input.contentType, bytes: input.bytes, used: false,
+        sink: { has: () => Boolean(store.get(id)?.file), put: (bytes) => { const s = store.get(id)!; s.file = { filename: s.filename, contentType: s.contentType, bytes }; } },
+      });
       return { id, uploadUrl: `${origin}/upload/${token}` };
     },
     libraryCommit: ({ id }, { userId }) => {
@@ -215,6 +242,45 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
         ...(job.libraryId ? { libraryId: job.libraryId } : {}),
       };
     },
+    cutoutStart: (input, { userId }) => {
+      if (opts.noCutoutService) throw new Fail(400, "invalid_request", NO_GPU);
+      const id = nextId("cutout");
+      const job: CutoutJob = { owner: userId ?? "u-test", filename: input.filename, contentType: input.contentType, bytes: input.bytes, durationSec: input.durationSec, started: false, asks: 0, seconds: 0 };
+      cutouts.set(id, job);
+      const token = secret();
+      uploads.set(token, { contentType: input.contentType, bytes: input.bytes, used: false, sink: { has: () => Boolean(job.file), put: (bytes) => { job.file = bytes; } } });
+      return { id, uploadUrl: `${origin}/upload/${token}` };
+    },
+    cutoutRun: ({ id }, { userId }) => {
+      const job = cutouts.get(id);
+      if (!job || job.owner !== (userId ?? "u-test")) throw new Fail(404, "not_found", `No cutout with id ${id}.`);
+      // A job already started just answers where it is, without a second charge.
+      if (job.started) return { id, status: "pending" };
+      if (opts.noCutoutService) throw new Fail(400, "invalid_request", NO_GPU);
+      if (!job.file) throw new Fail(400, "invalid_request", `Nothing was uploaded for ${id}. Send the video to the upload URL first.`);
+      const seconds = Math.ceil(job.durationSec);
+      const m = meter(userId);
+      if (m.cutoutSeconds + seconds > limits.cutoutSeconds) throw new Fail(429, "quota_exceeded", `Cutout quota used up. It resets on ${resetDate().slice(0, 10)}.`);
+      m.cutoutSeconds += seconds;
+      usage.cutoutSeconds += seconds;
+      job.started = true;
+      job.seconds = seconds;
+      return { id, status: "pending" };
+    },
+    cutoutStatus: ({ id }, { userId }) => {
+      const job = cutouts.get(id);
+      if (!job || job.owner !== (userId ?? "u-test")) throw new Fail(404, "not_found", `No cutout with id ${id}.`);
+      if (!job.started) throw new Fail(400, "invalid_request", `Cutout ${id} has not been run yet.`);
+      if (++job.asks === 1 || cutoutsHeld) return { id, status: "pending" };
+      // The uploaded video is deleted when the job ends, whatever the result.
+      job.file = undefined;
+      if (job.filename.includes(CUTOUT_FAIL)) {
+        // A failed cutout is not charged: the seconds are given back, once.
+        if (job.seconds) { meter(userId).cutoutSeconds -= job.seconds; usage.cutoutSeconds -= job.seconds; job.seconds = 0; }
+        return { id, status: "failed", message: "The video could not be processed. Try a different video." };
+      }
+      return { id, status: "done", url: fileUrl((cutoutFile ??= makeCutout())), ext: "webm", contentType: "video/webm" };
+    },
     publicLibrary: ({ q, kind, page }) => {
       // Published items only. Newest first; with a query, best match first and equal matches newest first.
       const want = q ? words(q) : undefined;
@@ -245,15 +311,14 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
       }
       if (req.method === "PUT" && url.pathname.startsWith("/upload/")) {
         const up = uploads.get(url.pathname.slice("/upload/".length));
-        const s = up && store.get(up.id);
-        if (!up || !s) return send(404, { error: { code: "not_found", message: "No such upload." } });
+        if (!up?.sink) return send(404, { error: { code: "not_found", message: "No such upload." } });
         // The URL is good for one PUT. Whether that PUT is accepted or refused, a second one is dead, and so is any PUT after commit.
         // (A real server may be kinder; the contract promises clients no more than this.)
-        if (up.used || s.file) throw new Fail(400, "invalid_request", "This upload URL has already been used.");
+        if (up.used || up.sink.has()) throw new Fail(400, "invalid_request", "This upload URL has already been used.");
         up.used = true;
         if (req.headers["content-type"] !== up.contentType) throw new Fail(400, "invalid_request", `The upload must be sent as ${up.contentType}.`);
         if (raw.length !== up.bytes) throw new Fail(400, "invalid_request", `The upload must be exactly ${up.bytes} bytes.`);
-        s.file = { filename: s.filename, contentType: s.contentType, bytes: new Uint8Array(raw) };
+        up.sink.put(new Uint8Array(raw));
         return send(200, {});
       }
 
@@ -281,6 +346,10 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
     usage,
     // Changes how many clips a user may start from now on, as `clipLimit` does at the start.
     setClipLimit(n: number) { limits.clips = n; },
+    // The same for the seconds of video a user may have cut out.
+    setCutoutLimit(n: number) { limits.cutoutSeconds = n; },
+    // Stops cutout jobs from finishing (true) or lets them finish again (false).
+    holdCutouts(hold: boolean) { cutoutsHeld = hold; },
     approve(userCode: string, userId = "u-test") { for (const d of devices.values()) if (d.userCode === userCode) { d.approved = true; d.userId = userId; } },
     // A ready token, for tests that are not about logging in.
     login(userId = "u-test") { const t = nextId("tok"); tokens.set(t, userId); return t; },