reelkit-cli 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -59,6 +59,7 @@ reelkit render
59
59
  | `reelkit assets voices` | List narration voices |
60
60
  | `reelkit assets voiceover` | Record narration from `plan.json` |
61
61
  | `reelkit assets gen image` | Generate a scene's illustration |
62
+ | `reelkit assets gen clip` | Generate a scene's video clip, or a green-screen one keyed to a transparent video |
62
63
  | `reelkit plan check` | Validate `plan.json` |
63
64
  | `reelkit check` | Check the composition without rendering |
64
65
  | `reelkit preview` | Two test frames per scene |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "reelkit-cli",
3
- "version": "0.1.2",
3
+ "version": "0.2.0",
4
4
  "description": "CLI and Claude skill for making short-form video with a shared asset library.",
5
5
  "license": "MIT",
6
6
  "author": "Daniel Livshin",
package/skill/SKILL.md CHANGED
@@ -29,7 +29,7 @@ Add `--footage` for a video the motion design should be laid over. User files st
29
29
  Read `reference/styles.md` and agree the video's look with the user: one of its looks, or their own. If they already described what they want, match it and confirm in one sentence. Ask anything still open in one message, each question with a default. If any on-screen text will be Hebrew, read `reference/hebrew-rtl.md` now: it changes how words may enter and how lines are written.
30
30
 
31
31
  ### 3. Plan
32
- Read `reference/scriptwriting.md` and `reference/scene-treatments.md`. Write the chosen look into the first scene's `notes`. Run `reelkit assets voices` and choose a voice that fits the idea, audience and language.
32
+ Read `reference/scriptwriting.md` and `reference/scene-treatments.md` (and `reference/clips.md` if any scene might be a video clip). Write the chosen look into the first scene's `notes`. Run `reelkit assets voices` and choose a voice that fits the idea, audience and language.
33
33
 
34
34
  Write `plan.json`:
35
35
 
@@ -47,6 +47,7 @@ Write `plan.json`:
47
47
  "treatment": "motion-graphic",
48
48
  "onScreenText": ["up to three short items"],
49
49
  "imagePrompt": null,
50
+ "clipPrompt": null,
50
51
  "shareable": false,
51
52
  "imageTags": [],
52
53
  "userAssetIds": [],
@@ -57,8 +58,9 @@ Write `plan.json`:
57
58
  ```
58
59
 
59
60
  - 3 to 8 scenes. Scene ids are short, unique, lowercase with dashes.
60
- - `treatment` is `motion-graphic`, `illustration` or `footage-overlay`. With footage, `mode` is `"footage"` and every scene is `footage-overlay`; otherwise `mode` is `"motion"` and no scene is.
61
+ - `treatment` is `motion-graphic`, `illustration`, `clip` or `footage-overlay`. With footage, `mode` is `"footage"` and every scene is `footage-overlay`; otherwise `mode` is `"motion"` and no scene is.
61
62
  - `imagePrompt` is set only for `illustration` scenes, with 3 to 6 `imageTags`. Set `shareable` to true only when the prompt is fully generic: no brand, product, person or detail specific to this user.
63
+ - `clipPrompt` is set only for `clip` scenes (a generated or reused video clip is the scene's picture; the rest of the scene uses `imageTags` and `shareable` as an illustration does). Clips are scarce: most videos have none or one or two.
62
64
  - `userAssetIds` lists the ids of the user's files shown in that scene.
63
65
  - `pace` is `slow`, `normal` or `fast`.
64
66
 
@@ -69,11 +71,13 @@ Run `reelkit plan check`; it prints the estimated length to tell the user. Fix e
69
71
  ### 4. Voice
70
72
  `reelkit assets voiceover --all`. It records each scene and prints the real length of the video. If the user wants a different voice, change `voiceId` in `plan.json` and run it again with `--redo`.
71
73
 
72
- ### 5. Images
74
+ ### 5. Images and clips
73
75
  Read `reference/asset-reuse.md`. For each illustration scene, search first:
74
76
  `reelkit assets search "<what the scene needs>" --kind image`
75
77
  Each result starts with a match percentage: how likely it is good enough to reuse. Pull the best result (`reelkit assets pull <id> --scene <sceneId>`) when its match is 60% or more and, reading its description, it fits the scene. Otherwise generate: `reelkit assets gen image --scene <sceneId>`. Never give two scenes the same image.
76
78
 
79
+ For each clip scene, read `reference/clips.md`, then search `reelkit assets search "<what the clip shows>" --kind clip` and pull a 60% match (`reelkit assets pull <id> --scene <sceneId>`), or generate: `reelkit assets gen clip --scene <sceneId>` (add `--green` for a green-screen subject). It waits for the clip, which takes minutes; check what is left with `reelkit whoami`.
80
+
77
81
  ### 6. Composition
78
82
  Read `reference/kit.md`, `reference/remotion-composition.md`, `reference/motion-design.md` and `reference/captions.md`.
79
83
 
@@ -0,0 +1,58 @@
1
+ ---
2
+ name: clips
3
+ description: Use when a scene might be a generated or reused video clip - B-roll that shows what the narration says, or a green-screen subject placed over your own graphics - including how to write the prompt, what a clip costs and how to layer it.
4
+ ---
5
+
6
+ # Video clips
7
+
8
+ A clip is a few seconds of generated video. It is the most expensive thing Reelkit makes: it takes minutes and each one uses part of a small monthly allowance. Use it where nothing else carries the point.
9
+
10
+ ## Is a clip worth it?
11
+ - Use a clip when the sentence is about something that moves and cannot be drawn well in code or as one still: a person doing something, an animal, weather, a crowd, water, a place seen from a moving camera.
12
+ - Do not use a clip for numbers, lists, steps, UI or anything with text (motion graphics do that better and cost nothing), nor for a concept one illustration shows (a still costs far less).
13
+ - Most videos need none, or one or two. Plan a clip scene with `"treatment": "clip"` and a `clipPrompt`; give it `imageTags` (3 to 6) like an illustration, and set `shareable` by the same rule as images.
14
+ - Check what is left before planning more: `reelkit whoami` shows `Clips: used/limit`, and when it resets. Clips are limited per month. If the allowance is gone, say so to the user and use an illustration or a motion graphic instead; do not work around it.
15
+
16
+ ## Search first
17
+ `reelkit assets search "<what the clip shows>" --kind clip`. Each result starts with a match percentage. Pull the best one (`reelkit assets pull <id> --scene <sceneId>`) when it is 60% or more and its description fits the sentence; it is free and instant. Otherwise generate: `reelkit assets gen clip --scene <sceneId>`. Never give two scenes the same clip.
18
+
19
+ ## Writing a clip prompt
20
+ - One subject, one action. "A fisherman pulls a net from a grey sea" is a clip; "a fisherman, his family at home, and the market" is three.
21
+ - Say the camera: "slow push-in", "handheld tracking shot", "static wide shot", "slow pan left".
22
+ - Say the light and the mood: "low golden sun", "overcast, soft light", "neon night".
23
+ - No text, letters, logos, signs or screens with content in the picture: the generator draws them wrong. Add the words yourself in the composition.
24
+ - It is 5 seconds by default (`--seconds 10` doubles the cost to the allowance only when the action truly needs it). Describe what happens in 5 seconds, not a story.
25
+ - The scene runs as long as its narration. A clip shorter than the scene holds its last frame, so end the action on a moment that can sit still.
26
+ - Generation takes up to several minutes. The command waits and prints a line every 30 seconds. If it gives up after ten minutes it prints the id and a `reelkit assets gen clip --resume <id> --scene <sceneId>` line: run it; it does not charge again.
27
+
28
+ ## B-roll rules
29
+ - The clip shows exactly what the sentence says, at the moment it says it. If the sentence is abstract, the clip is the wrong tool.
30
+ - Never the same clip twice in a video, and no two clip scenes in a row that look alike.
31
+ - Speech never runs more than 3 seconds without something changing on screen: a new clip, a new text beat, a cut to a graphic. A 5-second clip under a 9-second scene needs a text change or a second visual in the middle.
32
+ - Text over a clip must stay legible. Use `<ClipLayer src={urls[s.clipKey]} dim={0.25} />` to darken it, keep headlines in the calm part of the picture, and look at the preview frames for contrast on both the early and the late frame.
33
+
34
+ ## Green screen
35
+ A green-screen clip is a subject filmed on a flat pure green background. It is for a person or an object that you place over something of your own: a presenter pointing at the user's screenshot, a character beside your title, a product over a gradient.
36
+ - Generate it with `reelkit assets gen clip --scene <sceneId> --green`. The original is saved as `assets/clip-<sceneId>.mp4` and the green is keyed out locally into `assets/clip-<sceneId>.webm`, a video with a transparent background. The manifest has both: `s.clipKey` and `s.clipKeyedKey`. Use the `.webm`.
37
+ - Prompt it as a subject, not a scene: "A woman in a blue jacket, waist up, waves and then points to her left. Flat, evenly lit, pure green background (#00FF00)." Keep the whole subject inside the frame with room around it; no green anywhere on the subject (clothes, props, eyes); no shadows, floor or reflections on the background; no camera move, because keying a moving frame edge flickers.
38
+ - If the keyed edges look green or ragged in the preview, the prompt was at fault: regenerate once with a flatter green and an evenly lit subject, using `--redo`. A clip left unkeyed (the command says so) is keyed again for free by running the same command again.
39
+ - Layer it above the scene's background and below the captions:
40
+
41
+ ```tsx
42
+ import { Img } from "remotion";
43
+ import { Captions, KeyedClip, SceneFrame } from "reelkit/kit";
44
+
45
+ <SceneFrame from={s.startFrame} durationInFrames={s.durationFrames}>
46
+ <Img src={urls[s.userAssetKeys[0]]} style={{ width: "100%", height: "100%", objectFit: "cover" }} />
47
+ {s.clipKeyedKey ? <KeyedClip src={urls[s.clipKeyedKey]} x={0.72} y={1} scale={0.7} /> : null}
48
+ <Captions words={s.words} />
49
+ </SceneFrame>
50
+ ```
51
+
52
+ - A green-screen scene still has a `clip` treatment and a `clipPrompt`, with the background and the type in the composition. A library clip with the same use comes keyed too: `reelkit assets pull <id> --scene <sceneId>` writes the `.webm` beside the `.mp4` when the clip is marked green screen.
53
+ - It costs the same as any clip and is harder to get right: at most one or two green-screen clips per video.
54
+
55
+ ## Rules
56
+ - Search before generating, and generate once per scene. Do not regenerate to chase small improvements.
57
+ - `--share` only for a generic clip with nothing specific to this user; the user's own files stay private.
58
+ - Never claim a clip shows something it does not: look at the frames.
@@ -34,6 +34,14 @@ FootageLayer { src: string; muted?: boolean; dim?: number }
34
34
  Full-frame user footage. Place it once, outside the scenes, as the bottom layer. dim (0-1) darkens it for legibility.
35
35
  <FootageLayer src={urls[manifest.footageKey]} dim={0.25} />
36
36
 
37
+ ClipLayer { src: string; muted?: boolean; dim?: number }
38
+ Full-frame generated or library video clip as a scene's picture (treatment "clip"). Place it inside the scene's SceneFrame as the bottom layer, so it starts with the scene. When the scene is longer than the clip it holds the last frame; it never loops. dim (0-1) darkens it for legibility.
39
+ <ClipLayer src={urls[s.clipKey]} dim={0.2} />
40
+
41
+ KeyedClip { src: string; x?: number; y?: number; scale?: number; muted?: boolean }
42
+ A green-screen clip with the green keyed out (the transparent .webm, s.clipKeyedKey). Lay it over your own background or screenshot, below the captions. x and y place its bottom centre as fractions of the frame (default x 0.5, y 1: bottom centre); scale is its width as a fraction of the frame width (default 1). Holds its last frame when the scene is longer.
43
+ <KeyedClip src={urls[s.clipKeyedKey]} x={0.5} y={1} scale={0.8} />
44
+
37
45
  BgMesh { bg: string; hero: string; accent?: string }
38
46
  The bottom layer of the video: the base colour with two slow-drifting soft colour fields. Use it instead of a flat background.
39
47
  <BgMesh bg={palette.bg} hero={palette.hero} accent={palette.accent} />
@@ -78,7 +78,7 @@ Hook (first 1.5 s: the boldest visual and claim) → context (one line, one visu
78
78
  - Grain or paper texture is one static layer above everything, outside the camera group. Inside a zooming group it is slow and the grain swells into blotches.
79
79
 
80
80
  ## Banned, because they read as AI-made or cheap
81
- These hold unless the look chosen in `reference/styles.md` names an exception (a showreel look uses flat colour fields; the glowing-branches look uses glow on its lines).
81
+ These hold unless the look chosen in `reference/styles.md` names an exception (a showreel look uses flat colour fields; the neon look uses glow on its hero element and lines).
82
82
 
83
83
  Flat solid backgrounds, opacity-only fades, everything entering at once, rainbow or multi-stop gradients on text and UI, particles and sparkles, glows on interface chrome, emoji as graphics, bouncy easing, gratuitous 3D flips, a centred title on a plain gradient, mixed icon styles, lorem ipsum, and anything that looks like a stock template.
84
84
 
@@ -48,7 +48,7 @@ Each `scenes` entry:
48
48
  - Never hard-code a duration that the manifest already gives. Look scenes up by id when a scene needs bespoke content.
49
49
 
50
50
  ## Media
51
- - Reference media only as `urls[path]`, where `path` is the file's project path, using the paths in the manifest: `s.voiceoverKey`, `s.imageKey`, `s.userAssetKeys`, `manifest.footageKey`. Never hard-code a URL.
51
+ - Reference media only as `urls[path]`, where `path` is the file's project path, using the paths in the manifest: `s.voiceoverKey`, `s.imageKey`, `s.clipKey`, `s.clipKeyedKey`, `s.userAssetKeys`, `manifest.footageKey`. Never hard-code a URL.
52
52
  - A scene with narration has `s.voiceoverKey`: give it exactly one `<Voiceover src={urls[s.voiceoverKey]} />` inside its SceneFrame, and `<Captions words={s.words} />` unless the notes say otherwise.
53
53
  - Footage mode (`manifest.footageKey` is set): one `<FootageLayer>` at the bottom, outside the scenes.
54
54
  - User images: `<Img src={urls[key]} />` from remotion.
@@ -56,8 +56,9 @@ Each `scenes` entry:
56
56
  ## Layer stack (every video, bottom to top)
57
57
  1. Background: `<BgMesh>` from the kit, or `<FootageLayer>` in footage mode. Never a flat solid colour.
58
58
  2. Images: `<KenBurnsImage>` inside its scene. Every still image moves; alternate `direction` between consecutive image scenes.
59
- 3. Graphics and type, inside each scene's SceneFrame.
60
- 4. `<Grade>`, then `<Grain>`, then `<Vignette>`, once, at the very top of the video outside the scenes. In footage mode skip Grade.
59
+ 3. Clips: `<ClipLayer>` inside its scene (a clip scene's picture); a green-screen `<KeyedClip>` above the scene's background and below its type and captions. See `reference/clips.md`.
60
+ 4. Graphics and type, inside each scene's SceneFrame.
61
+ 5. `<Grade>`, then `<Grain>`, then `<Vignette>`, once, at the very top of the video outside the scenes. In footage mode skip Grade.
61
62
 
62
63
  ## Shared overlays (optional)
63
64
  - The library has ready-made overlay clips (`reelkit assets search "<description>" --kind overlay`): film marks, HUD frames, corner marks. They are white on black and are laid over the video with `<ScreenOverlay>`.
@@ -5,9 +5,10 @@ description: Use when choosing each scene's visual treatment and writing image p
5
5
 
6
6
  # Choosing scene treatments
7
7
 
8
- ## The three treatments
8
+ ## The four treatments
9
9
  - **motion-graphic** - animated text, numbers, shapes and icons drawn in code. The default. Best for hooks, lists, statistics, steps and calls to action.
10
10
  - **illustration** - one generated picture behind the text. Use when a concrete image carries the point better than type (an object, a place, a mood). At most half the scenes; each one costs money.
11
+ - **clip** - a few seconds of generated or reused video as the scene's picture (B-roll), or a green-screen subject over your own graphics. Needs a `clipPrompt`. Rare and costly: see `reference/clips.md` before using it.
11
12
  - **footage-overlay** - text and graphics over the user's own video. Required for every scene when footage was supplied, and never used otherwise.
12
13
 
13
14
  ## Image prompts (illustration scenes only)
@@ -19,40 +19,41 @@ Every look belongs to one mode. The mode decides which rules in `reference/motio
19
19
 
20
20
  **Showreel** (openers, announcements, anything that must stop the scroll)
21
21
  - Hard cuts on the beat between full-colour backgrounds; type that fills the frame. Flat solid colour fields are right here.
22
- - The camera never stops: a slow push, a degree or two of rotation, a short shake on each hit.
22
+ - The camera never stops: a slow push, a slight turn, a short shake on each hit.
23
23
  - No frame sits still longer than a third of a second, except the final hold.
24
24
  - At most three full-screen flashes in any second, and never a red strobe. This is a safety rule, not a taste rule.
25
25
 
26
26
  ## The catalogue
27
+ Each look is built from what the kit and the shared library already hold, so most of it is found with one search rather than written from nothing. The search words are suggestions for `reelkit assets search "<words>" --kind component`.
27
28
 
28
- ### 1. One shape, many interfaces (restrained)
29
- **When:** a product or app should look polished. **How:** a single rounded shape morphs through button, loader, player, toggle, tabs and chart; its width, height and radius move together on one spring, and the leading edge of a slider or toggle moves on a faster spring than the trailing edge so it stretches. A pointer, if shown, drives values directly while it drags and springs back on release. **Forbidden:** a cut between states, more than one accent colour.
29
+ ### 1. Product walkthrough (restrained)
30
+ **When:** an app, a site or a feature, with the user's real screenshots. **How:** the screenshot sits in a device or browser frame (search "phone mockup", "browser mockup") on a calm mesh background; interface moments arrive as real-looking parts (search "notification", "chat messages", "alert dialog", "status toast"); a pointer label names one thing at a time (search "callout"). One frame stays on screen and the content inside it changes. **Forbidden:** an invented screen, more than one callout at once, glow on interface parts.
30
31
 
31
- ### 2. Giant type on colour fields (showreel)
32
- **When:** an opener or a strong message of 3 to 6 words. **How:** one or two words per frame, a hard cut to a new full colour on each beat, each word entering a different way (a scale hit, widening from tight to wide, a short slide, a design-tool selection box that snaps on). End with the whole message together, held at least 1.5 seconds. **Forbidden:** soft gradients, outlined words, a still frame.
32
+ ### 2. Numbers that land (restrained)
33
+ **When:** results, growth, a comparison, anything with figures from the narration. **How:** one figure or one chart per scene (search "statistic counter", "line chart", "bar chart", "pie chart", "radial gauge"); the number counts up and finishes before the scene's midpoint; a before and after sits side by side (search "comparison"). The payoff number gets the biggest move of the video. **Forbidden:** any figure that is not in the plan, two charts in one frame, decorative data.
33
34
 
34
- ### 3. A word made of particles (showreel)
35
- **When:** revealing a name or brand, or a closing wow. **How:** sample the word's pixels once, then move each particle between targets (word, sphere, swirl, word) as a function of the frame; draw on a canvas, not thousands of elements. The word must be readable again at the end and held half a second. **Forbidden:** per-frame randomness.
35
+ ### 3. Headline hits (showreel)
36
+ **When:** an opener, a launch, a message of a few words that must stop the scroll. **How:** one or two words fill the frame (search "kinetic title"), hard cuts to a new full colour on the beat, each word entering a different way, and the whole message together at the end, held. The camera keeps a slow push the whole time. **Forbidden:** soft gradients, outlined words, a still frame before the final hold.
36
37
 
37
- ### 4. A field of blocks (showreel)
38
- **When:** an energetic background behind a headline, or anything about data. **How:** a grid of blocks rising and falling as a travelling wave, one accent object moving across it, the headline on a dark soft patch above. All blocks jump on the last word. **Forbidden:** letting the background out-shout the text.
38
+ ### 4. Cinematic story (restrained)
39
+ **When:** a mood, a place, a story told by a voice. **How:** generated or reused pictures with a slow move across them, a film grade and grain over everything, a poster-style title to open (search "cinematic title"), a film overlay if it suits (`--kind overlay`, search "film"). Few words on screen; the captions carry the text. **Forbidden:** interface parts, bright accent colours, fast cuts.
39
40
 
40
- ### 5. Rings of words (showreel)
41
- **When:** a list of values, services or topics. **How:** each line of words bends into a ring; rings turn at different slow speeds inside one another with the title in the centre. Read every ring in a preview frame. **Forbidden:** more than about ten words, rings too small to read.
41
+ ### 5. Neon and night (showreel)
42
+ **When:** technology, AI, nightlife, anything that should feel electric. **How:** near-black background, one glowing colour, a sign-like title (search "neon title"), thin lines that draw themselves. Glow is the look here, so it is allowed, on the hero element and the lines only. **Forbidden:** glow on body text, a second glowing colour, flashes above the limit.
42
43
 
43
- ### 6. Hand-drawn character (restrained)
44
- **When:** a personal, warm or funny message. **How:** a simple pencil character on paper; lines boil slightly by switching between two or three hand-offset versions every few frames (chosen by frame number, never at random), a speech bubble in a handwriting face. **Forbidden:** clean vector precision, more than two colours beside the pencil.
44
+ ### 6. Notebook (restrained)
45
+ **When:** a personal, warm or teaching tone. **How:** a paper background with faint static grain, a handwriting or rounded face, hand-drawn arrows, circles and underlines that draw themselves onto the point being made (search "hand drawn arrow circle underline"). Slight wobble comes from switching between two or three drawn versions by frame number, never at random. **Forbidden:** crisp interface chrome, more than two ink colours.
45
46
 
46
- ### 7. Storybook explainer (restrained)
47
- **When:** teaching how something works, in 3 to 5 steps. **How:** flat illustration with rounded shapes, one dark outline colour, a paper background and faint static grain. The camera moves into the object and a round cutaway opens to show the inside, instead of cutting to a new scene. Each step gets a short label beside the action with a thin leader line, on screen at least 1.3 seconds; end on a 2 to 4 word title over the result. **Forbidden:** realistic shadows, strong gradients, labels covering the drawing.
47
+ ### 7. Step by step (restrained)
48
+ **When:** a process, a how-to, a recipe, three to five steps. **How:** a numbered list that builds one step at a time (search "steps process"), or one drawing the camera moves through, stopping at each step with a short label beside the action. Each label stays at least 1.3 seconds; end on a short title over the result. **Forbidden:** all steps appearing at once, labels covering the thing they describe.
48
49
 
49
- ### 8. Glowing branches (showreel)
50
- **When:** AI, science, the moment an idea is born. **How:** lines of light that grow and fork like nerve cells, a spark running along them, on near-black. Glow is the look here, so it is allowed, on the lines only. **Forbidden:** glow on text, flashes above the limit.
50
+ ### 8. People and proof (restrained)
51
+ **When:** a testimonial, an introduction, a creator talking, social proof. **How:** the person's name and role in a lower third (search "name lower third"), a quote card with its stars (search "quote testimonial"), a follow line to close (search "social handle", "cta button"). Quotes and names come from the user, word for word. **Forbidden:** an invented quote, rating or follower count.
51
52
 
52
- ### 9. Sketch to object (restrained)
53
- **When:** showing how a physical thing is built. **How:** a pencil technical drawing draws itself line by line, gains colour and shade, then does what it was built to do. **Forbidden:** skipping the drawing stage; it is the point.
53
+ ### 9. Over footage (either mode)
54
+ **When:** the user supplied their own video. **How:** the footage is the picture; graphics are guests. One overlay at a time, clear of the speaker's face; a small punch-in on each sentence's key word; lower thirds and callouts from the library; captions always. **Forbidden:** covering the face, a graphic left on screen after its sentence, zooms on footage that already has its own.
54
55
 
55
- ### 10. Logo reveal (either mode)
56
+ ### 10. Logo sting (either mode)
56
57
  **When:** an opener or end card, from the user's own logo file. **How:** build toward the logo from its own shapes and colours and land on it exactly as supplied, held at least 1.5 seconds. **Forbidden:** redrawing, recolouring or stretching the logo; anyone else's logo or trademark.
57
58
 
58
59
  ## Your own
package/src/cli.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { Command } from "commander";
2
2
  import { ApiFailure } from "./api/client";
3
- import { assetsGenImage, assetsPull, assetsSearch, assetsUpload, assetsVoiceover, assetsVoices } from "./commands/assets";
3
+ import { assetsGenClip, assetsGenImage, assetsPull, assetsSearch, assetsUpload, assetsVoiceover, assetsVoices } from "./commands/assets";
4
4
  import { authLogin, authLogout, whoami } from "./commands/auth";
5
5
  import { check, preview, render } from "./commands/build";
6
6
  import { init } from "./commands/init";
@@ -73,13 +73,18 @@ assets.command("upload <file>").description("Add one of your own files to this p
73
73
  .action(run(assetsUpload));
74
74
  assets.command("search <query>").description("Search the shared library by meaning; each result shows how well it fits").option("--kind <kind>", "image, overlay, sfx, music, component or clip").option("--limit <n>", "how many results")
75
75
  .action(run(assetsSearch));
76
- assets.command("pull <id>").description("Download a library item into this project").option("--scene <sceneId>", "use it as this scene's image").option("--force", "replace a component file that already exists")
77
- .action(run(assetsPull));
76
+ assets.command("pull <id>").description("Download a library item into this project").option("--scene <sceneId>", "use it as this scene's image or clip").option("--force", "replace a component file that already exists")
77
+ .action(run((ctx, id, opts) => assetsPull(ctx, id, opts)));
78
78
  assets.command("voices").description("List the narration voices").action(run(assetsVoices));
79
79
  assets.command("voiceover").description("Record the narration from plan.json").option("--scene <sceneId>", "one scene").option("--all", "every scene").option("--redo", "record again even if it exists")
80
80
  .action(run(assetsVoiceover));
81
- assets.command("gen").description("Generate an asset").command("image [prompt]").description("Generate a scene's illustration; the prompt defaults to the plan's")
81
+ const gen = assets.command("gen").description("Generate an asset");
82
+ gen.command("image [prompt]").description("Generate a scene's illustration; the prompt defaults to the plan's")
82
83
  .requiredOption("--scene <sceneId>", "the scene it is for").option("--redo", "generate a new image even if the scene has one").action(run(assetsGenImage));
84
+ gen.command("clip [prompt]").description("Generate a scene's video clip (takes minutes); the prompt defaults to the plan's clipPrompt")
85
+ .requiredOption("--scene <sceneId>", "the scene it is for").option("--green", "film it on green and key the green out into a transparent .webm").option("--seconds <n>", "5 or 10 (default 5)")
86
+ .option("--share", "also add it to the shared library (only a generic clip)").option("--redo", "generate a new clip even if the scene has one").option("--resume <id>", "keep waiting for a clip already started, without paying again")
87
+ .action(run((ctx, prompt, opts) => assetsGenClip(ctx, prompt, opts)));
83
88
 
84
89
  program.command("plan").description("Work with plan.json").command("check").description("Validate plan.json and list what to fix or improve").action(run(planCheck));
85
90
 
@@ -1,11 +1,12 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { copyFileSync, mkdirSync, readFileSync } from "node:fs";
3
3
  import { basename, dirname, resolve } from "node:path";
4
- import { download, uploadTo } from "../api/client";
4
+ import { ApiFailure, download, uploadTo } from "../api/client";
5
5
  import { client, type Ctx, type Result } from "../context";
6
6
  import { LibraryKindSchema, UploadKindSchema } from "../contract";
7
7
  import { PACE_SPEED, type AssetManifest, type AssetRecord, type ScenePlan } from "../pipeline/schema";
8
- import { buildManifest, voiceoverStale, type Voiceover } from "../project/manifest";
8
+ import { keyGreen } from "../project/chromakey";
9
+ import { buildManifest, voiceoverStale, type ClipRecord, type Voiceover } from "../project/manifest";
9
10
  import { loadPlan } from "./plan";
10
11
  import { FILE_NAME } from "../render/validate";
11
12
  import { probeFile } from "../project/probe";
@@ -90,19 +91,35 @@ export function unsafeName(name: string, kind: string): boolean {
90
91
  return kind === "component" && (!FILE_NAME.test(name) || name === "Video.tsx");
91
92
  }
92
93
 
93
- export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean }): Promise<Result> {
94
+ // What a test may replace: the ffmpeg call that keys green out, and the clock the clip wait is timed by.
95
+ export type ClipDeps = { key?: (input: string, output: string) => Promise<void>; now?: () => number };
96
+
97
+ // The keyed copy sits beside its clip: clip.mp4 becomes clip.webm.
98
+ const keyedPathOf = (path: string) => path.replace(/\.[^./]+$/, "") + ".webm";
99
+
100
+ function setSceneClip(project: Project, sceneId: string, record: ClipRecord) {
101
+ project.writeJson(FILES.clips, { ...project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {}), [sceneId]: record });
102
+ }
103
+
104
+ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; force?: boolean }, deps: ClipDeps = {}): Promise<Result> {
94
105
  const project = openProject(ctx.cwd);
95
106
  if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id)) return { ok: false, summary: `"${id}" is not a library id. Copy the id from \`reelkit assets search\`.` };
96
107
  if (opts.scene) {
97
108
  // Checked before anything is downloaded.
98
- const illustrations = loadPlan(project).scenes.filter((s) => s.treatment === "illustration").map((s) => s.id);
99
- if (!illustrations.includes(opts.scene)) {
100
- return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}` };
109
+ const scenes = loadPlan(project).scenes;
110
+ const illustrations = scenes.filter((s) => s.treatment === "illustration").map((s) => s.id);
111
+ const clipScenes = scenes.filter((s) => s.treatment === "clip").map((s) => s.id);
112
+ if (!illustrations.includes(opts.scene) && !clipScenes.includes(opts.scene)) {
113
+ return { ok: false, summary: `Scene ${opts.scene} is not an illustration scene in plan.json. ${illustrations.length ? `The illustration scenes are: ${illustrations.join(", ")}.` : "The plan has no illustration scenes."}${clipScenes.length ? ` The clip scenes are: ${clipScenes.join(", ")}.` : ""}` };
101
114
  }
102
115
  }
103
116
  const pulled = await client(ctx)("libraryPull", { id });
104
117
  if (unsafeName(pulled.filename, pulled.item.kind)) return { ok: false, summary: `The library returned an unsafe file name for ${id}, so nothing was written. Report this item.` };
105
- if (opts.scene && pulled.item.kind !== "image") return { ok: false, summary: `${id} is ${pulled.item.kind}, not an image, so it cannot be a scene's image.` };
118
+ if (opts.scene) {
119
+ const treatment = loadPlan(project).scenes.find((x) => x.id === opts.scene)?.treatment;
120
+ if (treatment === "clip" && pulled.item.kind !== "clip") return { ok: false, summary: `${id} is ${pulled.item.kind}, not a clip, so it cannot be scene ${opts.scene}'s clip.` };
121
+ if (treatment !== "clip" && pulled.item.kind !== "image") return { ok: false, summary: `${id} is ${pulled.item.kind}, not an image, so it cannot be a scene's image.` };
122
+ }
106
123
  // A component is code, not media: it goes beside Video.tsx, which may only import files in src/.
107
124
  if (pulled.item.kind === "component") {
108
125
  const path = `src/${pulled.filename}`;
@@ -116,15 +133,24 @@ export async function assetsPull(ctx: Ctx, id: string, opts: { scene?: string; f
116
133
  const path = `assets/lib/${id}/${pulled.filename}`;
117
134
  await download(pulled.url, project.path(path));
118
135
  recordPull(project, id, { path, kind: pulled.item.kind, title: pulled.item.title, meta: pulled.item.meta });
136
+ // A clip filmed on green also comes keyed, so it can be laid over a background.
137
+ const green = pulled.item.kind === "clip" && pulled.item.meta.greenScreen === true;
138
+ const keyedPath = green ? keyedPathOf(path) : undefined;
139
+ if (keyedPath) await (deps.key ?? keyGreen)(project.path(path), project.path(keyedPath));
140
+ const sceneIsClip = pulled.item.kind === "clip";
119
141
  let note: string | undefined;
120
142
  if (opts.scene) {
121
- setSceneImage(project, opts.scene, path);
143
+ if (sceneIsClip) {
144
+ const durationSec = typeof pulled.item.meta.durationSec === "number" ? pulled.item.meta.durationSec : 5;
145
+ setSceneClip(project, opts.scene, { key: path, greenScreen: green, durationSec, ...(keyedPath ? { keyedKey: keyedPath } : {}) });
146
+ } else setSceneImage(project, opts.scene, path);
122
147
  if (project.exists(FILES.plan)) note = tryManifest(project, loadPlan(project)).note;
123
148
  }
149
+ const refs = keyedPath ? ` The original is urls["${path}"]; the keyed copy with a transparent background is urls["${keyedPath}"].` : ` Reference it in the composition as urls["${path}"].`;
124
150
  return {
125
- ok: true, data: { path, kind: pulled.item.kind },
151
+ ok: true, data: { path, kind: pulled.item.kind, ...(keyedPath ? { keyedPath } : {}) },
126
152
  summary: [
127
- opts.scene ? `Pulled ${id} as the image for scene ${opts.scene}.` : `Pulled ${id} to ${path}. Reference it in the composition as urls["${path}"].`,
153
+ opts.scene ? `Pulled ${id} as the ${sceneIsClip ? "clip" : "image"} for scene ${opts.scene}.${keyedPath ? refs : ""}` : `Pulled ${id} to ${path}.${refs}`,
128
154
  ...(note ? [note] : []),
129
155
  ].join("\n"),
130
156
  };
@@ -194,3 +220,113 @@ export async function assetsGenImage(ctx: Ctx, prompt: string | undefined, opts:
194
220
  summary: [`Generated the image for scene ${scene.id} at ${path}${img.libraryId ? `, and shared it to the library as ${img.libraryId} (awaiting review)` : ""}.`, ...(note ? [note] : [])].join("\n"),
195
221
  };
196
222
  }
223
+
224
+
225
+ const CLIP_POLL_MS = 5_000;
226
+ const CLIP_PROGRESS_MS = 30_000;
227
+ const CLIP_TIMEOUT_MS = 10 * 60_000;
228
+
229
+ // A clip takes minutes, so it is a job: start it (which reserves the clip from the quota), then ask every few seconds until it ends.
230
+ // The id is the only handle on a paid job, so every way out after the start says it and how to resume.
231
+ export async function assetsGenClip(
232
+ ctx: Ctx, prompt: string | undefined,
233
+ opts: { scene: string; green?: boolean; seconds?: string; share?: boolean; redo?: boolean; resume?: string },
234
+ deps: ClipDeps = {},
235
+ ): Promise<Result> {
236
+ const project = openProject(ctx.cwd);
237
+ const plan = loadPlan(project);
238
+ const scene = plan.scenes.find((s) => s.id === opts.scene);
239
+ if (!scene || scene.treatment !== "clip" || !scene.clipPrompt) return { ok: false, summary: `Scene ${opts.scene} is not a clip scene, so it does not take a clip. Set its treatment to "clip" and give it a clipPrompt in plan.json.` };
240
+ const seconds = opts.seconds === undefined ? 5 : Number(opts.seconds);
241
+ if (seconds !== 5 && seconds !== 10) return { ok: false, summary: `--seconds must be 5 or 10, not "${opts.seconds}".` };
242
+ if (opts.resume && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(opts.resume)) return { ok: false, summary: `"${opts.resume}" is not a clip id. Copy it from the earlier message.` };
243
+ if (opts.share && (!scene.shareable || scene.userAssetIds.length > 0 || project.footage())) {
244
+ return { ok: false, summary: `Scene ${scene.id} is private (it is not marked shareable in plan.json, or it shows your own files), so --share is refused. Run again without --share.` };
245
+ }
246
+
247
+ const key = deps.key ?? keyGreen;
248
+ const now = deps.now ?? Date.now;
249
+ const keyedKey = `assets/clip-${scene.id}.webm`;
250
+ const records = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
251
+ const have = records[scene.id];
252
+ // A scene that already has its clip is not charged for again, whatever prompt is given; only --redo asks for a new one.
253
+ if (have && project.exists(have.key) && !opts.redo && !opts.resume) {
254
+ if (opts.green && have.greenScreen && !have.keyedKey) {
255
+ // The clip was paid for but not keyed (ffmpeg failed or --green came later): keying is free, so do it now.
256
+ const keyed = await keyScene(project, scene.id, have, key);
257
+ if (!keyed.ok) return keyed;
258
+ return { ok: true, data: { path: have.key, keyedPath: keyedKey }, summary: `Keyed the green out of scene ${scene.id}'s clip into ${keyedKey}.` };
259
+ }
260
+ return { ok: true, data: { path: have.key, skipped: true }, summary: `Scene ${scene.id} already has a clip at ${have.key}. Use --redo to generate a new one.` };
261
+ }
262
+
263
+ const api = client(ctx);
264
+ const started = now();
265
+ let id = opts.resume;
266
+ if (!id) {
267
+ const job = await api("clipStart", {
268
+ prompt: prompt ?? scene.clipPrompt, aspect: plan.aspect, greenScreen: Boolean(opts.green), durationSec: seconds,
269
+ shareable: Boolean(opts.share), tags: scene.imageTags,
270
+ });
271
+ id = job.id;
272
+ ctx.log(`Started clip ${id} for scene ${scene.id}. It usually takes a few minutes.`);
273
+ } else ctx.log(`Waiting for clip ${id} for scene ${scene.id}.`);
274
+ const resume = `reelkit assets gen clip --resume ${id} --scene ${scene.id}`;
275
+
276
+ try {
277
+ let lastLog = started;
278
+ for (;;) {
279
+ const st = await api("clipStatus", { id });
280
+ if (st.status === "failed") {
281
+ return { ok: false, data: { id, status: "failed" }, summary: `The clip for scene ${scene.id} failed: ${st.message ?? "the video provider gave no reason."} You were not charged. Change the prompt and run the command again.` };
282
+ }
283
+ if (st.status === "done") {
284
+ const path = `assets/clip-${scene.id}.${st.ext}`;
285
+ await download(st.url!, project.path(path));
286
+ const green = Boolean(st.greenScreen ?? opts.green);
287
+ const record: ClipRecord = { key: path, greenScreen: green, durationSec: st.durationSec ?? seconds };
288
+ setSceneClip(project, scene.id, record);
289
+ let keyedPath: string | undefined;
290
+ if (green || opts.green) {
291
+ // The paid clip is already saved above; if keying fails, running the command again keys it without a new charge.
292
+ const keyed = await keyScene(project, scene.id, { ...record, greenScreen: true }, key);
293
+ if (!keyed.ok) return keyed;
294
+ keyedPath = keyedKey;
295
+ }
296
+ const { note } = tryManifest(project, plan);
297
+ return {
298
+ ok: true, data: { id, path, durationSec: record.durationSec, ...(keyedPath ? { keyedPath } : {}), libraryId: st.libraryId ?? null },
299
+ summary: [
300
+ `Generated the clip for scene ${scene.id} at ${path}${keyedPath ? `, keyed to ${keyedPath}` : ""}${st.libraryId ? `, and shared it to the library as ${st.libraryId} (awaiting review)` : ""}.`,
301
+ ...(note ? [note] : []),
302
+ ].join("\n"),
303
+ };
304
+ }
305
+ const elapsed = now() - started;
306
+ if (elapsed >= CLIP_TIMEOUT_MS) {
307
+ return { ok: false, data: { id, status: "pending" }, summary: `The clip for scene ${scene.id} (id ${id}) is not ready after ${CLIP_TIMEOUT_MS / 60_000} minutes. It is still being made and is not lost: run \`${resume}\` to keep waiting for it; that does not charge again.` };
308
+ }
309
+ if (now() - lastLog >= CLIP_PROGRESS_MS) {
310
+ lastLog = now();
311
+ ctx.log(`Still making the clip for scene ${scene.id} (${Math.round(elapsed / 1000)}s so far)...`);
312
+ }
313
+ await ctx.sleep(CLIP_POLL_MS);
314
+ }
315
+ } catch (e) {
316
+ // A clip that is not found or a login that is wrong has nothing to resume; anything else may be a hiccup on a clip that is paid for.
317
+ if (e instanceof ApiFailure && (e.code === "not_found" || e.code === "unauthenticated")) throw e;
318
+ return { ok: false, data: { id }, summary: `Lost contact while the clip for scene ${scene.id} (id ${id}) was being made: ${e instanceof Error ? e.message : String(e)} The clip is not lost: run \`${resume}\`; that does not charge again.` };
319
+ }
320
+ }
321
+
322
+ // Keys a scene's saved clip and records the keyed copy. A failure leaves the record as it was.
323
+ async function keyScene(project: Project, sceneId: string, record: ClipRecord, key: NonNullable<ClipDeps["key"]>): Promise<Result> {
324
+ const keyedKey = `assets/clip-${sceneId}.webm`;
325
+ try {
326
+ await key(project.path(record.key), project.path(keyedKey));
327
+ } catch (e) {
328
+ return { ok: false, data: { path: record.key }, summary: `The clip is saved at ${record.key}, but the green could not be keyed out: ${e instanceof Error ? e.message : String(e)}` };
329
+ }
330
+ setSceneClip(project, sceneId, { ...record, greenScreen: true, keyedKey });
331
+ return { ok: true, summary: "" };
332
+ }
@@ -113,6 +113,6 @@ export async function whoami(ctx: Ctx): Promise<Result> {
113
113
  const q = me.quota;
114
114
  return {
115
115
  ok: true, data: me,
116
- summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
116
+ summary: `${me.handle}\nVoiceover: ${q.voiceoverChars.used}/${q.voiceoverChars.limit} characters\nImages: ${q.images.used}/${q.images.limit}\nClips: ${q.clips.used}/${q.clips.limit}\nResets: ${q.resetsAt.slice(0, 10)}\nContributions: ${me.contributions}`,
117
117
  };
118
118
  }
@@ -39,9 +39,20 @@
39
39
  // a file URL serves one file, an upload URL accepts uploads for one object only.
40
40
  // - The public listing's `previewUrl` is a preview that needs no login. An item with a preview of its own gets that one; otherwise an `image`
41
41
  // or an `sfx` gets a URL of its own file (a picture and a short sound need no login to look at or hear), and every other kind (music,
42
- // overlay, clip, component), whose full file is long media or code, gets no `previewUrl`.
42
+ // overlay, clip, component), whose full file is long media or code, gets no `previewUrl`. The one exception is a clip of ten seconds
43
+ // or less (`meta.durationSec`): short generated clips are made to be shared, so such a clip is previewed by its own file.
43
44
  // - Every returned URL needs no headers and stays valid for at least five minutes.
44
45
  // - The generation routes (`voiceover`, `images`) are synchronous: they return the URL of the finished file.
46
+ // - A video clip takes minutes, so it is a job with two calls. `clipStart` checks the request, reserves one clip from the caller's monthly clip
47
+ // quota and answers at once with an id and `pending`; the client then asks `clipStatus` for that id until it is `done` or `failed`.
48
+ // `url`, `ext` and `contentType` are present exactly when the status is `done`, and `message` (one plain line saying what went wrong)
49
+ // exactly when it is `failed`. A failed clip is not charged: the reservation is released. A clip id belongs to its caller: another user's id,
50
+ // or an unknown one, is `not_found`.
51
+ // - When the quota is used up, `clipStart` is 429 `quota_exceeded` with the reset date, as for images. When the server has no video provider
52
+ // configured, `clipStart` is 400 `invalid_request` with the message "Clip generation is not available on this server yet."
53
+ // - A green-screen clip (`greenScreen: true`) is a subject filmed on a flat pure green background, meant to be keyed out by the client. The server
54
+ // stores and returns the original, green and all; it does not remove the background. A shareable clip also becomes a library item of kind
55
+ // `clip` in review, whose `meta` may carry `greenScreen: true` and `durationSec`, and its id is returned as `libraryId`.
45
56
  // - `durationSec` is the decoded audio length; `words` are in seconds from the start of that audio.
46
57
  // - Search returns published items only, and leaves out items that are not matches at all; the default `limit` is 8.
47
58
  import { z } from "zod";
@@ -89,7 +100,8 @@ export const LibraryItemSchema = z.object({
89
100
  title: z.string(),
90
101
  description: z.string(),
91
102
  tags: z.array(z.string()),
92
- // width, height, durationSec, aspect, props, source, licence: whatever applies to the kind.
103
+ // width, height, durationSec, aspect, props, source, licence: whatever applies to the kind. A clip may carry `greenScreen: true` (filmed on a
104
+ // flat green background, to be keyed out) and `durationSec`.
93
105
  meta: z.record(z.string(), z.unknown()),
94
106
  visibility: z.enum(["private", "review", "published"]),
95
107
  });
@@ -105,7 +117,7 @@ export type Voice = z.infer<typeof VoiceSchema>;
105
117
  const Meter = z.object({ used: z.number(), limit: z.number() });
106
118
  export const MeSchema = z.object({
107
119
  userId: z.string(), handle: z.string(),
108
- quota: z.object({ voiceoverChars: Meter, images: Meter, resetsAt: z.string() }),
120
+ quota: z.object({ voiceoverChars: Meter, images: Meter, clips: Meter, resetsAt: z.string() }),
109
121
  // How many of the user's own items are published in the shared library. An item still in review, or sent back, is not counted.
110
122
  contributions: z.number(),
111
123
  });
@@ -148,6 +160,20 @@ export const routes = {
148
160
  z.object({ prompt: text(2000).min(1), aspect: AspectSchema, shareable: z.boolean(), tags: z.array(text(40)).max(12) }),
149
161
  // libraryId is set when the image was also added to the shared library.
150
162
  z.object({ url: z.string(), ext: z.enum(["png", "jpg", "jpeg", "webp"]), contentType: z.string(), libraryId: z.string().optional() })),
163
+ clipStart: route("POST", "/clips", true,
164
+ z.object({
165
+ prompt: text(2000).min(1), aspect: AspectSchema, greenScreen: z.boolean().default(false), durationSec: z.union([z.literal(5), z.literal(10)]).default(5),
166
+ shareable: z.boolean(), tags: z.array(text(40)).max(12),
167
+ }),
168
+ z.object({ id: z.string(), status: z.literal("pending") })),
169
+ clipStatus: route("POST", "/clips/status", true,
170
+ z.object({ id: ItemId }),
171
+ z.object({
172
+ id: z.string(), status: z.enum(["pending", "done", "failed"]), message: z.string().optional(),
173
+ url: z.string().optional(), ext: z.enum(["mp4"]).optional(), contentType: z.string().optional(), durationSec: z.number().positive().optional(),
174
+ greenScreen: z.boolean().optional(), libraryId: z.string().optional(),
175
+ }).refine((r) => (r.status === "done") === (r.url !== undefined && r.ext !== undefined && r.contentType !== undefined) && (r.status === "failed") === (r.message !== undefined),
176
+ "url, ext and contentType belong to done and message to failed")),
151
177
  publicLibrary: route("GET", "/public/library", false,
152
178
  z.object({ q: text(200).optional(), kind: LibraryKindSchema.optional(), page: z.coerce.number().int().min(1).max(10000).optional() }),
153
179
  // With `q` the items are the best matches by meaning, best first, each with `match` (0..1), in one page (`hasMore` false). Under heavy
@@ -8,9 +8,12 @@ export type Aspect = z.infer<typeof AspectSchema>;
8
8
  export const SceneSchema = z.object({
9
9
  id: z.string().regex(/^[a-z0-9-]+$/).describe("short unique id, lowercase letters, digits, dashes"),
10
10
  narration: z.string().min(1).describe("the words spoken in this scene"),
11
- treatment: z.enum(["motion-graphic", "illustration", "footage-overlay"]),
11
+ // clip: a generated or reused video clip is the scene's picture.
12
+ treatment: z.enum(["motion-graphic", "illustration", "footage-overlay", "clip"]),
12
13
  onScreenText: z.array(z.string()),
13
14
  imagePrompt: z.string().nullable().describe("required when treatment is illustration, otherwise null"),
15
+ // Optional only so plans stored before clips existed still load.
16
+ clipPrompt: z.string().nullable().optional().describe("required when treatment is clip, otherwise null: one subject, one action, a camera move, lighting, no text"),
14
17
  shareable: z.boolean().describe("true only when imagePrompt is fully generic: no brand, product, person or detail specific to this user. Such images are saved to a library shared by all users."),
15
18
  imageTags: z.array(z.string()).describe("3 to 6 short tags describing the illustration; empty when imagePrompt is null"),
16
19
  userAssetIds: z.array(z.string()),
@@ -57,6 +60,9 @@ export const ManifestSceneSchema = z.object({
57
60
  voiceoverKey: z.string().optional(),
58
61
  words: z.array(WordTimingSchema),
59
62
  imageKey: z.string().optional(),
63
+ // The scene's video clip, and its transparent copy when it was made on a green background.
64
+ clipKey: z.string().optional(),
65
+ clipKeyedKey: z.string().optional(),
60
66
  userAssetKeys: z.array(z.string()),
61
67
  });
62
68
  export type ManifestScene = z.infer<typeof ManifestSceneSchema>;
@@ -103,6 +109,8 @@ export function validatePlan(
103
109
  for (const s of plan.scenes) {
104
110
  if (s.treatment === "illustration" && !s.imagePrompt)
105
111
  errors.push(`Scene ${s.id}: imagePrompt is required for an illustration.`);
112
+ if (s.treatment === "clip" && !s.clipPrompt)
113
+ errors.push(`Scene ${s.id}: clipPrompt is required for a clip.`);
106
114
  for (const id of s.userAssetIds)
107
115
  if (!allowed.has(id)) errors.push(`Scene ${s.id}: unknown user asset ${id}.`);
108
116
  }
@@ -0,0 +1,17 @@
1
+ import { execFile } from "node:child_process";
2
+ import { promisify } from "node:util";
3
+
4
+ const run = promisify(execFile);
5
+
6
+ // Turns the flat pure green of a green-screen clip into transparency: VP9 with an alpha channel, which Remotion plays with `transparent`.
7
+ // The key is on pure green with a little give for compression noise; the despill pulls the green glow off the subject's edges.
8
+ // The original is left alone.
9
+ export async function keyGreen(input: string, output: string): Promise<void> {
10
+ const filter = "chromakey=color=0x00FF00:similarity=0.2:blend=0.1,despill=type=green:mix=0.5:expand=0,format=yuva420p";
11
+ try {
12
+ await run("ffmpeg", ["-v", "error", "-y", "-i", input, "-vf", filter, "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-b:v", "0", "-crf", "30", "-an", output]);
13
+ } catch (e) {
14
+ if ((e as NodeJS.ErrnoException)?.code === "ENOENT") throw new Error("ffmpeg was not found. Install ffmpeg (macOS: `brew install ffmpeg`) and run the command again.");
15
+ throw new Error("ffmpeg could not key the green out of the clip. The original clip is saved; run the command again to retry.");
16
+ }
17
+ }
@@ -5,6 +5,9 @@ import { FILES, type Project } from "./project";
5
5
  // What a scene was recorded from is stored with it, so a changed script or voice is noticed.
6
6
  export type Voiceover = { key: string; durationSec: number; words: WordTiming[]; text?: string; voiceId?: string; speed?: number };
7
7
 
8
+ // A scene's video clip: the original, and the transparent copy when it was filmed on green and keyed out. Stored in assets/clips.json by scene id.
9
+ export type ClipRecord = { key: string; greenScreen: boolean; durationSec: number; keyedKey?: string };
10
+
8
11
  // True when the stored recording no longer matches what the plan asks for. An entry from before this was tracked counts as different.
9
12
  export function voiceoverStale(vo: Voiceover, scene: ScenePlan["scenes"][number], plan: ScenePlan): boolean {
10
13
  return vo.text !== scene.narration || vo.voiceId !== plan.voiceId || vo.speed !== PACE_SPEED[plan.pace ?? "normal"];
@@ -16,6 +19,7 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
16
19
  const voiceovers = project.readJsonOr<Record<string, Voiceover>>(FILES.voiceovers, {});
17
20
  if (plan.scenes.some((s) => !voiceovers[s.id])) return undefined;
18
21
  const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
22
+ const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
19
23
  const footage = project.footage();
20
24
  const byId = new Map(project.assets().map((a) => [a.id, a]));
21
25
 
@@ -41,6 +45,7 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
41
45
  voiceoverKey: voiceovers[scene.id].key,
42
46
  words: voiceovers[scene.id].words,
43
47
  ...(scene.treatment === "illustration" && images[scene.id] ? { imageKey: images[scene.id] } : {}),
48
+ ...(scene.treatment === "clip" && clips[scene.id] ? { clipKey: clips[scene.id].key, ...(clips[scene.id].keyedKey ? { clipKeyedKey: clips[scene.id].keyedKey } : {}) } : {}),
44
49
  userAssetKeys: scene.userAssetIds.map((id) => {
45
50
  const a = byId.get(id);
46
51
  if (!a) throw new Error(`Scene ${scene.id} references unknown asset ${id}. Add it with \`reelkit assets upload\`.`);
@@ -56,6 +61,7 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
56
61
  export function missingAssets(project: Project, plan: ScenePlan): string[] {
57
62
  const voiceovers = project.readJsonOr<Record<string, Voiceover>>(FILES.voiceovers, {});
58
63
  const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
64
+ const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
59
65
  const problems: string[] = [];
60
66
  for (const s of plan.scenes) {
61
67
  if (!voiceovers[s.id]) problems.push(`Scene ${s.id} has no voiceover.`);
@@ -66,6 +72,11 @@ export function missingAssets(project: Project, plan: ScenePlan): string[] {
66
72
  if (!images[s.id]) problems.push(`Scene ${s.id} has no image.`);
67
73
  else if (!project.exists(images[s.id])) problems.push(`Missing file: ${images[s.id]}`);
68
74
  }
75
+ if (s.treatment === "clip") {
76
+ const clip = clips[s.id];
77
+ if (!clip) problems.push(`Scene ${s.id} has no clip. Run \`reelkit assets gen clip --scene ${s.id}\`.`);
78
+ else for (const key of [clip.key, ...(clip.keyedKey ? [clip.keyedKey] : [])]) if (!project.exists(key)) problems.push(`Missing file: ${key}`);
79
+ }
69
80
  }
70
81
  return problems;
71
82
  }
@@ -5,7 +5,7 @@ import { AspectSchema, type AssetRecord } from "../pipeline/schema";
5
5
 
6
6
  export const FILES = {
7
7
  config: "reelkit.json", plan: "plan.json", manifest: "manifest.json",
8
- assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", library: "assets/library.json",
8
+ assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json",
9
9
  } as const;
10
10
 
11
11
  // "path: message", or just the message for a problem at the root of the file.
@@ -0,0 +1,12 @@
1
+ import React from "react";
2
+ import { useMedia } from "./media";
3
+ import { AbsoluteFill, OffthreadVideo } from "remotion";
4
+
5
+ // A generated or library video clip as a scene's picture. Place it inside the scene's SceneFrame, as the bottom layer, so it starts with the scene.
6
+ // When the scene runs longer than the clip, the clip holds its last frame (it does not loop: a looped person or animal jumps back and shows the seam).
7
+ export const ClipLayer: React.FC<{ src: string; muted?: boolean; dim?: number }> = ({ src, muted = true, dim = 0 }) => (
8
+ <AbsoluteFill>
9
+ <OffthreadVideo src={useMedia(src)} muted={muted} style={{ width: "100%", height: "100%", objectFit: "cover" }} />
10
+ {dim > 0 ? <AbsoluteFill style={{ backgroundColor: `rgba(0,0,0,${dim})` }} /> : null}
11
+ </AbsoluteFill>
12
+ );
@@ -0,0 +1,15 @@
1
+ import React from "react";
2
+ import { useMedia } from "./media";
3
+ import { AbsoluteFill, OffthreadVideo } from "remotion";
4
+
5
+ // A green-screen clip with its green keyed out (the transparent .webm that `--green` makes), laid over whatever is beneath it.
6
+ // x and y are where the clip's bottom-centre sits, as fractions of the frame (x 0 is the left edge, y 1 the bottom edge); scale is the clip's
7
+ // width as a fraction of the frame width. The clip has the frame's own shape, so it is as tall as the frame is, times scale.
8
+ // Like ClipLayer it holds its last frame when the scene is longer.
9
+ export const KeyedClip: React.FC<{ src: string; x?: number; y?: number; scale?: number; muted?: boolean }> = ({ src, x = 0.5, y = 1, scale = 1, muted = true }) => (
10
+ <AbsoluteFill style={{ pointerEvents: "none" }}>
11
+ <div style={{ position: "absolute", left: `${x * 100}%`, top: `${y * 100}%`, width: `${scale * 100}%`, transform: "translate(-50%, -100%)" }}>
12
+ <OffthreadVideo src={useMedia(src)} muted={muted} transparent style={{ width: "100%", display: "block" }} />
13
+ </div>
14
+ </AbsoluteFill>
15
+ );
@@ -32,6 +32,14 @@ FootageLayer { src: string; muted?: boolean; dim?: number }
32
32
  Full-frame user footage. Place it once, outside the scenes, as the bottom layer. dim (0-1) darkens it for legibility.
33
33
  <FootageLayer src={urls[manifest.footageKey]} dim={0.25} />
34
34
 
35
+ ClipLayer { src: string; muted?: boolean; dim?: number }
36
+ Full-frame generated or library video clip as a scene's picture (treatment "clip"). Place it inside the scene's SceneFrame as the bottom layer, so it starts with the scene. When the scene is longer than the clip it holds the last frame; it never loops. dim (0-1) darkens it for legibility.
37
+ <ClipLayer src={urls[s.clipKey]} dim={0.2} />
38
+
39
+ KeyedClip { src: string; x?: number; y?: number; scale?: number; muted?: boolean }
40
+ A green-screen clip with the green keyed out (the transparent .webm, s.clipKeyedKey). Lay it over your own background or screenshot, below the captions. x and y place its bottom centre as fractions of the frame (default x 0.5, y 1: bottom centre); scale is its width as a fraction of the frame width (default 1). Holds its last frame when the scene is longer.
41
+ <KeyedClip src={urls[s.clipKeyedKey]} x={0.5} y={1} scale={0.8} />
42
+
35
43
  BgMesh { bg: string; hero: string; accent?: string }
36
44
  The bottom layer of the video: the base colour with two slow-drifting soft colour fields. Use it instead of a flat background.
37
45
  <BgMesh bg={palette.bg} hero={palette.hero} accent={palette.accent} />
@@ -1,4 +1,5 @@
1
1
  export { Captions } from "./Captions";
2
+ export { ClipLayer } from "./ClipLayer";
2
3
  export { Counter } from "./Counter";
3
4
  export { Entrance, WordReveal } from "./Entrance";
4
5
  export { FootageLayer } from "./FootageLayer";
@@ -6,6 +7,7 @@ export { brandColor, Icon } from "./Icon";
6
7
  export type { GlyphName, IconName } from "./Icon";
7
8
  export type { BrandName } from "./brand-icons";
8
9
  export { KenBurnsImage } from "./KenBurnsImage";
10
+ export { KeyedClip } from "./KeyedClip";
9
11
  export { BgMesh, Grade, Grain, Vignette } from "./Layers";
10
12
  export { LowerThird } from "./LowerThird";
11
13
  export { SceneFrame } from "./SceneFrame";
@@ -15,8 +15,13 @@ export type Harness = {
15
15
  // What the website does when a signed-in user enters the code.
16
16
  approve(userCode: string): unknown;
17
17
  close(): void | Promise<void>;
18
+ // Optional, for the clip tests. A prompt that makes a clip job fail (the fake fails any prompt containing "clip-fail"); without it the
19
+ // failing-job case is skipped.
20
+ clipFailPrompt?: string;
21
+ // How long to wait between two asks for a clip's status, in milliseconds: a server whose fake provider needs a moment sets it. Default 20.
22
+ clipPollMs?: number;
18
23
  };
19
- export type StartOptions = { voiceoverChars?: number; images?: number };
24
+ export type StartOptions = { voiceoverChars?: number; images?: number; clips?: number };
20
25
 
21
26
  const PNG = new Uint8Array(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"));
22
27
  const DATE = /\d{4}-\d{2}-\d{2}/;
@@ -75,6 +80,8 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
75
80
  expect(me.handle.length).toBeGreaterThan(0);
76
81
  expect(me.quota.voiceoverChars.used).toBe(0);
77
82
  expect(me.quota.images.used).toBe(0);
83
+ expect(me.quota.clips.used).toBe(0);
84
+ expect(me.quota.clips.limit).toBeGreaterThan(0);
78
85
  expect(me.quota.voiceoverChars.limit).toBeGreaterThan(0);
79
86
  expect(me.contributions).toBe(0);
80
87
  const resets = new Date(me.quota.resetsAt);
@@ -222,6 +229,10 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
222
229
  ["a voiceover text over 5000 characters", (api: Api) => api("voiceover", { text: "a".repeat(5001) })],
223
230
  ["a speed over 1.3", (api: Api) => api("voiceover", { text: "hi", speed: 1.4 })],
224
231
  ["an unknown aspect", (api: Api) => api("images", { prompt: "a", aspect: "4:3" as never, shareable: false, tags: [] })],
232
+ ["a clip with no prompt", (api: Api) => api("clipStart", { prompt: "", aspect: "9:16", shareable: false, tags: [] })],
233
+ ["a clip of 7 seconds", (api: Api) => api("clipStart", { prompt: "a", aspect: "9:16", durationSec: 7 as never, shareable: false, tags: [] })],
234
+ ["a clip prompt with a NUL", (api: Api) => api("clipStart", { prompt: "a\u0000b", aspect: "9:16", shareable: false, tags: [] })],
235
+ ["a clip status id with a slash", (api: Api) => api("clipStatus", { id: "../x" })],
225
236
  ])("refuses %s as invalid_request", async (_what, call) => {
226
237
  expect((await failure(call(await as("user-bad000001")))).code).toBe("invalid_request");
227
238
  });
@@ -269,6 +280,8 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
269
280
  it("generation needs a login", async () => {
270
281
  expect((await failure(anon("voiceover", { text: "hi" }))).code).toBe("unauthenticated");
271
282
  expect((await failure(anon("images", { prompt: "a", aspect: "9:16", shareable: false, tags: [] }))).code).toBe("unauthenticated");
283
+ expect((await failure(anon("clipStart", { prompt: "a", aspect: "9:16", shareable: false, tags: [] }))).code).toBe("unauthenticated");
284
+ expect((await failure(anon("clipStatus", { id: "clip-x" }))).code).toBe("unauthenticated");
272
285
  });
273
286
 
274
287
  it("images: a private one is only a file; a shareable one also enters the library for review, owned by the caller", async () => {
@@ -296,6 +309,95 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
296
309
  });
297
310
  });
298
311
 
312
+ describe("clips", () => {
313
+ // Asks until the job is no longer pending: at most 20 times, with the harness's pause between asks.
314
+ const settle = async (api: Api, id: string) => {
315
+ for (let i = 0; i < 20; i++) {
316
+ const st = await api("clipStatus", { id });
317
+ if (st.status !== "pending") return st;
318
+ await new Promise((r) => setTimeout(r, h.clipPollMs ?? 20));
319
+ }
320
+ throw new Error(`clip ${id} was still pending after 20 asks`);
321
+ };
322
+ const ask = { aspect: "9:16" as const, shareable: false, tags: [] as string[] };
323
+
324
+ it("start answers pending at once, the job ends done with a file that downloads, and the clip is charged to this user only", async () => {
325
+ const api = await as("user-clip00001"), other = await as("user-clip00002");
326
+ const started = await api("clipStart", { ...ask, prompt: "a dog running on a beach" });
327
+ expect(started.status).toBe("pending");
328
+ expect(started.id).toBeTruthy();
329
+ // The clip is reserved at the start, before it is finished.
330
+ expect((await api("me", {})).quota.clips.used).toBe(1);
331
+ const done = await settle(api, started.id);
332
+ expect(done).toMatchObject({ id: started.id, status: "done", ext: "mp4" });
333
+ expect(done.url).toBeTruthy();
334
+ expect(done.contentType).toMatch(/^video\//);
335
+ expect(done.message).toBeUndefined();
336
+ await download(done.url!, join(dest, "clip.mp4"));
337
+ expect(readFileSync(join(dest, "clip.mp4")).length).toBeGreaterThan(0);
338
+ expect((await api("me", {})).quota.clips.used).toBe(1);
339
+ expect((await other("me", {})).quota.clips.used).toBe(0);
340
+ });
341
+
342
+ it("a clip id belongs to its caller: another user, or an unknown id, is not_found", async () => {
343
+ const api = await as("user-clip00003"), other = await as("user-clip00004");
344
+ const started = await api("clipStart", { ...ask, prompt: "a quiet harbour at dawn" });
345
+ expect((await failure(other("clipStatus", { id: started.id }))).code).toBe("not_found");
346
+ expect((await failure(api("clipStatus", { id: "clip-nosuchjob" }))).code).toBe("not_found");
347
+ });
348
+
349
+ it("a green-screen clip is accepted and finishes like any other", async () => {
350
+ const api = await as("user-clip00005");
351
+ const started = await api("clipStart", { ...ask, prompt: "a person waving, flat green background", greenScreen: true, durationSec: 10 });
352
+ const done = await settle(api, started.id);
353
+ expect(done.status).toBe("done");
354
+ await download(done.url!, join(dest, "green.mp4"));
355
+ expect(readFileSync(join(dest, "green.mp4")).length).toBeGreaterThan(0);
356
+ });
357
+
358
+ it("a shareable clip enters the library in review, owned by the caller, and is not found by search", async () => {
359
+ const api = await as("user-clip00006"), other = await as("user-clip00007");
360
+ const started = await api("clipStart", { ...ask, prompt: "a clipshare lighthouse in a storm", shareable: true, tags: ["lighthouse"] });
361
+ const done = await settle(api, started.id);
362
+ expect(done.status).toBe("done");
363
+ expect(done.libraryId).toBeTruthy();
364
+ expect((await api("libraryPull", { id: done.libraryId! })).item).toMatchObject({ kind: "clip", visibility: "review" });
365
+ expect((await api("librarySearch", { q: "clipshare lighthouse" })).items.map((i) => i.id)).not.toContain(done.libraryId);
366
+ expect((await failure(other("libraryPull", { id: done.libraryId! }))).code).toBe("not_found");
367
+ const plain = await settle(api, (await api("clipStart", { ...ask, prompt: "a private clip" })).id);
368
+ expect(plain.libraryId).toBeUndefined();
369
+ });
370
+
371
+ it("a failed job says why in one line and is not charged", async () => {
372
+ if (!h.clipFailPrompt) return;
373
+ const api = await as("user-clip00008");
374
+ const started = await api("clipStart", { ...ask, prompt: h.clipFailPrompt });
375
+ const st = await settle(api, started.id);
376
+ expect(st.status).toBe("failed");
377
+ expect(st.message?.length).toBeGreaterThan(0);
378
+ expect(st.message).not.toContain("\n");
379
+ expect(st.url).toBeUndefined();
380
+ expect((await api("me", {})).quota.clips.used).toBe(0);
381
+ });
382
+
383
+ it("at the limit a new clip is quota_exceeded with the reset date, charged nothing; other users are unaffected", async () => {
384
+ const t = await start({ clips: 1 });
385
+ try {
386
+ const api = createClient({ baseUrl: t.baseUrl, token: await t.login("user-clipq0001") });
387
+ const other = createClient({ baseUrl: t.baseUrl, token: await t.login("user-clipq0002") });
388
+ await api("clipStart", { ...ask, prompt: "first" });
389
+ const e = await failure(api("clipStart", { ...ask, prompt: "second" }));
390
+ expect(e.code).toBe("quota_exceeded");
391
+ expect(e.message).toMatch(DATE);
392
+ const me = await api("me", {});
393
+ expect(me.quota.clips).toEqual({ used: 1, limit: 1 });
394
+ expect(e.message).toContain(me.quota.resetsAt.slice(0, 10));
395
+ expect((await other("me", {})).quota.clips.used).toBe(0);
396
+ await other("clipStart", { ...ask, prompt: "third" });
397
+ } finally { await t.close(); }
398
+ });
399
+ });
400
+
299
401
  describe("quota", () => {
300
402
  it("passes exactly at the limit; one over is quota_exceeded with the reset date; other users are unaffected", async () => {
301
403
  const t = await start({ voiceoverChars: 10, images: 1 });
@@ -1,11 +1,17 @@
1
+ import { execFileSync } from "node:child_process";
1
2
  import { randomBytes } from "node:crypto";
3
+ import { mkdtempSync, readFileSync, rmSync } from "node:fs";
2
4
  import { createServer } from "node:http";
3
5
  import type { AddressInfo } from "node:net";
6
+ import { tmpdir } from "node:os";
7
+ import { join } from "node:path";
4
8
  import { MAX_UPLOAD_BYTES, routes, type ErrorCode, type LibraryItem, type RouteName, type Voice } from "../contract";
5
9
  import { PIXEL_PNG, silentWav } from "./fixtures";
6
10
 
7
11
  type Blob = { filename: string; contentType: string; bytes: Uint8Array };
8
12
  // owner is the uploader's user id; a seeded item has none. An upload is not an item for anyone until it is committed.
13
+ // A clip of ten seconds or less is previewed by its own file, like a picture or a sound effect.
14
+ const shortClip = (item: LibraryItem) => item.kind === "clip" && typeof item.meta.durationSec === "number" && item.meta.durationSec <= 10;
9
15
  type Stored = { item: LibraryItem; file?: Blob; filename: string; contentType: string; owner?: string; committed: boolean };
10
16
 
11
17
  export const FAKE_VOICES: Voice[] = [
@@ -37,10 +43,29 @@ function parseBody(raw: Buffer): unknown {
37
43
  catch { throw new Fail(400, "invalid_request", "The request body is not valid JSON."); }
38
44
  }
39
45
 
46
+ // A clip job: pending for its first status call, then done (or failed, when the prompt says so). `charged` is whether it still holds its reservation.
47
+ type ClipJob = { owner: string; prompt: string; aspect: string; greenScreen: boolean; durationSec: number; shareable: boolean; tags: string[]; asks: number; charged: boolean; libraryId?: string };
48
+ const CLIP_FAIL = "clip-fail";
49
+ // A prompt with this never finishes, so a test can see a client give up waiting.
50
+ const CLIP_STALL = "clip-stall";
51
+
52
+ // A real one-second MP4, made with ffmpeg. A green-screen one is pure green with a small red box, so keying it out has something to show.
53
+ function makeClip(greenScreen: boolean): Blob {
54
+ const dir = mkdtempSync(join(tmpdir(), "rk-fakeclip-"));
55
+ const out = join(dir, "clip.mp4");
56
+ try {
57
+ const source = greenScreen ? "color=c=0x00FF00:s=320x180:r=10:d=1,drawbox=x=120:y=50:w=80:h=80:color=0xFF0000:t=fill" : "testsrc=s=320x180:r=10:d=1";
58
+ execFileSync("ffmpeg", ["-v", "error", "-f", "lavfi", "-i", source, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", out], { stdio: "pipe" });
59
+ return { filename: "clip.mp4", contentType: "video/mp4", bytes: new Uint8Array(readFileSync(out)) };
60
+ } catch (e) {
61
+ throw new Error(`The fake API could not make its test clip: ffmpeg is needed (${e instanceof Error ? e.message : String(e)}).`);
62
+ } finally { rmSync(dir, { recursive: true, force: true }); }
63
+ }
64
+
40
65
  export type FakeApi = Awaited<ReturnType<typeof startFakeApi>>;
41
66
 
42
67
  // An in-memory stand-in for the Reelkit API. It implements every route in the contract.
43
- export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLimit?: number; autoApprove?: boolean } = {}) {
68
+ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLimit?: number; clipLimit?: number; noClipProvider?: boolean; autoApprove?: boolean } = {}) {
44
69
  // token -> the user it belongs to
45
70
  const tokens = new Map<string, string>();
46
71
  const devices = new Map<string, { userCode: string; approved: boolean; userId: string }>();
@@ -51,15 +76,19 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
51
76
  // real server needs to be, so a client that retries a PUT, or sends a second one, fails here instead of working by luck.
52
77
  const uploads = new Map<string, { id: string; contentType: string; bytes: number; used: boolean }>();
53
78
  // Totals over every user (for tests of the CLI), and the same counts per user (what a quota is measured against).
54
- const usage = { chars: 0, images: 0, pulls: 0, uploads: 0 };
55
- const meters = new Map<string, { chars: number; images: number; uploads: number }>();
79
+ const usage = { chars: 0, images: 0, clips: 0, pulls: 0, uploads: 0 };
80
+ const meters = new Map<string, { chars: number; images: number; clips: number; uploads: number }>();
56
81
  const meter = (userId: string | undefined) => {
57
82
  const id = userId ?? "u-test";
58
- if (!meters.has(id)) meters.set(id, { chars: 0, images: 0, uploads: 0 });
83
+ if (!meters.has(id)) meters.set(id, { chars: 0, images: 0, clips: 0, uploads: 0 });
59
84
  return meters.get(id)!;
60
85
  };
61
86
  const resetDate = () => nextMonthStart().toISOString();
62
- const limits = { chars: opts.voiceoverCharLimit ?? 10_000, images: opts.imageLimit ?? 30 };
87
+ const limits = { chars: opts.voiceoverCharLimit ?? 10_000, images: opts.imageLimit ?? 30, clips: opts.clipLimit ?? 5 };
88
+ const jobs = new Map<string, ClipJob>();
89
+ // Made on first use and kept for the life of this instance.
90
+ const clipFiles: Partial<Record<"plain" | "green", Blob>> = {};
91
+ const clipFile = (green: boolean) => (clipFiles[green ? "green" : "plain"] ??= makeClip(green));
63
92
  const ALNUM = "abcdefghijklmnopqrstuvwxyz0123456789";
64
93
  const nextId = (prefix: string) => `${prefix}-${Array.from(randomBytes(12), (b) => ALNUM[b % ALNUM.length]).join("")}`;
65
94
  const secret = () => randomBytes(16).toString("hex");
@@ -88,7 +117,7 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
88
117
  const m = meter(userId);
89
118
  return {
90
119
  userId: userId ?? "u-test", handle: handleOf(userId ?? "u-test"),
91
- quota: { voiceoverChars: { used: m.chars, limit: limits.chars }, images: { used: m.images, limit: limits.images }, resetsAt: resetDate() },
120
+ quota: { voiceoverChars: { used: m.chars, limit: limits.chars }, images: { used: m.images, limit: limits.images }, clips: { used: m.clips, limit: limits.clips }, resetsAt: resetDate() },
92
121
  // What the user gave the shared library that was accepted: their own items that are published. One waiting for review does not count.
93
122
  contributions: [...store.values()].filter((x) => x.owner === userId && x.committed && x.item.visibility === "published").length,
94
123
  };
@@ -155,6 +184,37 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
155
184
  if (shareable) store.set(id, { item: { id, kind: "image", title: cut(prompt, 60), description: prompt, tags, meta: { aspect }, visibility: "review" }, file, filename: file.filename, contentType: file.contentType, owner: userId, committed: true });
156
185
  return { url: fileUrl(file), ext: "png", contentType: "image/png", ...(shareable ? { libraryId: id } : {}) };
157
186
  },
187
+ clipStart: (input, { userId }) => {
188
+ if (opts.noClipProvider) throw new Fail(400, "invalid_request", "Clip generation is not available on this server yet.");
189
+ const m = meter(userId);
190
+ if (m.clips + 1 > limits.clips) throw new Fail(429, "quota_exceeded", `Clip quota used up. It resets on ${resetDate().slice(0, 10)}.`);
191
+ m.clips++;
192
+ usage.clips++;
193
+ const id = nextId("clip");
194
+ jobs.set(id, { owner: userId ?? "u-test", prompt: input.prompt, aspect: input.aspect, greenScreen: input.greenScreen, durationSec: input.durationSec, shareable: input.shareable, tags: input.tags, asks: 0, charged: true });
195
+ return { id, status: "pending" };
196
+ },
197
+ clipStatus: ({ id }, { userId }) => {
198
+ const job = jobs.get(id);
199
+ if (!job || job.owner !== (userId ?? "u-test")) throw new Fail(404, "not_found", `No clip with id ${id}.`);
200
+ // The first ask finds it still working; from the second it has ended.
201
+ if (++job.asks === 1 || job.prompt.includes(CLIP_STALL)) return { id, status: "pending" };
202
+ if (job.prompt.includes(CLIP_FAIL)) {
203
+ // A failed clip is not charged: the reservation is given back, once.
204
+ if (job.charged) { job.charged = false; meter(userId).clips--; usage.clips--; }
205
+ return { id, status: "failed", message: "The video provider could not make this clip. Change the prompt and try again." };
206
+ }
207
+ const file = clipFile(job.greenScreen);
208
+ if (job.shareable && !job.libraryId) {
209
+ job.libraryId = nextId("clip");
210
+ const meta = { aspect: job.aspect, durationSec: job.durationSec, ...(job.greenScreen ? { greenScreen: true } : {}) };
211
+ store.set(job.libraryId, { item: { id: job.libraryId, kind: "clip", title: cut(job.prompt, 60), description: job.prompt, tags: job.tags, meta, visibility: "review" }, file, filename: file.filename, contentType: file.contentType, owner: userId, committed: true });
212
+ }
213
+ return {
214
+ id, status: "done", url: fileUrl(file), ext: "mp4", contentType: file.contentType, durationSec: job.durationSec, greenScreen: job.greenScreen,
215
+ ...(job.libraryId ? { libraryId: job.libraryId } : {}),
216
+ };
217
+ },
158
218
  publicLibrary: ({ q, kind, page }) => {
159
219
  // Published items only. Newest first; with a query, best match first and equal matches newest first.
160
220
  const want = q ? words(q) : undefined;
@@ -164,7 +224,7 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
164
224
  .filter((x) => x.s.item.visibility === "published" && (!kind || x.s.item.kind === kind) && (!want || x.score > 0))
165
225
  .sort((a, b) => b.score - a.score || b.order - a.order).map((x) => x.s);
166
226
  const p = page ?? 1, size = 24;
167
- const items = all.slice((p - 1) * size, p * size).map((s) => (s.file && PREVIEW_OF_ORIGINAL.includes(s.item.kind) ? { ...s.item, previewUrl: fileUrl(s.file) } : s.item));
227
+ const items = all.slice((p - 1) * size, p * size).map((s) => (s.file && (PREVIEW_OF_ORIGINAL.includes(s.item.kind) || shortClip(s.item)) ? { ...s.item, previewUrl: fileUrl(s.file) } : s.item));
168
228
  return { items, page: p, hasMore: all.length > p * size };
169
229
  },
170
230
  };
@@ -219,6 +279,8 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
219
279
  return {
220
280
  baseUrl: `${origin}/api/v1`,
221
281
  usage,
282
+ // Changes how many clips a user may start from now on, as `clipLimit` does at the start.
283
+ setClipLimit(n: number) { limits.clips = n; },
222
284
  approve(userCode: string, userId = "u-test") { for (const d of devices.values()) if (d.userCode === userCode) { d.approved = true; d.userId = userId; } },
223
285
  // A ready token, for tests that are not about logging in.
224
286
  login(userId = "u-test") { const t = nextId("tok"); tokens.set(t, userId); return t; },