vargai 0.4.0-alpha.1 → 0.4.0-alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -5,9 +5,46 @@ AI video generation from your terminal.
5
5
  ## Quick Start
6
6
 
7
7
  ```bash
8
- bun install @vargai/sdk
8
+ bun install vargai ai
9
9
  ```
10
10
 
11
+ ### SDK Usage
12
+
13
+ ```typescript
14
+ import { generateImage } from "ai";
15
+ import { File, fal, generateElement, generateVideo, scene } from "vargai";
16
+
17
+ // generate a character from reference image
18
+ const { element: character } = await generateElement({
19
+ model: fal.imageModel("nano-banana-pro/edit"),
20
+ type: "character",
21
+ prompt: {
22
+ text: "cartoon character, simple style",
23
+ images: [await File.fromPath("media/reference.jpg").arrayBuffer()],
24
+ },
25
+ });
26
+
27
+ // generate scene with character
28
+ const { image: frame } = await generateImage({
29
+ model: fal.imageModel("nano-banana-pro"),
30
+ prompt: scene`${character} walks through a forest`,
31
+ });
32
+
33
+ // animate the frame
34
+ const { video } = await generateVideo({
35
+ model: fal.videoModel("wan-2.5"),
36
+ prompt: {
37
+ text: `${character.text} walks through a forest`,
38
+ images: [frame.base64],
39
+ },
40
+ duration: 5,
41
+ });
42
+
43
+ await Bun.write("output/scene.mp4", video.uint8Array);
44
+ ```
45
+
46
+ ### CLI Usage
47
+
11
48
  ```bash
12
49
  varg run image --prompt "cyberpunk cityscape at night"
13
50
  varg run video --prompt "camera flies through clouds" --duration 5
package/package.json CHANGED
@@ -33,7 +33,8 @@
33
33
  "lint-staged": "^16.2.7"
34
34
  },
35
35
  "peerDependencies": {
36
- "typescript": "^5"
36
+ "typescript": "^5",
37
+ "ai": "^6.0.0"
37
38
  },
38
39
  "dependencies": {
39
40
  "@ai-sdk/fal": "^1.0.23",
@@ -62,7 +63,7 @@
62
63
  "replicate": "^1.4.0",
63
64
  "zod": "^4.2.1"
64
65
  },
65
- "version": "0.4.0-alpha.1",
66
+ "version": "0.4.0-alpha2",
66
67
  "exports": {
67
68
  ".": "./src/index.ts",
68
69
  "./ai": "./src/ai-sdk/index.ts",
@@ -25,6 +25,25 @@ export {
25
25
  type GenerateVideoResult,
26
26
  generateVideo,
27
27
  } from "./generate-video";
28
+ export {
29
+ generatePlaceholder,
30
+ type ImagePlaceholderFallbackOptions,
31
+ imagePlaceholderFallbackMiddleware,
32
+ type MusicModelMiddleware,
33
+ type MusicPlaceholderFallbackOptions,
34
+ musicPlaceholderFallbackMiddleware,
35
+ type PlaceholderFallbackOptions,
36
+ type PlaceholderOptions,
37
+ type PlaceholderResult,
38
+ placeholderFallbackMiddleware,
39
+ type RenderMode,
40
+ type VideoModelMiddleware,
41
+ withImagePlaceholderFallback,
42
+ withMusicPlaceholderFallback,
43
+ withPlaceholderFallback,
44
+ wrapMusicModel,
45
+ wrapVideoModel,
46
+ } from "./middleware";
28
47
  export type {
29
48
  MusicModelV3,
30
49
  MusicModelV3CallOptions,
@@ -42,8 +61,8 @@ export {
42
61
  type ElevenLabsProvider,
43
62
  elevenlabs,
44
63
  VOICES,
45
- } from "./providers/elevenlabs-provider";
46
- export { createFal, type FalProvider, fal } from "./providers/fal-provider";
64
+ } from "./providers/elevenlabs";
65
+ export { createFal, type FalProvider, fal } from "./providers/fal";
47
66
  export {
48
67
  createHiggsfield,
49
68
  type HiggsfieldImageModelSettings,
@@ -0,0 +1,25 @@
1
+ export {
2
+ generatePlaceholder,
3
+ type PlaceholderOptions,
4
+ type PlaceholderResult,
5
+ } from "./placeholder";
6
+ export {
7
+ type ImagePlaceholderFallbackOptions,
8
+ imagePlaceholderFallbackMiddleware,
9
+ withImagePlaceholderFallback,
10
+ } from "./wrap-image-model";
11
+ export {
12
+ type MusicModelMiddleware,
13
+ type MusicPlaceholderFallbackOptions,
14
+ musicPlaceholderFallbackMiddleware,
15
+ withMusicPlaceholderFallback,
16
+ wrapMusicModel,
17
+ } from "./wrap-music-model";
18
+ export {
19
+ type PlaceholderFallbackOptions,
20
+ placeholderFallbackMiddleware,
21
+ type RenderMode,
22
+ type VideoModelMiddleware,
23
+ withPlaceholderFallback,
24
+ wrapVideoModel,
25
+ } from "./wrap-video-model";
@@ -0,0 +1,111 @@
1
+ import { unlink } from "node:fs/promises";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import { $ } from "bun";
5
+
6
+ export interface PlaceholderOptions {
7
+ type: "image" | "video" | "audio";
8
+ prompt: string;
9
+ duration?: number;
10
+ width?: number;
11
+ height?: number;
12
+ }
13
+
14
+ export interface PlaceholderResult {
15
+ data: Uint8Array;
16
+ placeholder: true;
17
+ }
18
+
19
+ function promptToColor(prompt: string): string {
20
+ let hash = 0;
21
+ for (let i = 0; i < prompt.length; i++) {
22
+ const char = prompt.charCodeAt(i);
23
+ hash = (hash << 5) - hash + char;
24
+ hash = hash & hash;
25
+ }
26
+
27
+ const h = Math.abs(hash) % 360;
28
+ const s = 40 + (Math.abs(hash >> 8) % 30);
29
+ const l = 35 + (Math.abs(hash >> 16) % 20);
30
+
31
+ return `hsl(${h},${s}%,${l}%)`;
32
+ }
33
+
34
+ function hslToHex(hsl: string): string {
35
+ const match = hsl.match(/hsl\((\d+),(\d+)%,(\d+)%\)/);
36
+ if (!match) return "333333";
37
+
38
+ const h = Number.parseInt(match[1]!) / 360;
39
+ const s = Number.parseInt(match[2]!) / 100;
40
+ const l = Number.parseInt(match[3]!) / 100;
41
+
42
+ const hue2rgb = (p: number, q: number, t: number) => {
43
+ if (t < 0) t += 1;
44
+ if (t > 1) t -= 1;
45
+ if (t < 1 / 6) return p + (q - p) * 6 * t;
46
+ if (t < 1 / 2) return q;
47
+ if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6;
48
+ return p;
49
+ };
50
+
51
+ const q = l < 0.5 ? l * (1 + s) : l + s - l * s;
52
+ const p = 2 * l - q;
53
+ const r = Math.round(hue2rgb(p, q, h + 1 / 3) * 255);
54
+ const g = Math.round(hue2rgb(p, q, h) * 255);
55
+ const b = Math.round(hue2rgb(p, q, h - 1 / 3) * 255);
56
+
57
+ return `${r.toString(16).padStart(2, "0")}${g.toString(16).padStart(2, "0")}${b.toString(16).padStart(2, "0")}`;
58
+ }
59
+
60
+ function truncatePrompt(text: string, maxLen: number): string {
61
+ const clean = text.replace(/[^a-zA-Z0-9 .,!?-]/g, "");
62
+ if (clean.length <= maxLen) return clean;
63
+ return `${clean.slice(0, maxLen - 3)}...`;
64
+ }
65
+
66
+ export async function generatePlaceholder(
67
+ options: PlaceholderOptions,
68
+ ): Promise<PlaceholderResult> {
69
+ const { type, prompt, duration = 3, width = 1080, height = 1920 } = options;
70
+
71
+ const color = promptToColor(prompt);
72
+ const hexColor = hslToHex(color);
73
+ const labelFontSize = Math.floor(Math.min(width, height) / 20);
74
+ const promptFontSize = Math.floor(Math.min(width, height) / 35);
75
+ const maxChars = Math.floor((width * 0.7) / (promptFontSize * 0.5));
76
+ const typeLabel = type.toUpperCase();
77
+ const promptText = truncatePrompt(prompt, maxChars);
78
+
79
+ const ext = type === "audio" ? "mp3" : type === "image" ? "png" : "mp4";
80
+ const outputPath = join(
81
+ tmpdir(),
82
+ `placeholder_${Date.now()}_${Math.random().toString(36).slice(2)}.${ext}`,
83
+ );
84
+
85
+ try {
86
+ if (type === "audio") {
87
+ await $`ffmpeg -y -f lavfi -i anullsrc=r=44100:cl=stereo -t ${duration} -c:a libmp3lame ${outputPath}`.quiet();
88
+ } else if (type === "image") {
89
+ const colorInput = `color=c=0x${hexColor}:s=${width}x${height}:d=1`;
90
+ const labelY = `(h/2)-${labelFontSize}`;
91
+ const promptY = `(h/2)+${Math.floor(labelFontSize * 0.5)}`;
92
+ const drawLabel = `drawtext=text='${typeLabel}':fontcolor=white:fontsize=${labelFontSize}:x=(w-text_w)/2:y=${labelY}`;
93
+ const drawPrompt = `drawtext=text='${promptText}':fontcolor=white@0.7:fontsize=${promptFontSize}:x=(w-text_w)/2:y=${promptY}`;
94
+ await $`ffmpeg -y -f lavfi -i ${colorInput} -vf ${drawLabel},${drawPrompt} -frames:v 1 -update 1 ${outputPath}`.quiet();
95
+ } else {
96
+ const colorInput = `color=c=0x${hexColor}:s=${width}x${height}:d=${duration}:r=30`;
97
+ const labelY = `(h/2)-${labelFontSize}`;
98
+ const promptY = `(h/2)+${Math.floor(labelFontSize * 0.5)}`;
99
+ const drawLabel = `drawtext=text='${typeLabel}':fontcolor=white:fontsize=${labelFontSize}:x=(w-text_w)/2:y=${labelY}`;
100
+ const drawPrompt = `drawtext=text='${promptText}':fontcolor=white@0.7:fontsize=${promptFontSize}:x=(w-text_w)/2:y=${promptY}`;
101
+ await $`ffmpeg -y -f lavfi -i ${colorInput} -vf ${drawLabel},${drawPrompt} -c:v libx264 -preset ultrafast -pix_fmt yuv420p ${outputPath}`.quiet();
102
+ }
103
+
104
+ const data = await Bun.file(outputPath).bytes();
105
+ await unlink(outputPath).catch(() => {});
106
+ return { data: new Uint8Array(data), placeholder: true };
107
+ } catch (e) {
108
+ await unlink(outputPath).catch(() => {});
109
+ throw e;
110
+ }
111
+ }
@@ -0,0 +1,86 @@
1
+ import type {
2
+ ImageModelV3,
3
+ ImageModelV3CallOptions,
4
+ ImageModelV3Middleware,
5
+ } from "@ai-sdk/provider";
6
+ import { wrapImageModel } from "ai";
7
+ import { generatePlaceholder } from "./placeholder";
8
+ import type { RenderMode } from "./wrap-video-model";
9
+
10
+ export interface ImagePlaceholderFallbackOptions {
11
+ mode: RenderMode;
12
+ onFallback?: (error: Error, prompt: string) => void;
13
+ }
14
+
15
+ export function imagePlaceholderFallbackMiddleware(
16
+ options: ImagePlaceholderFallbackOptions,
17
+ ): ImageModelV3Middleware {
18
+ const { mode, onFallback } = options;
19
+
20
+ return {
21
+ specificationVersion: "v3",
22
+ wrapGenerate: async ({ doGenerate, params, model }) => {
23
+ const createPlaceholderResult = async () => {
24
+ const [width, height] = (params.size?.split("x").map(Number) ?? [
25
+ 1024, 1024,
26
+ ]) as [number, number];
27
+ const prompt =
28
+ typeof params.prompt === "string"
29
+ ? params.prompt
30
+ : ((params.prompt as { text?: string } | undefined)?.text ??
31
+ "placeholder");
32
+
33
+ const placeholder = await generatePlaceholder({
34
+ type: "image",
35
+ prompt,
36
+ width,
37
+ height,
38
+ });
39
+
40
+ return {
41
+ images: [placeholder.data],
42
+ warnings: [
43
+ {
44
+ type: "other" as const,
45
+ message: "placeholder: provider skipped or failed",
46
+ },
47
+ ],
48
+ response: {
49
+ timestamp: new Date(),
50
+ modelId: model.modelId,
51
+ headers: undefined,
52
+ },
53
+ };
54
+ };
55
+
56
+ if (mode === "preview") {
57
+ return createPlaceholderResult();
58
+ }
59
+
60
+ try {
61
+ return await doGenerate();
62
+ } catch (e) {
63
+ if (mode === "strict") throw e;
64
+
65
+ const error = e instanceof Error ? e : new Error(String(e));
66
+ const promptText =
67
+ typeof params.prompt === "string"
68
+ ? params.prompt
69
+ : ((params.prompt as { text?: string } | undefined)?.text ??
70
+ "placeholder");
71
+ onFallback?.(error, promptText);
72
+ return createPlaceholderResult();
73
+ }
74
+ },
75
+ };
76
+ }
77
+
78
+ export function withImagePlaceholderFallback(
79
+ model: ImageModelV3,
80
+ options: ImagePlaceholderFallbackOptions,
81
+ ): ImageModelV3 {
82
+ return wrapImageModel({
83
+ model,
84
+ middleware: imagePlaceholderFallbackMiddleware(options),
85
+ });
86
+ }
@@ -0,0 +1,108 @@
1
+ import type { MusicModelV3, MusicModelV3CallOptions } from "../music-model";
2
+ import { generatePlaceholder } from "./placeholder";
3
+ import type { RenderMode } from "./wrap-video-model";
4
+
5
+ export interface MusicModelMiddleware {
6
+ transformParams?: (options: {
7
+ params: MusicModelV3CallOptions;
8
+ model: MusicModelV3;
9
+ }) => PromiseLike<MusicModelV3CallOptions> | MusicModelV3CallOptions;
10
+
11
+ wrapGenerate?: (options: {
12
+ doGenerate: () => PromiseLike<
13
+ Awaited<ReturnType<MusicModelV3["doGenerate"]>>
14
+ >;
15
+ params: MusicModelV3CallOptions;
16
+ model: MusicModelV3;
17
+ }) => PromiseLike<Awaited<ReturnType<MusicModelV3["doGenerate"]>>>;
18
+ }
19
+
20
+ export function wrapMusicModel({
21
+ model,
22
+ middleware,
23
+ }: {
24
+ model: MusicModelV3;
25
+ middleware: MusicModelMiddleware;
26
+ }): MusicModelV3 {
27
+ const { transformParams, wrapGenerate } = middleware;
28
+
29
+ return {
30
+ specificationVersion: "v3",
31
+ provider: model.provider,
32
+ modelId: model.modelId,
33
+
34
+ async doGenerate(params: MusicModelV3CallOptions) {
35
+ const transformedParams = transformParams
36
+ ? await transformParams({ params, model })
37
+ : params;
38
+
39
+ const doGenerate = () => model.doGenerate(transformedParams);
40
+
41
+ return wrapGenerate
42
+ ? wrapGenerate({ doGenerate, params: transformedParams, model })
43
+ : doGenerate();
44
+ },
45
+ };
46
+ }
47
+
48
+ export interface MusicPlaceholderFallbackOptions {
49
+ mode: RenderMode;
50
+ onFallback?: (error: Error, prompt: string) => void;
51
+ }
52
+
53
+ export function musicPlaceholderFallbackMiddleware(
54
+ options: MusicPlaceholderFallbackOptions,
55
+ ): MusicModelMiddleware {
56
+ const { mode, onFallback } = options;
57
+
58
+ return {
59
+ wrapGenerate: async ({ doGenerate, params, model }) => {
60
+ const createPlaceholderResult = async () => {
61
+ const placeholder = await generatePlaceholder({
62
+ type: "audio",
63
+ prompt: params.prompt,
64
+ duration: params.duration ?? 10,
65
+ });
66
+
67
+ return {
68
+ audio: placeholder.data,
69
+ warnings: [
70
+ {
71
+ type: "other" as const,
72
+ message: "placeholder: provider skipped or failed",
73
+ },
74
+ ],
75
+ response: {
76
+ timestamp: new Date(),
77
+ modelId: model.modelId,
78
+ headers: undefined,
79
+ },
80
+ };
81
+ };
82
+
83
+ if (mode === "preview") {
84
+ return createPlaceholderResult();
85
+ }
86
+
87
+ try {
88
+ return await doGenerate();
89
+ } catch (e) {
90
+ if (mode === "strict") throw e;
91
+
92
+ const error = e instanceof Error ? e : new Error(String(e));
93
+ onFallback?.(error, params.prompt);
94
+ return createPlaceholderResult();
95
+ }
96
+ },
97
+ };
98
+ }
99
+
100
+ export function withMusicPlaceholderFallback(
101
+ model: MusicModelV3,
102
+ options: MusicPlaceholderFallbackOptions,
103
+ ): MusicModelV3 {
104
+ return wrapMusicModel({
105
+ model,
106
+ middleware: musicPlaceholderFallbackMiddleware(options),
107
+ });
108
+ }
@@ -0,0 +1,115 @@
1
+ import type { VideoModelV3, VideoModelV3CallOptions } from "../video-model";
2
+ import { generatePlaceholder } from "./placeholder";
3
+
4
+ export type RenderMode = "strict" | "default" | "preview";
5
+
6
+ export interface VideoModelMiddleware {
7
+ transformParams?: (options: {
8
+ params: VideoModelV3CallOptions;
9
+ model: VideoModelV3;
10
+ }) => PromiseLike<VideoModelV3CallOptions> | VideoModelV3CallOptions;
11
+
12
+ wrapGenerate?: (options: {
13
+ doGenerate: () => PromiseLike<
14
+ Awaited<ReturnType<VideoModelV3["doGenerate"]>>
15
+ >;
16
+ params: VideoModelV3CallOptions;
17
+ model: VideoModelV3;
18
+ }) => PromiseLike<Awaited<ReturnType<VideoModelV3["doGenerate"]>>>;
19
+ }
20
+
21
+ export function wrapVideoModel({
22
+ model,
23
+ middleware,
24
+ }: {
25
+ model: VideoModelV3;
26
+ middleware: VideoModelMiddleware;
27
+ }): VideoModelV3 {
28
+ const { transformParams, wrapGenerate } = middleware;
29
+
30
+ return {
31
+ specificationVersion: "v3",
32
+ provider: model.provider,
33
+ modelId: model.modelId,
34
+ maxVideosPerCall: model.maxVideosPerCall,
35
+
36
+ async doGenerate(params: VideoModelV3CallOptions) {
37
+ const transformedParams = transformParams
38
+ ? await transformParams({ params, model })
39
+ : params;
40
+
41
+ const doGenerate = () => model.doGenerate(transformedParams);
42
+
43
+ return wrapGenerate
44
+ ? wrapGenerate({ doGenerate, params: transformedParams, model })
45
+ : doGenerate();
46
+ },
47
+ };
48
+ }
49
+
50
+ export interface PlaceholderFallbackOptions {
51
+ mode: RenderMode;
52
+ onFallback?: (error: Error, prompt: string) => void;
53
+ }
54
+
55
+ export function placeholderFallbackMiddleware(
56
+ options: PlaceholderFallbackOptions,
57
+ ): VideoModelMiddleware {
58
+ const { mode, onFallback } = options;
59
+
60
+ return {
61
+ wrapGenerate: async ({ doGenerate, params, model }) => {
62
+ const createPlaceholderResult = async () => {
63
+ const [width, height] = (params.resolution?.split("x").map(Number) ?? [
64
+ 1080, 1920,
65
+ ]) as [number, number];
66
+ const placeholder = await generatePlaceholder({
67
+ type: "video",
68
+ prompt: params.prompt,
69
+ duration: params.duration ?? 3,
70
+ width,
71
+ height,
72
+ });
73
+
74
+ return {
75
+ videos: [placeholder.data],
76
+ warnings: [
77
+ {
78
+ type: "other" as const,
79
+ message: "placeholder: provider skipped or failed",
80
+ },
81
+ ],
82
+ response: {
83
+ timestamp: new Date(),
84
+ modelId: model.modelId,
85
+ headers: undefined,
86
+ },
87
+ };
88
+ };
89
+
90
+ if (mode === "preview") {
91
+ return createPlaceholderResult();
92
+ }
93
+
94
+ try {
95
+ return await doGenerate();
96
+ } catch (e) {
97
+ if (mode === "strict") throw e;
98
+
99
+ const error = e instanceof Error ? e : new Error(String(e));
100
+ onFallback?.(error, params.prompt);
101
+ return createPlaceholderResult();
102
+ }
103
+ },
104
+ };
105
+ }
106
+
107
+ export function withPlaceholderFallback(
108
+ model: VideoModelV3,
109
+ options: PlaceholderFallbackOptions,
110
+ ): VideoModelV3 {
111
+ return wrapVideoModel({
112
+ model,
113
+ middleware: placeholderFallbackMiddleware(options),
114
+ });
115
+ }
@@ -1,6 +1,6 @@
1
1
  import { defineCommand } from "citty";
2
2
  import { render } from "../../react/render";
3
- import type { VargElement } from "../../react/types";
3
+ import type { RenderMode, VargElement } from "../../react/types";
4
4
 
5
5
  export const renderCmd = defineCommand({
6
6
  meta: {
@@ -30,6 +30,16 @@ export const renderCmd = defineCommand({
30
30
  description: "minimal output",
31
31
  default: false,
32
32
  },
33
+ strict: {
34
+ type: "boolean",
35
+ description: "fail on provider errors (no fallback)",
36
+ default: false,
37
+ },
38
+ preview: {
39
+ type: "boolean",
40
+ description: "skip all generation, use placeholders only",
41
+ default: false,
42
+ },
33
43
  },
34
44
  async run({ args }) {
35
45
  const file = args.file as string;
@@ -55,13 +65,26 @@ export const renderCmd = defineCommand({
55
65
  .split("/")
56
66
  .pop()}.mp4`;
57
67
 
68
+ const mode: RenderMode = args.strict
69
+ ? "strict"
70
+ : args.preview
71
+ ? "preview"
72
+ : "default";
73
+
58
74
  if (!args.quiet) {
59
- console.log(`rendering ${file} → ${outputPath}`);
75
+ const modeLabel =
76
+ mode === "preview"
77
+ ? " (preview)"
78
+ : mode === "strict"
79
+ ? " (strict)"
80
+ : "";
81
+ console.log(`rendering ${file} → ${outputPath}${modeLabel}`);
60
82
  }
61
83
 
62
84
  const buffer = await render(component, {
63
85
  output: outputPath,
64
86
  cache: args.cache,
87
+ mode,
65
88
  });
66
89
 
67
90
  if (!args.quiet) {
@@ -1,5 +1,5 @@
1
- import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
2
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { elevenlabs } from "../../ai-sdk/providers/elevenlabs";
2
+ import { fal } from "../../ai-sdk/providers/fal";
3
3
  import { Animate, Clip, Image, Render, Speech, Title } from "..";
4
4
 
5
5
  // Non-linear tree: multiple clips with independent branches
@@ -1,4 +1,4 @@
1
- import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
1
+ import { elevenlabs } from "../../ai-sdk/providers/elevenlabs";
2
2
  import { Captions, Clip, Image, Render, render, Speech } from "..";
3
3
 
4
4
  async function main() {
@@ -1,5 +1,5 @@
1
- import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
2
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { elevenlabs } from "../../ai-sdk/providers/elevenlabs";
2
+ import { fal } from "../../ai-sdk/providers/fal";
3
3
  import { Animate, Clip, Image, Music, Render, render } from "..";
4
4
 
5
5
  const MADI_REF =
@@ -1,4 +1,4 @@
1
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { fal } from "../../ai-sdk/providers/fal";
2
2
  import { Clip, Grid, Image, Render, render, Title } from "..";
3
3
 
4
4
  const CHARACTER_PROMPTS = [
@@ -1,5 +1,5 @@
1
- import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
2
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { elevenlabs } from "../../ai-sdk/providers/elevenlabs";
2
+ import { fal } from "../../ai-sdk/providers/fal";
3
3
  import { Animate, Clip, Image, Music, Render } from "..";
4
4
 
5
5
  const MADI_REF =
@@ -1,4 +1,4 @@
1
- import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
1
+ import { elevenlabs } from "../../ai-sdk/providers/elevenlabs";
2
2
  import { Clip, Image, Music, Render, render } from "..";
3
3
 
4
4
  async function main() {
@@ -1,4 +1,4 @@
1
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { fal } from "../../ai-sdk/providers/fal";
2
2
  import { Clip, Image, Render, Video } from "..";
3
3
 
4
4
  // character: young woman, short dark brown bob with wispy bangs, oval face, fair skin,
@@ -1,4 +1,4 @@
1
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { fal } from "../../ai-sdk/providers/fal";
2
2
  import { Clip, Image, Render, render, SplitLayout as Split, Title } from "..";
3
3
 
4
4
  async function main() {
@@ -1,4 +1,4 @@
1
- import { fal } from "../../ai-sdk/providers/fal-provider";
1
+ import { fal } from "../../ai-sdk/providers/fal";
2
2
  import { Clip, Grid, Render, render, Title, Video } from "..";
3
3
 
4
4
  async function main() {
@@ -1,6 +1,6 @@
1
1
  import { describe, expect, test } from "bun:test";
2
2
  import { existsSync, unlinkSync } from "node:fs";
3
- import { fal } from "../ai-sdk/providers/fal-provider";
3
+ import { fal } from "../ai-sdk/providers/fal";
4
4
  import {
5
5
  Animate,
6
6
  Captions,
@@ -1,7 +1,12 @@
1
- import { generateImage } from "ai";
1
+ import { generateImage, wrapImageModel } from "ai";
2
2
  import { withCache } from "../../ai-sdk/cache";
3
3
  import { fileCache } from "../../ai-sdk/file-cache";
4
4
  import { generateVideo } from "../../ai-sdk/generate-video";
5
+ import {
6
+ imagePlaceholderFallbackMiddleware,
7
+ placeholderFallbackMiddleware,
8
+ wrapVideoModel,
9
+ } from "../../ai-sdk/middleware";
5
10
  import { editly } from "../../ai-sdk/providers/editly";
6
11
  import type {
7
12
  AudioTrack,
@@ -14,6 +19,7 @@ import type {
14
19
  ClipProps,
15
20
  MusicProps,
16
21
  OverlayProps,
22
+ RenderMode,
17
23
  RenderOptions,
18
24
  RenderProps,
19
25
  SpeechProps,
@@ -48,17 +54,75 @@ export async function renderRoot(
48
54
  const props = element.props as RenderProps;
49
55
  const progress = createProgressTracker(options.quiet ?? false);
50
56
 
57
+ const mode: RenderMode = options.mode ?? "default";
58
+ const placeholderCount = { images: 0, videos: 0, total: 0 };
59
+
60
+ const onFallback = (error: Error, prompt: string) => {
61
+ if (!options.quiet) {
62
+ console.warn(
63
+ `\x1b[33m⚠ provider failed: ${error.message} → placeholder\x1b[0m`,
64
+ );
65
+ }
66
+ };
67
+
68
+ const trackPlaceholder = (type: "image" | "video") => {
69
+ placeholderCount[type === "image" ? "images" : "videos"]++;
70
+ placeholderCount.total++;
71
+ };
72
+
73
+ const wrapGenerateImage: typeof generateImage = async (opts) => {
74
+ if (
75
+ typeof opts.model === "string" ||
76
+ opts.model.specificationVersion !== "v3"
77
+ ) {
78
+ return generateImage(opts);
79
+ }
80
+ const wrappedModel = wrapImageModel({
81
+ model: opts.model,
82
+ middleware: imagePlaceholderFallbackMiddleware({
83
+ mode,
84
+ onFallback: (error, prompt) => {
85
+ trackPlaceholder("image");
86
+ onFallback(error, prompt);
87
+ },
88
+ }),
89
+ });
90
+ const result = await generateImage({ ...opts, model: wrappedModel });
91
+ if (mode === "preview") trackPlaceholder("image");
92
+ return result;
93
+ };
94
+
95
+ const wrapGenerateVideo: typeof generateVideo = async (opts) => {
96
+ const wrappedModel = wrapVideoModel({
97
+ model: opts.model,
98
+ middleware: placeholderFallbackMiddleware({
99
+ mode,
100
+ onFallback: (error, prompt) => {
101
+ trackPlaceholder("video");
102
+ onFallback(error, prompt);
103
+ },
104
+ }),
105
+ });
106
+ const result = await generateVideo({ ...opts, model: wrappedModel });
107
+ if (mode === "preview") trackPlaceholder("video");
108
+ return result;
109
+ };
110
+
51
111
  const ctx: RenderContext = {
52
112
  width: props.width ?? 1920,
53
113
  height: props.height ?? 1080,
54
114
  fps: props.fps ?? 30,
55
115
  cache: options.cache ? fileCache({ dir: options.cache }) : undefined,
56
116
  generateImage: options.cache
57
- ? withCache(generateImage, { storage: fileCache({ dir: options.cache }) })
58
- : generateImage,
117
+ ? withCache(wrapGenerateImage, {
118
+ storage: fileCache({ dir: options.cache }),
119
+ })
120
+ : wrapGenerateImage,
59
121
  generateVideo: options.cache
60
- ? withCache(generateVideo, { storage: fileCache({ dir: options.cache }) })
61
- : generateVideo,
122
+ ? withCache(wrapGenerateVideo, {
123
+ storage: fileCache({ dir: options.cache }),
124
+ })
125
+ : wrapGenerateVideo,
62
126
  tempFiles: [],
63
127
  progress,
64
128
  pending: new Map(),
@@ -238,6 +302,18 @@ export async function renderRoot(
238
302
  completeTask(progress, captionsTaskId);
239
303
  }
240
304
 
305
+ if (!options.quiet && placeholderCount.total > 0) {
306
+ if (mode === "preview") {
307
+ console.log(
308
+ `\x1b[36mℹ preview mode: ${placeholderCount.total} placeholders used (${placeholderCount.images} images, ${placeholderCount.videos} videos)\x1b[0m`,
309
+ );
310
+ } else {
311
+ console.warn(
312
+ `\x1b[33m⚠ ${placeholderCount.total} elements used placeholders - run with --strict for production\x1b[0m`,
313
+ );
314
+ }
315
+ }
316
+
241
317
  const result = await Bun.file(finalOutPath).arrayBuffer();
242
318
  return new Uint8Array(result);
243
319
  }
@@ -206,10 +206,13 @@ export interface PackshotProps extends BaseProps {
206
206
  duration?: number;
207
207
  }
208
208
 
209
+ export type RenderMode = "strict" | "default" | "preview";
210
+
209
211
  export interface RenderOptions {
210
212
  output?: string;
211
213
  cache?: string;
212
214
  quiet?: boolean;
215
+ mode?: RenderMode;
213
216
  }
214
217
 
215
218
  export interface ElementPropsMap {