vargai 0.3.0 → 0.3.1

package/README.md

@@ -5,9 +5,46 @@ AI video generation from your terminal.
 ## Quick Start
 
 ```bash
-bun install @vargai/sdk
+bun install vargai ai
 ```
 
+### SDK Usage
+
+```typescript
+import { generateImage } from "ai";
+import { File, fal, generateElement, generateVideo, scene } from "vargai";
+
+// generate a character from reference image
+const { element: character } = await generateElement({
+  model: fal.imageModel("nano-banana-pro/edit"),
+  type: "character",
+  prompt: {
+    text: "cartoon character, simple style",
+    images: [await File.fromPath("media/reference.jpg").arrayBuffer()],
+  },
+});
+
+// generate scene with character
+const { image: frame } = await generateImage({
+  model: fal.imageModel("nano-banana-pro"),
+  prompt: scene`${character} walks through a forest`,
+});
+
+// animate the frame
+const { video } = await generateVideo({
+  model: fal.videoModel("wan-2.5"),
+  prompt: {
+    text: `${character.text} walks through a forest`,
+    images: [frame.base64],
+  },
+  duration: 5,
+});
+
+await Bun.write("output/scene.mp4", video.uint8Array);
+```
+
+### CLI Usage
+
 ```bash
 varg run image --prompt "cyberpunk cityscape at night"
 varg run video --prompt "camera flies through clouds" --duration 5

package/bun.lock

@@ -40,6 +40,7 @@
         "lint-staged": "^16.2.7",
       },
       "peerDependencies": {
+        "ai": "^6.0.0",
         "typescript": "^5",
       },
     },

package/package.json

@@ -32,7 +32,8 @@
     "lint-staged": "^16.2.7"
   },
   "peerDependencies": {
-    "typescript": "^5"
+    "typescript": "^5",
+    "ai": "^6.0.0"
   },
   "dependencies": {
     "@ai-sdk/fal": "^1.0.23",
@@ -59,7 +60,7 @@
     "replicate": "^1.4.0",
     "zod": "^4.2.1"
   },
-  "version": "0.3.0",
+  "version": "0.3.1",
   "exports": {
     ".": "./src/index.ts",
     "./core": "./src/core/index.ts",

package/src/ai-sdk/index.ts

@@ -25,6 +25,25 @@ export {
   type GenerateVideoResult,
   generateVideo,
 } from "./generate-video";
+export {
+  generatePlaceholder,
+  type ImagePlaceholderFallbackOptions,
+  imagePlaceholderFallbackMiddleware,
+  type MusicModelMiddleware,
+  type MusicPlaceholderFallbackOptions,
+  musicPlaceholderFallbackMiddleware,
+  type PlaceholderFallbackOptions,
+  type PlaceholderOptions,
+  type PlaceholderResult,
+  placeholderFallbackMiddleware,
+  type RenderMode,
+  type VideoModelMiddleware,
+  withImagePlaceholderFallback,
+  withMusicPlaceholderFallback,
+  withPlaceholderFallback,
+  wrapMusicModel,
+  wrapVideoModel,
+} from "./middleware";
 export type {
   MusicModelV3,
   MusicModelV3CallOptions,
@@ -42,8 +61,8 @@ export {
   type ElevenLabsProvider,
   elevenlabs,
   VOICES,
-} from "./providers/elevenlabs-provider";
-export { createFal, type FalProvider, fal } from "./providers/fal-provider";
+} from "./providers/elevenlabs";
+export { createFal, type FalProvider, fal } from "./providers/fal";
 export {
   createHiggsfield,
   type HiggsfieldImageModelSettings,

package/src/ai-sdk/middleware/index.ts

@@ -0,0 +1,25 @@
+export {
+  generatePlaceholder,
+  type PlaceholderOptions,
+  type PlaceholderResult,
+} from "./placeholder";
+export {
+  type ImagePlaceholderFallbackOptions,
+  imagePlaceholderFallbackMiddleware,
+  withImagePlaceholderFallback,
+} from "./wrap-image-model";
+export {
+  type MusicModelMiddleware,
+  type MusicPlaceholderFallbackOptions,
+  musicPlaceholderFallbackMiddleware,
+  withMusicPlaceholderFallback,
+  wrapMusicModel,
+} from "./wrap-music-model";
+export {
+  type PlaceholderFallbackOptions,
+  placeholderFallbackMiddleware,
+  type RenderMode,
+  type VideoModelMiddleware,
+  withPlaceholderFallback,
+  wrapVideoModel,
+} from "./wrap-video-model";

package/src/ai-sdk/middleware/placeholder.ts

@@ -0,0 +1,111 @@
+import { unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { $ } from "bun";
+
+export interface PlaceholderOptions {
+  type: "image" | "video" | "audio";
+  prompt: string;
+  duration?: number;
+  width?: number;
+  height?: number;
+}
+
+export interface PlaceholderResult {
+  data: Uint8Array;
+  placeholder: true;
+}
+
+function promptToColor(prompt: string): string {
+  let hash = 0;
+  for (let i = 0; i < prompt.length; i++) {
+    const char = prompt.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash;
+  }
+
+  const h = Math.abs(hash) % 360;
+  const s = 40 + (Math.abs(hash >> 8) % 30);
+  const l = 35 + (Math.abs(hash >> 16) % 20);
+
+  return `hsl(${h},${s}%,${l}%)`;
+}
+
+function hslToHex(hsl: string): string {
+  const match = hsl.match(/hsl\((\d+),(\d+)%,(\d+)%\)/);
+  if (!match) return "333333";
+
+  const h = Number.parseInt(match[1]!) / 360;
+  const s = Number.parseInt(match[2]!) / 100;
+  const l = Number.parseInt(match[3]!) / 100;
+
+  const hue2rgb = (p: number, q: number, t: number) => {
+    if (t < 0) t += 1;
+    if (t > 1) t -= 1;
+    if (t < 1 / 6) return p + (q - p) * 6 * t;
+    if (t < 1 / 2) return q;
+    if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6;
+    return p;
+  };
+
+  const q = l < 0.5 ? l * (1 + s) : l + s - l * s;
+  const p = 2 * l - q;
+  const r = Math.round(hue2rgb(p, q, h + 1 / 3) * 255);
+  const g = Math.round(hue2rgb(p, q, h) * 255);
+  const b = Math.round(hue2rgb(p, q, h - 1 / 3) * 255);
+
+  return `${r.toString(16).padStart(2, "0")}${g.toString(16).padStart(2, "0")}${b.toString(16).padStart(2, "0")}`;
+}
+
+function truncatePrompt(text: string, maxLen: number): string {
+  const clean = text.replace(/[^a-zA-Z0-9 .,!?-]/g, "");
+  if (clean.length <= maxLen) return clean;
+  return `${clean.slice(0, maxLen - 3)}...`;
+}
+
+export async function generatePlaceholder(
+  options: PlaceholderOptions,
+): Promise<PlaceholderResult> {
+  const { type, prompt, duration = 3, width = 1080, height = 1920 } = options;
+
+  const color = promptToColor(prompt);
+  const hexColor = hslToHex(color);
+  const labelFontSize = Math.floor(Math.min(width, height) / 20);
+  const promptFontSize = Math.floor(Math.min(width, height) / 35);
+  const maxChars = Math.floor((width * 0.7) / (promptFontSize * 0.5));
+  const typeLabel = type.toUpperCase();
+  const promptText = truncatePrompt(prompt, maxChars);
+
+  const ext = type === "audio" ? "mp3" : type === "image" ? "png" : "mp4";
+  const outputPath = join(
+    tmpdir(),
+    `placeholder_${Date.now()}_${Math.random().toString(36).slice(2)}.${ext}`,
+  );
+
+  try {
+    if (type === "audio") {
+      await $`ffmpeg -y -f lavfi -i anullsrc=r=44100:cl=stereo -t ${duration} -c:a libmp3lame ${outputPath}`.quiet();
+    } else if (type === "image") {
+      const colorInput = `color=c=0x${hexColor}:s=${width}x${height}:d=1`;
+      const labelY = `(h/2)-${labelFontSize}`;
+      const promptY = `(h/2)+${Math.floor(labelFontSize * 0.5)}`;
+      const drawLabel = `drawtext=text='${typeLabel}':fontcolor=white:fontsize=${labelFontSize}:x=(w-text_w)/2:y=${labelY}`;
+      const drawPrompt = `drawtext=text='${promptText}':fontcolor=white@0.7:fontsize=${promptFontSize}:x=(w-text_w)/2:y=${promptY}`;
+      await $`ffmpeg -y -f lavfi -i ${colorInput} -vf ${drawLabel},${drawPrompt} -frames:v 1 -update 1 ${outputPath}`.quiet();
+    } else {
+      const colorInput = `color=c=0x${hexColor}:s=${width}x${height}:d=${duration}:r=30`;
+      const labelY = `(h/2)-${labelFontSize}`;
+      const promptY = `(h/2)+${Math.floor(labelFontSize * 0.5)}`;
+      const drawLabel = `drawtext=text='${typeLabel}':fontcolor=white:fontsize=${labelFontSize}:x=(w-text_w)/2:y=${labelY}`;
+      const drawPrompt = `drawtext=text='${promptText}':fontcolor=white@0.7:fontsize=${promptFontSize}:x=(w-text_w)/2:y=${promptY}`;
+      await $`ffmpeg -y -f lavfi -i ${colorInput} -vf ${drawLabel},${drawPrompt} -c:v libx264 -preset ultrafast -pix_fmt yuv420p ${outputPath}`.quiet();
+    }
+
+    const data = await Bun.file(outputPath).bytes();
+    await unlink(outputPath).catch(() => {});
+    return { data: new Uint8Array(data), placeholder: true };
+  } catch (e) {
+    await unlink(outputPath).catch(() => {});
+    throw e;
+  }
+}

package/src/ai-sdk/middleware/wrap-image-model.ts

@@ -0,0 +1,86 @@
+import type {
+  ImageModelV3,
+  ImageModelV3CallOptions,
+  ImageModelV3Middleware,
+} from "@ai-sdk/provider";
+import { wrapImageModel } from "ai";
+import { generatePlaceholder } from "./placeholder";
+import type { RenderMode } from "./wrap-video-model";
+
+export interface ImagePlaceholderFallbackOptions {
+  mode: RenderMode;
+  onFallback?: (error: Error, prompt: string) => void;
+}
+
+export function imagePlaceholderFallbackMiddleware(
+  options: ImagePlaceholderFallbackOptions,
+): ImageModelV3Middleware {
+  const { mode, onFallback } = options;
+
+  return {
+    specificationVersion: "v3",
+    wrapGenerate: async ({ doGenerate, params, model }) => {
+      const createPlaceholderResult = async () => {
+        const [width, height] = (params.size?.split("x").map(Number) ?? [
+          1024, 1024,
+        ]) as [number, number];
+        const prompt =
+          typeof params.prompt === "string"
+            ? params.prompt
+            : ((params.prompt as { text?: string } | undefined)?.text ??
+              "placeholder");
+
+        const placeholder = await generatePlaceholder({
+          type: "image",
+          prompt,
+          width,
+          height,
+        });
+
+        return {
+          images: [placeholder.data],
+          warnings: [
+            {
+              type: "other" as const,
+              message: "placeholder: provider skipped or failed",
+            },
+          ],
+          response: {
+            timestamp: new Date(),
+            modelId: model.modelId,
+            headers: undefined,
+          },
+        };
+      };
+
+      if (mode === "preview") {
+        return createPlaceholderResult();
+      }
+
+      try {
+        return await doGenerate();
+      } catch (e) {
+        if (mode === "strict") throw e;
+
+        const error = e instanceof Error ? e : new Error(String(e));
+        const promptText =
+          typeof params.prompt === "string"
+            ? params.prompt
+            : ((params.prompt as { text?: string } | undefined)?.text ??
+              "placeholder");
+        onFallback?.(error, promptText);
+        return createPlaceholderResult();
+      }
+    },
+  };
+}
+
+export function withImagePlaceholderFallback(
+  model: ImageModelV3,
+  options: ImagePlaceholderFallbackOptions,
+): ImageModelV3 {
+  return wrapImageModel({
+    model,
+    middleware: imagePlaceholderFallbackMiddleware(options),
+  });
+}

package/src/ai-sdk/middleware/wrap-music-model.ts

@@ -0,0 +1,108 @@
+import type { MusicModelV3, MusicModelV3CallOptions } from "../music-model";
+import { generatePlaceholder } from "./placeholder";
+import type { RenderMode } from "./wrap-video-model";
+
+export interface MusicModelMiddleware {
+  transformParams?: (options: {
+    params: MusicModelV3CallOptions;
+    model: MusicModelV3;
+  }) => PromiseLike<MusicModelV3CallOptions> | MusicModelV3CallOptions;
+
+  wrapGenerate?: (options: {
+    doGenerate: () => PromiseLike<
+      Awaited<ReturnType<MusicModelV3["doGenerate"]>>
+    >;
+    params: MusicModelV3CallOptions;
+    model: MusicModelV3;
+  }) => PromiseLike<Awaited<ReturnType<MusicModelV3["doGenerate"]>>>;
+}
+
+export function wrapMusicModel({
+  model,
+  middleware,
+}: {
+  model: MusicModelV3;
+  middleware: MusicModelMiddleware;
+}): MusicModelV3 {
+  const { transformParams, wrapGenerate } = middleware;
+
+  return {
+    specificationVersion: "v3",
+    provider: model.provider,
+    modelId: model.modelId,
+
+    async doGenerate(params: MusicModelV3CallOptions) {
+      const transformedParams = transformParams
+        ? await transformParams({ params, model })
+        : params;
+
+      const doGenerate = () => model.doGenerate(transformedParams);
+
+      return wrapGenerate
+        ? wrapGenerate({ doGenerate, params: transformedParams, model })
+        : doGenerate();
+    },
+  };
+}
+
+export interface MusicPlaceholderFallbackOptions {
+  mode: RenderMode;
+  onFallback?: (error: Error, prompt: string) => void;
+}
+
+export function musicPlaceholderFallbackMiddleware(
+  options: MusicPlaceholderFallbackOptions,
+): MusicModelMiddleware {
+  const { mode, onFallback } = options;
+
+  return {
+    wrapGenerate: async ({ doGenerate, params, model }) => {
+      const createPlaceholderResult = async () => {
+        const placeholder = await generatePlaceholder({
+          type: "audio",
+          prompt: params.prompt,
+          duration: params.duration ?? 10,
+        });
+
+        return {
+          audio: placeholder.data,
+          warnings: [
+            {
+              type: "other" as const,
+              message: "placeholder: provider skipped or failed",
+            },
+          ],
+          response: {
+            timestamp: new Date(),
+            modelId: model.modelId,
+            headers: undefined,
+          },
+        };
+      };
+
+      if (mode === "preview") {
+        return createPlaceholderResult();
+      }
+
+      try {
+        return await doGenerate();
+      } catch (e) {
+        if (mode === "strict") throw e;
+
+        const error = e instanceof Error ? e : new Error(String(e));
+        onFallback?.(error, params.prompt);
+        return createPlaceholderResult();
+      }
+    },
+  };
+}
+
+export function withMusicPlaceholderFallback(
+  model: MusicModelV3,
+  options: MusicPlaceholderFallbackOptions,
+): MusicModelV3 {
+  return wrapMusicModel({
+    model,
+    middleware: musicPlaceholderFallbackMiddleware(options),
+  });
+}

package/src/ai-sdk/middleware/wrap-video-model.ts

@@ -0,0 +1,115 @@
+import type { VideoModelV3, VideoModelV3CallOptions } from "../video-model";
+import { generatePlaceholder } from "./placeholder";
+
+export type RenderMode = "strict" | "default" | "preview";
+
+export interface VideoModelMiddleware {
+  transformParams?: (options: {
+    params: VideoModelV3CallOptions;
+    model: VideoModelV3;
+  }) => PromiseLike<VideoModelV3CallOptions> | VideoModelV3CallOptions;
+
+  wrapGenerate?: (options: {
+    doGenerate: () => PromiseLike<
+      Awaited<ReturnType<VideoModelV3["doGenerate"]>>
+    >;
+    params: VideoModelV3CallOptions;
+    model: VideoModelV3;
+  }) => PromiseLike<Awaited<ReturnType<VideoModelV3["doGenerate"]>>>;
+}
+
+export function wrapVideoModel({
+  model,
+  middleware,
+}: {
+  model: VideoModelV3;
+  middleware: VideoModelMiddleware;
+}): VideoModelV3 {
+  const { transformParams, wrapGenerate } = middleware;
+
+  return {
+    specificationVersion: "v3",
+    provider: model.provider,
+    modelId: model.modelId,
+    maxVideosPerCall: model.maxVideosPerCall,
+
+    async doGenerate(params: VideoModelV3CallOptions) {
+      const transformedParams = transformParams
+        ? await transformParams({ params, model })
+        : params;
+
+      const doGenerate = () => model.doGenerate(transformedParams);
+
+      return wrapGenerate
+        ? wrapGenerate({ doGenerate, params: transformedParams, model })
+        : doGenerate();
+    },
+  };
+}
+
+export interface PlaceholderFallbackOptions {
+  mode: RenderMode;
+  onFallback?: (error: Error, prompt: string) => void;
+}
+
+export function placeholderFallbackMiddleware(
+  options: PlaceholderFallbackOptions,
+): VideoModelMiddleware {
+  const { mode, onFallback } = options;
+
+  return {
+    wrapGenerate: async ({ doGenerate, params, model }) => {
+      const createPlaceholderResult = async () => {
+        const [width, height] = (params.resolution?.split("x").map(Number) ?? [
+          1080, 1920,
+        ]) as [number, number];
+        const placeholder = await generatePlaceholder({
+          type: "video",
+          prompt: params.prompt,
+          duration: params.duration ?? 3,
+          width,
+          height,
+        });
+
+        return {
+          videos: [placeholder.data],
+          warnings: [
+            {
+              type: "other" as const,
+              message: "placeholder: provider skipped or failed",
+            },
+          ],
+          response: {
+            timestamp: new Date(),
+            modelId: model.modelId,
+            headers: undefined,
+          },
+        };
+      };
+
+      if (mode === "preview") {
+        return createPlaceholderResult();
+      }
+
+      try {
+        return await doGenerate();
+      } catch (e) {
+        if (mode === "strict") throw e;
+
+        const error = e instanceof Error ? e : new Error(String(e));
+        onFallback?.(error, params.prompt);
+        return createPlaceholderResult();
+      }
+    },
+  };
+}
+
+export function withPlaceholderFallback(
+  model: VideoModelV3,
+  options: PlaceholderFallbackOptions,
+): VideoModelV3 {
+  return wrapVideoModel({
+    model,
+    middleware: placeholderFallbackMiddleware(options),
+  });
+}