vargai 0.4.0-alpha35 → 0.4.0-alpha36

package/package.json

@@ -68,7 +68,7 @@
     "sharp": "^0.34.5",
     "zod": "^4.2.1"
   },
-  "version": "0.4.0-alpha35",
+  "version": "0.4.0-alpha36",
   "exports": {
     ".": "./src/index.ts",
     "./ai": "./src/ai-sdk/index.ts",

package/src/ai-sdk/providers/editly/backends/local.ts

@@ -44,7 +44,7 @@ export class LocalBackend implements FFmpegBackend {
       if (typeof input === "string") {
         args.push("-i", input);
       } else if ("raw" in input) {
-        args.push(...input.raw.split(" "));
+        args.push(...input.raw);
       } else {
         if (input.options) args.push(...input.options);
         args.push("-i", input.path);

package/src/ai-sdk/providers/editly/backends/types.ts

@@ -22,8 +22,8 @@ export type FFmpegInput =
       options?: string[];
     }
   | {
-      /** Raw ffmpeg args that don't use -i (e.g. "-f lavfi -i color=black") */
-      raw: string;
+      /** Raw ffmpeg args that don't use -i (e.g. ["-f", "lavfi", "-i", "color=black"]) */
+      raw: string[];
     };
 
 /**

package/src/ai-sdk/providers/editly/rendi/index.ts

@@ -134,7 +134,10 @@ export class RendiBackend implements FFmpegBackend {
 
     const replaceWithPlaceholders = (str: string): string => {
       let result = str;
-      for (const [url, ph] of pathToPlaceholder) {
+      const sortedEntries = [...pathToPlaceholder.entries()].sort(
+        (a, b) => b[0].length - a[0].length,
+      );
+      for (const [url, ph] of sortedEntries) {
         if (result.includes(url)) {
           result = result.replaceAll(url, ph);
         }

package/src/ai-sdk/providers/fal.ts

@@ -43,6 +43,11 @@ const VIDEO_MODELS: Record<string, { t2v: string; i2v: string }> = {
     t2v: "fal-ai/minimax-video/text-to-video",
     i2v: "fal-ai/minimax-video/image-to-video",
   },
+  // LTX-2 19B Distilled - video with native audio generation
+  "ltx-2-19b-distilled": {
+    t2v: "fal-ai/ltx-2-19b/distilled/text-to-video",
+    i2v: "fal-ai/ltx-2-19b/distilled/image-to-video",
+  },
 };
 
 // Motion control models - video-to-video with motion transfer
@@ -182,6 +187,7 @@ class FalVideoModel implements VideoModelV3 {
     const isLipsync = LIPSYNC_MODELS[this.modelId] !== undefined;
     const isMotionControl = MOTION_CONTROL_MODELS[this.modelId] !== undefined;
     const isKlingV26 = this.modelId === "kling-v2.6";
+    const isLtx2 = this.modelId === "ltx-2-19b-distilled";
 
     const endpoint = isLipsync
       ? this.resolveLipsyncEndpoint()
@@ -241,8 +247,21 @@ class FalVideoModel implements VideoModelV3 {
       // Standard video generation
       input.prompt = prompt;
 
-      // Duration must be string "5" or "10" for Kling v2.6
-      if (isKlingV26) {
+      // LTX-2 uses num_frames instead of duration, and has different defaults
+      if (isLtx2) {
+        // LTX-2: convert duration to num_frames (25fps default)
+        // Always set num_frames from duration unless explicitly provided via providerOptions
+        if (input.num_frames === undefined) {
+          const fps = (input.fps as number) ?? 25;
+          const durationSec = duration ?? 5; // default 5 seconds
+          input.num_frames = Math.round(durationSec * fps);
+        }
+        // LTX-2 uses video_size instead of aspect_ratio
+        if (input.video_size === undefined) {
+          input.video_size = "auto";
+        }
+      } else if (isKlingV26) {
+        // Duration must be string "5" or "10" for Kling v2.6
         input.duration = String(duration ?? 5);
       } else {
         input.duration = duration ?? 5;
@@ -255,23 +274,36 @@ class FalVideoModel implements VideoModelV3 {
         if (imageFiles.length > 0) {
           // First image is start image
           input.image_url = await fileToUrl(imageFiles[0]!);
-          // Second image (if provided) is end image for Kling v2.6
-          if (isKlingV26 && imageFiles.length > 1) {
+          // Second image (if provided) is end image for Kling v2.6 and LTX-2
+          if ((isKlingV26 || isLtx2) && imageFiles.length > 1) {
             input.end_image_url = await fileToUrl(imageFiles[1]!);
           }
         }
-      } else {
+      } else if (!isLtx2) {
+        // LTX-2 uses video_size, not aspect_ratio
         input.aspect_ratio = aspectRatio ?? "16:9";
       }
 
-      // Kling v2.6 supports native audio generation
-      if (isKlingV26) {
+      // Kling v2.6 and LTX-2 support native audio generation
+      if (isKlingV26 || isLtx2) {
         // Default to generating audio unless explicitly disabled
         if (input.generate_audio === undefined) {
           input.generate_audio = true;
         }
       }
 
+      // LTX-2 specific defaults
+      if (isLtx2) {
+        // Enable multiscale for better coherence (default: true)
+        if (input.use_multiscale === undefined) {
+          input.use_multiscale = true;
+        }
+        // Enable prompt expansion for better results (default: true)
+        if (input.enable_prompt_expansion === undefined) {
+          input.enable_prompt_expansion = true;
+        }
+      }
+
       const audioFile = files?.find((f) =>
         getMediaType(f)?.startsWith("audio/"),
       );
@@ -280,12 +312,17 @@ class FalVideoModel implements VideoModelV3 {
       }
     }
 
+    // LTX-2 supports seed, other models don't
     if (options.seed !== undefined) {
-      warnings.push({
-        type: "unsupported",
-        feature: "seed",
-        details: "Seed is not supported by this model",
-      });
+      if (isLtx2) {
+        input.seed = options.seed;
+      } else {
+        warnings.push({
+          type: "unsupported",
+          feature: "seed",
+          details: "Seed is not supported by this model",
+        });
+      }
     }
 
     if (options.resolution !== undefined) {
@@ -296,12 +333,17 @@ class FalVideoModel implements VideoModelV3 {
       });
     }
 
+    // LTX-2 supports fps configuration
     if (options.fps !== undefined) {
-      warnings.push({
-        type: "unsupported",
-        feature: "fps",
-        details: "FPS is not configurable for this model",
-      });
+      if (isLtx2) {
+        input.fps = options.fps;
+      } else {
+        warnings.push({
+          type: "unsupported",
+          feature: "fps",
+          details: "FPS is not configurable for this model",
+        });
+      }
     }
 
     const result = await fal.subscribe(endpoint, {

package/src/cli/commands/storyboard.tsx

@@ -135,6 +135,19 @@ function extractNestedFromPrompt(prompt: unknown): StoryboardElement[] {
     for (const img of p.images) {
       if (img && typeof img === "object" && "type" in img) {
         nested.push(extractElementInfo(img as VargElement));
+      } else if (typeof img === "string") {
+        const isUrl = img.startsWith("http://") || img.startsWith("https://");
+        const isLocalFile =
+          img.startsWith("/") || img.startsWith("./") || img.includes(".");
+        if (isUrl || isLocalFile) {
+          nested.push({
+            type: "input",
+            src: img,
+            details: {
+              inputType: isUrl ? "url" : "file",
+            },
+          });
+        }
       }
     }
   }
@@ -378,12 +391,22 @@ const TYPE_COLORS: Record<string, string> = {
   split: "#818cf8",
   slider: "#2dd4bf",
   swipe: "#fb923c",
+  input: "#9ca3af",
 };
 
 function escapeHtml(str: string): string {
   return str.replace(/</g, "&lt;").replace(/>/g, "&gt;");
 }
 
+function escapeAttr(str: string): string {
+  return str
+    .replace(/&/g, "&amp;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#39;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;");
+}
+
 function generateHtml(storyboard: Storyboard, sourceFile: string): string {
   const escapedSourceFile = escapeHtml(sourceFile);
 
@@ -398,9 +421,6 @@ function generateHtml(storyboard: Storyboard, sourceFile: string): string {
     const childPrefix =
       depth === 0 ? "" : parentPrefix + (isLast ? "   " : "│  ");
 
-    const promptOrText = el.prompt || el.text || el.src || "";
-    const shortPrompt = promptOrText ? escapeHtml(promptOrText) : "";
-
     const children =
       (el.details.children as StoryboardElement[] | undefined) || [];
     const childrenHtml = children
@@ -414,6 +434,29 @@ function generateHtml(storyboard: Storyboard, sourceFile: string): string {
       )
       .join("");
 
+    const isInputWithUrl =
+      el.type === "input" &&
+      el.src &&
+      (el.src.startsWith("http://") || el.src.startsWith("https://"));
+
+    if (isInputWithUrl) {
+      const shortUrl =
+        el.src!.length > 50 ? `${el.src!.slice(0, 50)}...` : el.src!;
+      const escapedSrc = escapeAttr(el.src!);
+      return `
+      <div class="tree-node" style="--depth: ${depth}">
+        <span class="tree-prefix">${parentPrefix}${connector}</span>
+        <span class="type-tag" style="background: ${color}">${el.type}</span>
+        <span class="input-preview-wrapper">
+          <a href="${escapedSrc}" target="_blank" rel="noopener noreferrer" class="tree-prompt input-url">${escapeHtml(shortUrl)}</a>
+          <span class="input-preview-tooltip"><img src="${escapedSrc}" alt="preview" /></span>
+        </span>
+      </div>${childrenHtml}`;
+    }
+
+    const promptOrText = el.prompt || el.text || el.src || "";
+    const shortPrompt = promptOrText ? escapeHtml(promptOrText) : "";
+
     return `
       <div class="tree-node" style="--depth: ${depth}">
         <span class="tree-prefix">${parentPrefix}${connector}</span>
@@ -476,10 +519,34 @@ function generateHtml(storyboard: Storyboard, sourceFile: string): string {
         const isLast = i === children.length - 1;
         const connector = isLast ? "└─" : "├─";
         const color = TYPE_COLORS[child.type] || "#666";
-        const childPrompt = child.prompt || child.text || "";
         const grandChildren =
           (child.details.children as StoryboardElement[]) || [];
 
+        const isInputWithUrl =
+          child.type === "input" &&
+          child.src &&
+          (child.src.startsWith("http://") || child.src.startsWith("https://"));
+
+        if (isInputWithUrl) {
+          const shortUrl =
+            child.src!.length > 60
+              ? `${child.src!.slice(0, 60)}...`
+              : child.src!;
+          const escapedSrc = escapeAttr(child.src!);
+          return `
+          <div class="timeline-nested">
+            <span class="nested-connector">${connector}</span>
+            <span class="nested-type" style="background: ${color}">${child.type}</span>
+            <span class="input-preview-wrapper">
+              <a href="${escapedSrc}" target="_blank" rel="noopener noreferrer" class="nested-prompt input-url">${escapeHtml(shortUrl)}</a>
+              <span class="input-preview-tooltip"><img src="${escapedSrc}" alt="preview" /></span>
+            </span>
+          </div>
+          ${grandChildren.length > 0 ? renderNestedTree(grandChildren, depth + 1) : ""}`;
+        }
+
+        const childPrompt = child.prompt || child.text || child.src || "";
+
         return `
           <div class="timeline-nested">
             <span class="nested-connector">${connector}</span>
@@ -1201,6 +1268,47 @@ function generateHtml(storyboard: Storyboard, sourceFile: string): string {
       width: 100%;
       margin-top: 0.25rem;
     }
+    
+    .input-preview-wrapper {
+      position: relative;
+      display: inline-block;
+    }
+    
+    .input-url {
+      color: var(--accent-sky);
+      text-decoration: none;
+      word-break: break-all;
+    }
+    
+    .input-url:hover {
+      text-decoration: underline;
+    }
+    
+    .input-preview-tooltip {
+      display: none;
+      position: absolute;
+      left: 0;
+      top: 100%;
+      margin-top: 8px;
+      z-index: 1000;
+      background: var(--bg-card);
+      border: 1px solid var(--border-soft);
+      border-radius: var(--radius-squishy);
+      box-shadow: var(--shadow-soft);
+      padding: 8px;
+      max-width: 300px;
+    }
+    
+    .input-preview-tooltip img {
+      max-width: 100%;
+      max-height: 200px;
+      border-radius: 8px;
+      display: block;
+    }
+    
+    .input-preview-wrapper:hover .input-preview-tooltip {
+      display: block;
+    }
   </style>
 </head>
 <body>

package/src/react/examples/local-files-test.tsx

@@ -0,0 +1,19 @@
+import { fal } from "../../ai-sdk/providers/fal";
+import { Clip, Image, Render, Video } from "..";
+
+export default (
+  <Render width={1080} height={1920}>
+    <Clip duration={3}>
+      <Image src="media/cyberpunk-street.png" />
+    </Clip>
+    <Clip duration={3}>
+      <Video
+        prompt={{
+          text: "camera pans across the scene",
+          images: [Image({ src: "media/fal-coffee-shop.png" })],
+        }}
+        model={fal.videoModel("kling-v2.5")}
+      />
+    </Clip>
+  </Render>
+);

package/src/react/examples/ltx2-test.tsx

@@ -0,0 +1,25 @@
+import { fal } from "../../ai-sdk/providers/fal";
+import { Clip, Render, Video } from "..";
+
+export default (
+  <Render width={1248} height={704}>
+    <Clip>
+      <Video
+        prompt={{
+          text: "Camera slowly dollies in toward her face, city lights flicker",
+          images: [
+            "https://storage.googleapis.com/falserverless/example_inputs/ltxv-2-i2v-input.jpg",
+          ],
+        }}
+        model={fal.videoModel("ltx-2-19b-distilled")}
+        keepAudio
+        providerOptions={{
+          fal: {
+            generate_audio: true,
+            camera_lora: "dolly_in",
+          },
+        }}
+      />
+    </Clip>
+  </Render>
+);

package/src/react/index.ts

@@ -1,3 +1,4 @@
+export type { CacheStorage } from "../ai-sdk/cache";
 export type { SizeValue } from "../ai-sdk/providers/editly/types";
 export { assets } from "./assets";
 export {

package/src/react/renderers/context.ts

@@ -1,5 +1,5 @@
 import type { generateImage } from "ai";
-import type { fileCache } from "../../ai-sdk/file-cache";
+import type { CacheStorage } from "../../ai-sdk/cache";
 import type { generateVideo } from "../../ai-sdk/generate-video";
 import type { DefaultModels } from "../types";
 import type { ProgressTracker } from "./progress";
@@ -8,7 +8,7 @@ export interface RenderContext {
   width: number;
   height: number;
   fps: number;
-  cache?: ReturnType<typeof fileCache>;
+  cache?: CacheStorage;
   generateImage: typeof generateImage;
   generateVideo: typeof generateVideo;
   tempFiles: string[];

package/src/react/renderers/image.ts

@@ -85,6 +85,7 @@ export async function renderImage(
       model,
       prompt: resolvedPrompt,
       aspectRatio: props.aspectRatio,
+      providerOptions: props.providerOptions,
       n: 1,
       cacheKey,
     } as Parameters<typeof generateImage>[0]);

package/src/react/renderers/render.ts

@@ -1,5 +1,5 @@
 import { generateImage, wrapImageModel } from "ai";
-import { withCache } from "../../ai-sdk/cache";
+import { type CacheStorage, withCache } from "../../ai-sdk/cache";
 import { fileCache } from "../../ai-sdk/file-cache";
 import { generateVideo } from "../../ai-sdk/generate-video";
 import {
@@ -48,6 +48,16 @@ interface RenderedOverlay {
   isVideo: boolean;
 }
 
+function resolveCacheStorage(
+  cache: string | CacheStorage | undefined,
+): CacheStorage | undefined {
+  if (!cache) return undefined;
+  if (typeof cache === "string") {
+    return fileCache({ dir: cache });
+  }
+  return cache;
+}
+
 export async function renderRoot(
   element: VargElement<"render">,
   options: RenderOptions,
@@ -63,12 +73,14 @@ export async function renderRoot(
     placeholderCount.total++;
   };
 
-  const cachedGenerateImage = options.cache
-    ? withCache(generateImage, { storage: fileCache({ dir: options.cache }) })
+  const cacheStorage = resolveCacheStorage(options.cache);
+
+  const cachedGenerateImage = cacheStorage
+    ? withCache(generateImage, { storage: cacheStorage })
     : generateImage;
 
-  const cachedGenerateVideo = options.cache
-    ? withCache(generateVideo, { storage: fileCache({ dir: options.cache }) })
+  const cachedGenerateVideo = cacheStorage
+    ? withCache(generateVideo, { storage: cacheStorage })
     : generateVideo;
 
   const wrapGenerateImage: typeof generateImage = async (opts) => {
@@ -114,7 +126,7 @@ export async function renderRoot(
     width: props.width ?? 1920,
     height: props.height ?? 1080,
     fps: props.fps ?? 30,
-    cache: options.cache ? fileCache({ dir: options.cache }) : undefined,
+    cache: cacheStorage,
     generateImage: wrapGenerateImage,
     generateVideo: wrapGenerateVideo,
     tempFiles: [],

package/src/react/renderers/video.ts

@@ -147,6 +147,7 @@ export async function renderVideo(
       prompt: resolvedPrompt,
       duration: props.duration ?? 5,
       aspectRatio: props.aspectRatio,
+      providerOptions: props.providerOptions,
       cacheKey,
     } as Parameters<typeof generateVideo>[0]);
 

package/src/react/types.ts

@@ -1,5 +1,10 @@
-import type { ImageModelV3, SpeechModelV3 } from "@ai-sdk/provider";
+import type {
+  ImageModelV3,
+  SharedV3ProviderOptions,
+  SpeechModelV3,
+} from "@ai-sdk/provider";
 import type { FFmpegBackend } from "@/ai-sdk/providers/editly/backends";
+import type { CacheStorage } from "../ai-sdk/cache";
 import type { MusicModelV3 } from "../ai-sdk/music-model";
 import type {
   CropPosition,
@@ -101,6 +106,8 @@ export interface ImageProps extends BaseProps, PositionProps {
   position?: Position;
   size?: { width: string; height: string };
   removeBackground?: boolean;
+  /** Provider-specific options (e.g., fal: { acceleration: "high" }) */
+  providerOptions?: SharedV3ProviderOptions;
 }
 
 export type VideoPrompt =
@@ -122,6 +129,8 @@ export type VideoProps = BaseProps &
     resize?: ResizeMode;
     cropPosition?: CropPosition;
     aspectRatio?: `${number}:${number}`;
+    /** Provider-specific options (e.g., fal: { generate_audio: true }) */
+    providerOptions?: SharedV3ProviderOptions;
   };
 
 export interface SpeechProps extends BaseProps, VolumeProps {
@@ -256,7 +265,7 @@ export interface DefaultModels {
 
 export interface RenderOptions {
   output?: string;
-  cache?: string;
+  cache?: string | CacheStorage;
   quiet?: boolean;
   verbose?: boolean;
   mode?: RenderMode;

package/src/studio/step-renderer.ts

@@ -1,5 +1,5 @@
 import { generateImage } from "ai";
-import { withCache } from "../ai-sdk/cache";
+import { type CacheStorage, withCache } from "../ai-sdk/cache";
 import { fileCache } from "../ai-sdk/file-cache";
 import { generateVideo } from "../ai-sdk/generate-video";
 import type { RenderContext } from "../react/renderers/context";
@@ -27,21 +27,26 @@ const sessions = new Map<string, StepSession>();
 export function createStepSession(
   code: string,
   rootElement: VargElement,
-  cacheDir?: string,
+  cache?: string | CacheStorage,
 ): StepSession {
   const props = rootElement.props as RenderProps;
-  const cache = cacheDir ? fileCache({ dir: cacheDir }) : undefined;
+  const cacheStorage =
+    cache === undefined
+      ? undefined
+      : typeof cache === "string"
+        ? fileCache({ dir: cache })
+        : cache;
 
   const ctx: RenderContext = {
     width: props.width ?? 1920,
     height: props.height ?? 1080,
     fps: props.fps ?? 30,
-    cache,
-    generateImage: cache
-      ? withCache(generateImage, { storage: cache })
+    cache: cacheStorage,
+    generateImage: cacheStorage
+      ? withCache(generateImage, { storage: cacheStorage })
       : generateImage,
-    generateVideo: cache
-      ? withCache(generateVideo, { storage: cache })
+    generateVideo: cacheStorage
+      ? withCache(generateVideo, { storage: cacheStorage })
       : generateVideo,
     tempFiles: [],
     progress: createProgressTracker(false),
@@ -223,7 +228,7 @@ export async function finalizeRender(
 
   await render(session.rootElement, {
     output: outputPath,
-    cache: session.ctx.cache ? ".cache/ai" : undefined,
+    cache: session.ctx.cache,
     quiet: true,
   });