vargai 0.4.0-alpha2 → 0.4.0-alpha21

package/src/react/renderers/video.ts

@@ -9,6 +9,7 @@ import type {
 import type { RenderContext } from "./context";
 import { renderImage } from "./image";
 import { addTask, completeTask, startTask } from "./progress";
+import { renderSpeech } from "./speech";
 import { computeCacheKey, toFileUrl } from "./utils";
 
 async function resolveImageInput(
@@ -27,13 +28,46 @@ async function resolveImageInput(
   return new Uint8Array(await response.arrayBuffer());
 }
 
-async function resolveMediaInput(
-  input: Uint8Array | string | undefined,
+async function resolveAudioInput(
+  input: Uint8Array | string | VargElement<"speech"> | undefined,
+  ctx: RenderContext,
 ): Promise<Uint8Array | undefined> {
   if (!input) return undefined;
   if (input instanceof Uint8Array) return input;
-  const response = await fetch(toFileUrl(input));
-  return new Uint8Array(await response.arrayBuffer());
+  if (typeof input === "string") {
+    const response = await fetch(toFileUrl(input));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  // It's a Speech element - render it first
+  if (input.type === "speech") {
+    const { path } = await renderSpeech(input, ctx);
+    const response = await fetch(toFileUrl(path));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  throw new Error(
+    `Unsupported audio input type: ${(input as VargElement).type}`,
+  );
+}
+
+async function resolveVideoInput(
+  input: Uint8Array | string | VargElement<"video"> | undefined,
+  ctx: RenderContext,
+): Promise<Uint8Array | undefined> {
+  if (!input) return undefined;
+  if (input instanceof Uint8Array) return input;
+  if (typeof input === "string") {
+    const response = await fetch(toFileUrl(input));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  // It's a Video element - render it first
+  if (input.type === "video") {
+    const path = await renderVideo(input, ctx);
+    const response = await fetch(toFileUrl(path));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  throw new Error(
+    `Unsupported video input type: ${(input as VargElement).type}`,
+  );
 }
 
 async function resolvePrompt(
@@ -55,8 +89,8 @@ async function resolvePrompt(
     prompt.images
       ? Promise.all(prompt.images.map((img) => resolveImageInput(img, ctx)))
       : undefined,
-    resolveMediaInput(prompt.audio),
-    resolveMediaInput(prompt.video),
+    resolveAudioInput(prompt.audio, ctx),
+    resolveVideoInput(prompt.video, ctx),
   ]);
   return {
     text: prompt.text,
@@ -81,9 +115,11 @@ export async function renderVideo(
     throw new Error("Video element requires either 'prompt' or 'src'");
   }
 
-  const model = props.model;
+  const model = props.model ?? ctx.defaults?.video;
   if (!model) {
-    throw new Error("Video element requires 'model' prop when using prompt");
+    throw new Error(
+      "Video element requires 'model' prop (or set defaults.video in render options)",
+    );
   }
 
   // Compute cache key for deduplication
@@ -109,7 +145,8 @@ export async function renderVideo(
     const { video } = await ctx.generateVideo({
       model,
       prompt: resolvedPrompt,
-      duration: 5,
+      duration: props.duration ?? 5,
+      aspectRatio: props.aspectRatio,
       cacheKey,
     } as Parameters<typeof generateVideo>[0]);
 

package/src/react/types.ts

@@ -14,7 +14,6 @@ export type VargElementType =
   | "overlay"
   | "image"
   | "video"
-  | "animate"
   | "speech"
   | "talking-head"
   | "title"
@@ -69,12 +68,17 @@ export interface RenderProps extends BaseProps {
   height?: number;
   fps?: number;
   normalize?: boolean;
+  shortest?: boolean;
   children?: VargNode;
 }
 
 export interface ClipProps extends BaseProps {
   duration?: number | "auto";
   transition?: TransitionOptions;
+  /** Start trim point in seconds (e.g., 1 to start from 1 second) */
+  cutFrom?: number;
+  /** End trim point in seconds (e.g., 3 to end at 3 seconds) */
+  cutTo?: number;
   children?: VargNode;
 }
 
@@ -102,8 +106,8 @@ export type VideoPrompt =
   | {
       text?: string;
       images?: ImageInput[];
-      audio?: Uint8Array | string;
-      video?: Uint8Array | string;
+      audio?: Uint8Array | string | VargElement<"speech">;
+      video?: Uint8Array | string | VargElement<"video">;
     };
 
 export type VideoProps = BaseProps &
@@ -114,17 +118,9 @@ export type VideoProps = BaseProps &
     src?: string;
     model?: VideoModelV3;
     resize?: ResizeMode;
+    aspectRatio?: `${number}:${number}`;
   };
 
-// Image-to-video animation
-export interface AnimateProps extends BaseProps, PositionProps {
-  image?: VargElement<"image">;
-  src?: string;
-  model?: VideoModelV3;
-  motion?: string;
-  duration?: number;
-}
-
 export interface SpeechProps extends BaseProps, VolumeProps {
   voice?: string;
   model?: SpeechModelV3;
@@ -206,13 +202,22 @@ export interface PackshotProps extends BaseProps {
   duration?: number;
 }
 
-export type RenderMode = "strict" | "default" | "preview";
+export type RenderMode = "strict" | "preview";
+
+export interface DefaultModels {
+  image?: ImageModelV3;
+  video?: VideoModelV3;
+  speech?: SpeechModelV3;
+  music?: MusicModelV3;
+}
 
 export interface RenderOptions {
   output?: string;
   cache?: string;
   quiet?: boolean;
+  verbose?: boolean;
   mode?: RenderMode;
+  defaults?: DefaultModels;
 }
 
 export interface ElementPropsMap {
@@ -221,7 +226,6 @@ export interface ElementPropsMap {
   overlay: OverlayProps;
   image: ImageProps;
   video: VideoProps;
-  animate: AnimateProps;
   speech: SpeechProps;
   "talking-head": TalkingHeadProps;
   title: TitleProps;

package/src/studio/stages.ts

@@ -1,6 +1,6 @@
 import type { VargElement, VargNode } from "../react/types";
 
-export type StageType = "image" | "video" | "animate" | "speech" | "music";
+export type StageType = "image" | "video" | "speech" | "music";
 
 export interface RenderStage {
   id: string;
@@ -70,11 +70,6 @@ export function extractStages(element: VargElement): ExtractedStages {
       return "video";
     }
 
-    if (type === "animate") {
-      const motion = props.motion;
-      return motion ? `animate: ${motion}` : "animate";
-    }
-
     if (type === "speech") {
       const text = getTextContent(element.children);
       return `speech: ${text.slice(0, 30)}${text.length > 30 ? "..." : ""}`;
@@ -120,13 +115,7 @@ export function extractStages(element: VargElement): ExtractedStages {
     const collectedDeps: string[] = [...parentDeps];
 
     // Check if this is a renderable stage
-    const stageTypes: StageType[] = [
-      "image",
-      "video",
-      "animate",
-      "speech",
-      "music",
-    ];
+    const stageTypes: StageType[] = ["image", "video", "speech", "music"];
 
     if (stageTypes.includes(element.type as StageType)) {
       const stageType = element.type as StageType;
@@ -137,10 +126,10 @@ export function extractStages(element: VargElement): ExtractedStages {
         return [];
       }
 
-      // For video/animate with image inputs, we need to find dependent images first
+      // For video with image inputs, we need to find dependent images first
       const imageDeps: string[] = [];
 
-      if (stageType === "video" || stageType === "animate") {
+      if (stageType === "video") {
         // Check prompt.images for nested Image elements
         const prompt = props.prompt as { images?: VargNode[] } | undefined;
         if (prompt?.images) {
@@ -158,15 +147,6 @@ export function extractStages(element: VargElement): ExtractedStages {
             }
           }
         }
-
-        // Check for image prop in animate
-        if (stageType === "animate" && props.image) {
-          const imgElement = props.image as VargElement;
-          if (imgElement.type === "image") {
-            const deps = walkTree(imgElement, currentPath, collectedDeps);
-            imageDeps.push(...deps);
-          }
-        }
       }
 
       const id = generateId();

package/src/studio/step-renderer.ts

@@ -2,7 +2,6 @@ import { generateImage } from "ai";
 import { withCache } from "../ai-sdk/cache";
 import { fileCache } from "../ai-sdk/file-cache";
 import { generateVideo } from "../ai-sdk/generate-video";
-import { renderAnimate } from "../react/renderers/animate";
 import type { RenderContext } from "../react/renderers/context";
 import { renderImage } from "../react/renderers/image";
 import { renderMusic } from "../react/renderers/music";
@@ -124,20 +123,6 @@ export async function executeStage(
         break;
       }
 
-      case "animate": {
-        const path = await renderAnimate(
-          stage.element as VargElement<"animate">,
-          session.ctx,
-        );
-        result = {
-          type: "video",
-          path,
-          previewUrl: `/api/step/preview/${session.id}/${stageId}`,
-          mimeType: "video/mp4",
-        };
-        break;
-      }
-
       case "speech": {
         const speechResult = await renderSpeech(
           stage.element as VargElement<"speech">,

package/test-sync-v2.ts

@@ -0,0 +1,30 @@
+/**
+ * Quick sync-v2 lipsync test
+ * Required env: FAL_API_KEY
+ */
+
+import { File, fal, generateVideo } from "./src/ai-sdk/index";
+
+async function main() {
+  const videoPath = "output/extracted-videos/tyler/tyler-10.mp4";
+  const audioPath = "output/extracted-videos/tyler/tyler-10.mp4"; // use same video's audio for now
+
+  console.log("loading media files...");
+  const videoFile = File.fromPath(videoPath);
+  const audioFile = File.fromPath(audioPath);
+
+  console.log("lipsyncing with sync-v2...");
+  const { video } = await generateVideo({
+    model: fal.videoModel("sync-v2"),
+    prompt: {
+      video: await videoFile.data(),
+      audio: await audioFile.data(),
+    },
+  });
+
+  console.log(`lipsynced video: ${video.uint8Array.byteLength} bytes`);
+  await Bun.write("output/test-sync-v2.mp4", video.uint8Array);
+  console.log("done! saved to output/test-sync-v2.mp4");
+}
+
+main().catch(console.error);

package/test-sync-v2.tsx

@@ -0,0 +1,29 @@
+/**
+ * Simple sync-v2 lipsync test in React format
+ * Takes an existing video + audio and lipsyncs them together
+ *
+ * Run: bunx vargai render test-sync-v2.tsx
+ */
+import { fal } from "vargai/ai";
+import { Clip, Render, Video } from "vargai/react";
+
+// Source video (existing talking head video)
+const SOURCE_VIDEO = "output/extracted-videos/tyler/tyler-10.mp4";
+
+// Source audio (pre-generated speech)
+const SOURCE_AUDIO = "output/test-speech.mp3";
+
+export default (
+  <Render width={1080} height={1920}>
+    <Clip duration={10}>
+      {/* Lipsync: video + audio -> sync-v2 */}
+      <Video
+        prompt={{
+          video: SOURCE_VIDEO,
+          audio: SOURCE_AUDIO,
+        }}
+        model={fal.videoModel("sync-v2")}
+      />
+    </Clip>
+  </Render>
+);

package/tsconfig.json

@@ -6,7 +6,7 @@
     "module": "ESNext",
     "moduleDetection": "force",
     "jsx": "react-jsx",
-    "jsxImportSource": "@/react/runtime",
+    "jsxImportSource": "vargai",
     "allowJs": true,
 
     // Bundler mode
@@ -30,10 +30,12 @@
     // Base URL for imports
     "baseUrl": ".",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "vargai/jsx-runtime": ["./src/react/runtime/jsx-runtime.ts"],
+      "vargai/jsx-dev-runtime": ["./src/react/runtime/jsx-dev-runtime.ts"]
     }
   },
-  "include": ["src/**/*"],
+  "include": ["src/**/*", "launch-videos/**/*"],
   "exclude": [
     "node_modules",
     "action",

package/video.tsx

@@ -0,0 +1,7 @@
+export default (
+  <Render>
+    <Clip>
+      <Video prompt="a cat coding" model={fal.videoModel("wan-2.5")} />
+    </Clip>
+  </Render>
+);

package/src/react/cli.ts

@@ -1,52 +0,0 @@
-#!/usr/bin/env bun
-
-import { parseArgs } from "node:util";
-import { render } from "./render";
-import type { VargElement } from "./types";
-
-const { values, positionals } = parseArgs({
-  args: Bun.argv.slice(2),
-  options: {
-    output: { type: "string", short: "o" },
-    cache: { type: "string", short: "c", default: ".cache/ai" },
-    quiet: { type: "boolean", short: "q", default: false },
-  },
-  allowPositionals: true,
-});
-
-const [file] = positionals;
-
-if (!file) {
-  console.error("usage: bun react/cli.ts <component.tsx> [-o output.mp4]");
-  process.exit(1);
-}
-
-const resolvedPath = Bun.resolveSync(file, process.cwd());
-const mod = await import(resolvedPath);
-const component: VargElement = mod.default;
-
-if (!component || component.type !== "render") {
-  console.error("error: default export must be a <Render> element");
-  process.exit(1);
-}
-
-const outputPath =
-  values.output ??
-  `output/${file
-    .replace(/\.tsx?$/, "")
-    .split("/")
-    .pop()}.mp4`;
-
-if (!values.quiet) {
-  console.log(`rendering ${file} → ${outputPath}`);
-}
-
-const buffer = await render(component, {
-  output: outputPath,
-  cache: values.cache,
-  quiet: values.quiet,
-});
-
-if (!values.quiet) {
-  console.log(`done! ${buffer.byteLength} bytes → ${outputPath}`);
-}

package/src/react/renderers/animate.ts

@@ -1,59 +0,0 @@
-import { File } from "../../ai-sdk/file";
-import type { generateVideo } from "../../ai-sdk/generate-video";
-import type { AnimateProps, VargElement } from "../types";
-import type { RenderContext } from "./context";
-import { renderImage } from "./image";
-import { addTask, completeTask, startTask } from "./progress";
-import { computeCacheKey, resolvePath } from "./utils";
-
-export async function renderAnimate(
-  element: VargElement<"animate">,
-  ctx: RenderContext,
-): Promise<string> {
-  const props = element.props as AnimateProps;
-
-  let imagePath: string;
-  if (props.src) {
-    imagePath = props.src;
-  } else if (props.image) {
-    if (props.image.type !== "image") {
-      throw new Error(
-        `Animate 'image' prop must be an <Image /> element, got <${props.image.type} />`,
-      );
-    }
-    imagePath = await renderImage(props.image as VargElement<"image">, ctx);
-  } else {
-    throw new Error("Animate element requires either 'src' or 'image' prop");
-  }
-
-  const model = props.model;
-  if (!model) {
-    throw new Error("Animate element requires 'model' prop");
-  }
-
-  const imageData = await Bun.file(resolvePath(imagePath)).arrayBuffer();
-  const cacheKey = computeCacheKey(element);
-
-  const modelId = typeof model === "string" ? model : model.modelId;
-  const taskId = ctx.progress
-    ? addTask(ctx.progress, "animate", modelId)
-    : null;
-  if (taskId && ctx.progress) startTask(ctx.progress, taskId);
-
-  const { video } = await ctx.generateVideo({
-    model,
-    prompt: {
-      text: props.motion ?? "",
-      images: [new Uint8Array(imageData)],
-    },
-    duration: props.duration ?? 5,
-    cacheKey,
-  } as Parameters<typeof generateVideo>[0]);
-
-  if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
-
-  const tempPath = await File.toTemp(video);
-  ctx.tempFiles.push(tempPath);
-
-  return tempPath;
-}