vargai 0.4.0-alpha2 → 0.4.0-alpha21

package/src/ai-sdk/middleware/wrap-music-model.ts

@@ -53,11 +53,11 @@ export interface MusicPlaceholderFallbackOptions {
 export function musicPlaceholderFallbackMiddleware(
   options: MusicPlaceholderFallbackOptions,
 ): MusicModelMiddleware {
-  const { mode, onFallback } = options;
+  const { mode } = options;
 
   return {
     wrapGenerate: async ({ doGenerate, params, model }) => {
-      const createPlaceholderResult = async () => {
+      if (mode === "preview") {
         const placeholder = await generatePlaceholder({
           type: "audio",
           prompt: params.prompt,
@@ -69,7 +69,7 @@ export function musicPlaceholderFallbackMiddleware(
           warnings: [
             {
               type: "other" as const,
-              message: "placeholder: provider skipped or failed",
+              message: "placeholder: preview mode",
             },
           ],
           response: {
@@ -78,21 +78,9 @@ export function musicPlaceholderFallbackMiddleware(
             headers: undefined,
           },
         };
-      };
-
-      if (mode === "preview") {
-        return createPlaceholderResult();
       }
 
-      try {
-        return await doGenerate();
-      } catch (e) {
-        if (mode === "strict") throw e;
-
-        const error = e instanceof Error ? e : new Error(String(e));
-        onFallback?.(error, params.prompt);
-        return createPlaceholderResult();
-      }
+      return doGenerate();
     },
   };
 }

package/src/ai-sdk/middleware/wrap-video-model.ts

@@ -1,7 +1,7 @@
 import type { VideoModelV3, VideoModelV3CallOptions } from "../video-model";
 import { generatePlaceholder } from "./placeholder";
 
-export type RenderMode = "strict" | "default" | "preview";
+export type RenderMode = "strict" | "preview";
 
 export interface VideoModelMiddleware {
   transformParams?: (options: {
@@ -55,11 +55,11 @@ export interface PlaceholderFallbackOptions {
 export function placeholderFallbackMiddleware(
   options: PlaceholderFallbackOptions,
 ): VideoModelMiddleware {
-  const { mode, onFallback } = options;
+  const { mode } = options;
 
   return {
     wrapGenerate: async ({ doGenerate, params, model }) => {
-      const createPlaceholderResult = async () => {
+      if (mode === "preview") {
         const [width, height] = (params.resolution?.split("x").map(Number) ?? [
           1080, 1920,
         ]) as [number, number];
@@ -76,7 +76,7 @@ export function placeholderFallbackMiddleware(
           warnings: [
             {
               type: "other" as const,
-              message: "placeholder: provider skipped or failed",
+              message: "placeholder: preview mode",
             },
           ],
           response: {
@@ -85,21 +85,9 @@ export function placeholderFallbackMiddleware(
             headers: undefined,
           },
         };
-      };
-
-      if (mode === "preview") {
-        return createPlaceholderResult();
       }
 
-      try {
-        return await doGenerate();
-      } catch (e) {
-        if (mode === "strict") throw e;
-
-        const error = e instanceof Error ? e : new Error(String(e));
-        onFallback?.(error, params.prompt);
-        return createPlaceholderResult();
-      }
+      return doGenerate();
     },
   };
 }

package/src/ai-sdk/providers/editly/index.ts

@@ -137,6 +137,15 @@ function isOverlayLayer(layer: Layer): boolean {
   return isVideoOverlayLayer(layer) || isImageOverlayLayer(layer);
 }
 
+function isTextOverlayLayer(layer: Layer): boolean {
+  return (
+    layer.type === "title" ||
+    layer.type === "subtitle" ||
+    layer.type === "news-title" ||
+    layer.type === "slide-in-text"
+  );
+}
+
 function buildBaseClipFilter(
   clip: ProcessedClip,
   clipIndex: number,
@@ -164,7 +173,10 @@ function buildBaseClipFilter(
   let baseLabel = "";
   let inputIdx = inputOffset;
 
-  const baseLayers = clip.layers.filter((l) => l && !isOverlayLayer(l));
+  // Filter out overlay layers AND text overlay layers (text will be applied after image overlays)
+  const baseLayers = clip.layers.filter(
+    (l) => l && !isOverlayLayer(l) && !isTextOverlayLayer(l),
+  );
 
   for (let i = 0; i < baseLayers.length; i++) {
     const layer = baseLayers[i];
@@ -201,58 +213,6 @@ function buildBaseClipFilter(
         inputIdx++;
       }
     }
-
-    if (layer.type === "title") {
-      const titleFilter = getTitleFilter(
-        layer as TitleLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `title${clipIndex}_${i}`;
-      filters.push(`${titleFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
-
-    if (layer.type === "subtitle") {
-      const subtitleFilter = getSubtitleFilter(
-        layer as SubtitleLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `sub${clipIndex}_${i}`;
-      filters.push(`${subtitleFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
-
-    if (layer.type === "news-title") {
-      const newsFilter = getNewsTitleFilter(
-        layer as NewsTitleLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `news${clipIndex}_${i}`;
-      filters.push(`${newsFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
-
-    if (layer.type === "slide-in-text") {
-      const slideFilter = getSlideInTextFilter(
-        layer as SlideInTextLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `slide${clipIndex}_${i}`;
-      filters.push(`${slideFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
   }
 
   return {
@@ -358,6 +318,41 @@ function collectAudioLayers(
   return audioLayers;
 }
 
+type TextLayer = TitleLayer | SubtitleLayer | NewsTitleLayer | SlideInTextLayer;
+
+interface TimedTextLayer {
+  layer: TextLayer;
+  startTime: number;
+  duration: number;
+}
+
+function collectTextLayers(clips: ProcessedClip[]): TimedTextLayer[] {
+  const textLayers: TimedTextLayer[] = [];
+  let currentTime = 0;
+
+  for (let i = 0; i < clips.length; i++) {
+    const clip = clips[i];
+    if (!clip) continue;
+
+    for (const layer of clip.layers) {
+      if (layer && isTextOverlayLayer(layer)) {
+        textLayers.push({
+          layer: layer as TextLayer,
+          startTime: currentTime,
+          duration: clip.duration,
+        });
+      }
+    }
+
+    currentTime += clip.duration;
+    if (i < clips.length - 1) {
+      currentTime -= clip.transition.duration;
+    }
+  }
+
+  return textLayers;
+}
+
 function buildTransitionFilter(
   fromLabel: string,
   toLabel: string,
@@ -744,6 +739,67 @@ export async function editly(config: EditlyConfig): Promise<void> {
     finalVideoLabel = currentBase;
   }
 
+  const textLayers = collectTextLayers(clips);
+  if (textLayers.length > 0) {
+    let currentBase = finalVideoLabel;
+
+    for (let i = 0; i < textLayers.length; i++) {
+      const timedLayer = textLayers[i];
+      if (!timedLayer) continue;
+
+      const { layer, startTime, duration } = timedLayer;
+      const outputLabel = `vwithtext${i}`;
+
+      const timedLayerWithEnable = {
+        ...layer,
+        start: layer.start ?? startTime,
+        stop: layer.stop ?? startTime + duration,
+      };
+
+      if (layer.type === "title") {
+        const titleFilter = getTitleFilter(
+          timedLayerWithEnable as TitleLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${titleFilter}[${outputLabel}]`);
+      } else if (layer.type === "subtitle") {
+        const subtitleFilter = getSubtitleFilter(
+          timedLayerWithEnable as SubtitleLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${subtitleFilter}[${outputLabel}]`);
+      } else if (layer.type === "news-title") {
+        const newsFilter = getNewsTitleFilter(
+          timedLayerWithEnable as NewsTitleLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${newsFilter}[${outputLabel}]`);
+      } else if (layer.type === "slide-in-text") {
+        const slideFilter = getSlideInTextFilter(
+          timedLayerWithEnable as SlideInTextLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${slideFilter}[${outputLabel}]`);
+      }
+
+      currentBase = outputLabel;
+    }
+
+    finalVideoLabel = currentBase;
+  }
+
   const clipAudioLayers = collectAudioLayers(clips);
   const videoInputCount = allInputs.length;
   const audioFilter = buildAudioFilter(
@@ -796,6 +852,7 @@ export async function editly(config: EditlyConfig): Promise<void> {
     "-r",
     String(fps),
     ...outputArgs,
+    ...(config.shortest ? ["-shortest"] : []),
     "-y",
     outPath,
   ];

package/src/ai-sdk/providers/editly/types.ts

@@ -309,6 +309,8 @@ export interface EditlyConfig {
   audioNorm?: AudioNormalizationOptions;
   verbose?: boolean;
   enableFfmpegLog?: boolean;
+  /** End output when shortest stream ends (video or audio) */
+  shortest?: boolean;
 }
 
 // Internal types used by our implementation

package/src/ai-sdk/providers/elevenlabs.ts

@@ -196,8 +196,16 @@ export function createElevenLabs(
   };
 }
 
-export const elevenlabs_provider = createElevenLabs();
-export { elevenlabs_provider as elevenlabs, VOICES };
+let _elevenlabs: ElevenLabsProvider | undefined;
+export const elevenlabs = new Proxy({} as ElevenLabsProvider, {
+  get(_, prop) {
+    if (!_elevenlabs) {
+      _elevenlabs = createElevenLabs();
+    }
+    return _elevenlabs[prop as keyof ElevenLabsProvider];
+  },
+});
+export { VOICES };
 
 export interface GenerateMusicOptions {
   prompt: string;

package/src/ai-sdk/providers/fal.ts

@@ -58,7 +58,12 @@ const IMAGE_MODELS: Record<string, string> = {
 };
 
 // Models that use image_size instead of aspect_ratio
-const IMAGE_SIZE_MODELS = new Set(["seedream-v4.5/edit"]);
+const IMAGE_SIZE_MODELS = new Set([
+  "flux-schnell",
+  "flux-dev",
+  "flux-pro",
+  "seedream-v4.5/edit",
+]);
 
 // Map aspect ratio strings to image_size enum values
 const ASPECT_RATIO_TO_IMAGE_SIZE: Record<string, string> = {

package/src/cli/commands/find.tsx

@@ -1,3 +1,4 @@
+/** @jsxImportSource react */
 /**
  * varg find command
  * Ink-based search view

package/src/cli/commands/hello.ts

@@ -0,0 +1,85 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { defineCommand } from "citty";
+
+const HELLO_TEMPLATE = `/** @jsxImportSource vargai */
+import { Render, Clip, Image, Video, assets } from "vargai/react";
+import { fal } from "vargai/ai";
+
+const girl = Image({
+  prompt: {
+    text: \`Using the attached reference images, generate a photorealistic three-quarter editorial portrait of the exact same character — maintain identical face, hairstyle, and proportions from Image 1.
+
+Framing: Head and shoulders, cropped at upper chest. Direct eye contact with camera.
+
+Natural confident expression, relaxed shoulders.
+Preserve the outfit neckline and visible clothing details from reference.
+
+Background: Deep black with two contrasting orange gradient accents matching Reference 2. Soft gradient bleed, no hard edges.
+
+Shot on 85mm f/1.4 lens, shallow depth of field. Clean studio lighting — soft key light on face, subtle rim light on hair and shoulders for separation. High-end fashion editorial aesthetic.\`,
+    images: [assets.characters.orangeGirl, assets.backgrounds.orangeGradient],
+  },
+  model: fal.imageModel("nano-banana-pro/edit"),
+  aspectRatio: "9:16",
+});
+
+export default (
+  <Render width={1080} height={1920}>
+    <Clip duration={5}>
+      <Video
+        prompt={{
+          text: "She waves hello warmly, natural smile, friendly expression. Studio lighting, authentic confident slightly playful atmosphere. Camera static. Intense orange lighting.",
+          images: [girl],
+        }}
+        model={fal.videoModel("kling-v2.5")}
+      />
+    </Clip>
+  </Render>
+);
+`;
+
+export const helloCmd = defineCommand({
+  meta: {
+    name: "hello",
+    description: "create hello.tsx starter video",
+  },
+  args: {
+    directory: {
+      type: "positional",
+      description: "directory (default: current)",
+      required: false,
+    },
+  },
+  async run({ args }) {
+    const dir = (args.directory as string) || ".";
+    const cwd = dir === "." ? process.cwd() : join(process.cwd(), dir);
+
+    if (!existsSync(cwd) && dir !== ".") {
+      mkdirSync(cwd, { recursive: true });
+      console.log(`created ${dir}/`);
+    }
+
+    const outputDir = join(cwd, "output");
+    if (!existsSync(outputDir)) {
+      mkdirSync(outputDir, { recursive: true });
+      console.log(`created output/`);
+    }
+
+    const cacheDir = join(cwd, ".cache/ai");
+    if (!existsSync(cacheDir)) {
+      mkdirSync(cacheDir, { recursive: true });
+      console.log(`created .cache/ai/`);
+    }
+
+    const helloPath = join(cwd, "hello.tsx");
+    if (existsSync(helloPath)) {
+      console.log(`hello.tsx already exists, skipping`);
+    } else {
+      writeFileSync(helloPath, HELLO_TEMPLATE);
+      console.log(`created hello.tsx`);
+    }
+
+    console.log(`\ndone! run: bunx vargai render hello.tsx`);
+  },
+});

package/src/cli/commands/help.tsx

@@ -1,7 +1,4 @@
-/**
- * varg help command
- * Ink-based help display
- */
+/** @jsxImportSource react */
 
 import { defineCommand } from "citty";
 import { Box, Text } from "ink";
@@ -24,44 +21,35 @@ function CommandRow({ name, description }: CommandRowProps) {
 
 function HelpView() {
   const examples = [
-    {
-      command: 'varg run video --prompt "ocean waves"',
-      description: "generate a video from text",
-    },
-    {
-      command: 'varg run video --prompt "person talking" --image photo.jpg',
-      description: "generate video from image",
-    },
-    {
-      command: 'varg run voice --text "hello world" --voice sam',
-      description: "text to speech",
-    },
-    { command: "varg list", description: "see all available" },
-    {
-      command: "varg which video",
-      description: "inspect an action or model",
-    },
+    { command: "vargai hello", description: "create hello.tsx starter" },
+    { command: "vargai render hello.tsx", description: "render jsx to video" },
+    { command: "vargai init", description: "full setup with api keys" },
   ];
 
   return (
-    <VargBox title="varg">
+    <VargBox title="vargai">
       <Box marginBottom={1}>
-        <Text>ai video infrastructure from your terminal</Text>
+        <Text>ai video generation sdk. jsx for videos.</Text>
       </Box>
 
       <Header>COMMANDS</Header>
       <Box flexDirection="column" marginY={1}>
-        <CommandRow name="run" description="run a model, action, or skill" />
-        <CommandRow name="list" description="discover what's available" />
+        <CommandRow name="hello" description="create hello.tsx starter video" />
+        <CommandRow name="render" description="render jsx component to video" />
+        <CommandRow name="preview" description="fast preview (placeholders)" />
+        <CommandRow name="init" description="full setup with api keys" />
         <CommandRow
-          name="find"
-          description="search for models, actions, skills"
+          name="studio"
+          description="visual editor at localhost:8282"
+        />
+        <CommandRow name="run" description="run a single model or action" />
+        <CommandRow
+          name="list"
+          description="discover models, actions, skills"
         />
-        <CommandRow name="which" description="inspect a specific item" />
-        <CommandRow name="help" description="show this help" />
       </Box>
 
-      <Header>EXAMPLES</Header>
+      <Header>QUICKSTART</Header>
       <Box marginTop={1}>
         <HelpBlock examples={examples} />
       </Box>

package/src/cli/commands/index.ts

@@ -1,6 +1,14 @@
 export { findCmd, showFindHelp } from "./find.tsx";
+export { helloCmd } from "./hello.ts";
 export { helpCmd, showHelp } from "./help.tsx";
+export { initCmd, showInitHelp } from "./init.tsx";
 export { listCmd, showListHelp } from "./list.tsx";
-export { renderCmd } from "./render.tsx";
+export {
+  previewCmd,
+  renderCmd,
+  showPreviewHelp,
+  showRenderHelp,
+} from "./render.tsx";
 export { runCmd, showRunHelp, showTargetHelp } from "./run.tsx";
+export { studioCmd } from "./studio.ts";
 export { showWhichHelp, whichCmd } from "./which.tsx";