vargai 0.4.0-alpha30 → 0.4.0-alpha31

package/package.json

@@ -65,10 +65,11 @@
     "react-dom": "^19.2.0",
     "remotion": "^4.0.377",
     "replicate": "^1.4.0",
+    "sharp": "^0.34.5",
     "vargai": "^0.4.0-alpha11",
     "zod": "^4.2.1"
   },
-  "version": "0.4.0-alpha30",
+  "version": "0.4.0-alpha31",
   "exports": {
     ".": "./src/index.ts",
     "./ai": "./src/ai-sdk/index.ts",

package/src/ai-sdk/providers/editly/editly.test.ts

@@ -1105,4 +1105,50 @@ describe("editly", () => {
     expect(info.height).toBe(1920);
     expect(info.duration).toBeCloseTo(3, 0);
   });
+
+  test("video overlay with cropPosition", async () => {
+    const outPath = "output/editly-test-crop-position.mp4";
+    if (existsSync(outPath)) unlinkSync(outPath);
+
+    await editly({
+      outPath,
+      width: 1080,
+      height: 1920,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "fill-color", color: "#000000" },
+            {
+              type: "video",
+              path: VIDEO_1,
+              width: 1080,
+              height: 960,
+              left: 0,
+              top: 0,
+              resizeMode: "cover",
+              cropPosition: "top",
+            },
+            {
+              type: "video",
+              path: VIDEO_2,
+              width: 1080,
+              height: 960,
+              left: 0,
+              top: 960,
+              resizeMode: "cover",
+              cropPosition: "bottom",
+            },
+          ],
+        },
+      ],
+    });
+
+    expect(existsSync(outPath)).toBe(true);
+    const info = await ffprobe(outPath);
+    expect(info.width).toBe(1080);
+    expect(info.height).toBe(1920);
+    expect(info.duration).toBeCloseTo(3, 0);
+  });
 });

package/src/ai-sdk/providers/editly/layers.ts

@@ -1,4 +1,5 @@
 import type {
+  CropPosition,
   FillColorLayer,
   ImageLayer,
   ImageOverlayLayer,
@@ -15,6 +16,33 @@ import type {
   VideoLayer,
 } from "./types";
 
+function getCropPositionExpr(position: CropPosition | undefined): {
+  x: string;
+  y: string;
+} {
+  switch (position) {
+    case "top-left":
+      return { x: "0", y: "0" };
+    case "top":
+      return { x: "(iw-ow)/2", y: "0" };
+    case "top-right":
+      return { x: "iw-ow", y: "0" };
+    case "left":
+      return { x: "0", y: "(ih-oh)/2" };
+    case "right":
+      return { x: "iw-ow", y: "(ih-oh)/2" };
+    case "bottom-left":
+      return { x: "0", y: "ih-oh" };
+    case "bottom":
+      return { x: "(iw-ow)/2", y: "ih-oh" };
+    case "bottom-right":
+      return { x: "iw-ow", y: "ih-oh" };
+    case "center":
+    default:
+      return { x: "(iw-ow)/2", y: "(ih-oh)/2" };
+  }
+}
+
 function escapeDrawText(text: string): string {
   return text
     .replace(/\\/g, "\\\\")
@@ -161,9 +189,14 @@ export function getVideoFilterWithTrim(
   const layerHeight = parseSize(layer.height, height);
 
   if (isOverlay) {
-    filters.push(
-      `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=decrease`,
-    );
+    let scaleFilter = `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=decrease`;
+    if (layer.resizeMode === "cover") {
+      const { x, y } = getCropPositionExpr(layer.cropPosition);
+      scaleFilter = `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=increase,crop=${layerWidth}:${layerHeight}:${x}:${y}`;
+    } else if (layer.resizeMode === "stretch") {
+      scaleFilter = `scale=${layerWidth}:${layerHeight}`;
+    }
+    filters.push(scaleFilter);
     filters.push("setsar=1");
     filters.push("fps=30");
     filters.push("settb=1/30");
@@ -555,7 +588,13 @@ export function getTitleFilter(
 ): string {
   const text = escapeDrawText(layer.text);
   const color = layer.textColor ?? "white";
-  const fontSize = Math.round(Math.min(width, height) * 0.08);
+
+  // Auto-size font to fit within 90% of frame width (same approach as subtitle)
+  const maxFontSize = Math.round(Math.min(width, height) * 0.08);
+  const maxTextWidth = width * 0.9;
+  // Average char width ≈ fontSize * 0.55 for sans-serif fonts
+  const fittedFontSize = Math.floor(maxTextWidth / (layer.text.length * 0.55));
+  const fontSize = Math.max(16, Math.min(maxFontSize, fittedFontSize));
 
   let x = "(w-text_w)/2";
   let y = "(h-text_h)/2";
@@ -587,7 +626,13 @@ export function getSubtitleFilter(
   const text = escapeDrawText(layer.text);
   const textColor = layer.textColor ?? "white";
   const bgColor = layer.backgroundColor ?? "black@0.7";
-  const fontSize = Math.round(Math.min(width, height) * 0.05);
+
+  // Auto-size font to fit within 90% of frame width
+  const maxFontSize = Math.round(Math.min(width, height) * 0.05);
+  const maxTextWidth = width * 0.9;
+  // Average char width ≈ fontSize * 0.55 for sans-serif fonts
+  const fittedFontSize = Math.floor(maxTextWidth / (layer.text.length * 0.55));
+  const fontSize = Math.max(16, Math.min(maxFontSize, fittedFontSize));
   const boxPadding = Math.round(fontSize * 0.4);
 
   const fontFile = layer.fontPath
@@ -626,7 +671,14 @@ export function getTitleBackgroundFilter(
 
   const text = escapeDrawText(layer.text);
   const textColor = layer.textColor ?? "white";
-  const fontSize = Math.round(Math.min(width, height) * 0.1);
+
+  // Auto-size font to fit within 90% of frame width
+  const maxFontSizeBg = Math.round(Math.min(width, height) * 0.1);
+  const maxTextWidthBg = width * 0.9;
+  const fittedFontSizeBg = Math.floor(
+    maxTextWidthBg / (layer.text.length * 0.55),
+  );
+  const fontSize = Math.max(16, Math.min(maxFontSizeBg, fittedFontSizeBg));
 
   const fontFile = layer.fontPath
     ? `:fontfile='${escapeDrawText(layer.fontPath)}'`

package/src/ai-sdk/providers/editly/types.ts

@@ -87,6 +87,21 @@ export interface TextLayer extends BaseLayer {
   fontFamily?: string;
 }
 
+/**
+ * Crop position anchor for cover mode.
+ * NOTE: This is a varg extension to editly, not in the original.
+ */
+export type CropPosition =
+  | "center"
+  | "top"
+  | "bottom"
+  | "left"
+  | "right"
+  | "top-left"
+  | "top-right"
+  | "bottom-left"
+  | "bottom-right";
+
 /**
  * For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`.
  * If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`.
@@ -95,6 +110,7 @@ export interface VideoLayer extends BaseLayer {
   type: "video";
   path: string;
   resizeMode?: ResizeMode;
+  cropPosition?: CropPosition;
   cutFrom?: number;
   cutTo?: number;
   width?: SizeValue;

package/src/ai-sdk/providers/fal.ts

@@ -14,6 +14,11 @@ import { fal } from "@fal-ai/client";
 import type { VideoModelV3, VideoModelV3CallOptions } from "../video-model";
 
 const VIDEO_MODELS: Record<string, { t2v: string; i2v: string }> = {
+  // Kling v2.6 - latest with native audio generation
+  "kling-v2.6": {
+    t2v: "fal-ai/kling-video/v2.6/pro/text-to-video",
+    i2v: "fal-ai/kling-video/v2.6/pro/image-to-video",
+  },
   "kling-v2.5": {
     t2v: "fal-ai/kling-video/v2.5-turbo/pro/text-to-video",
     i2v: "fal-ai/kling-video/v2.5-turbo/pro/image-to-video",
@@ -40,6 +45,13 @@ const VIDEO_MODELS: Record<string, { t2v: string; i2v: string }> = {
   },
 };
 
+// Motion control models - video-to-video with motion transfer
+const MOTION_CONTROL_MODELS: Record<string, string> = {
+  "kling-v2.6-motion": "fal-ai/kling-video/v2.6/pro/motion-control",
+  "kling-v2.6-motion-standard":
+    "fal-ai/kling-video/v2.6/standard/motion-control",
+};
+
 // lipsync models - video + audio input
 const LIPSYNC_MODELS: Record<string, string> = {
   "sync-v2": "fal-ai/sync-lipsync",
@@ -157,7 +169,7 @@ class FalVideoModel implements VideoModelV3 {
     } = options;
     const warnings: SharedV3Warning[] = [];
 
-    const _hasVideoInput = files?.some((f) =>
+    const hasVideoInput = files?.some((f) =>
       getMediaType(f)?.startsWith("video/"),
     );
     const hasImageInput = files?.some((f) =>
@@ -168,15 +180,21 @@ class FalVideoModel implements VideoModelV3 {
     );
 
     const isLipsync = LIPSYNC_MODELS[this.modelId] !== undefined;
+    const isMotionControl = MOTION_CONTROL_MODELS[this.modelId] !== undefined;
+    const isKlingV26 = this.modelId === "kling-v2.6";
+
     const endpoint = isLipsync
       ? this.resolveLipsyncEndpoint()
-      : this.resolveEndpoint(hasImageInput ?? false);
+      : isMotionControl
+        ? this.resolveMotionControlEndpoint()
+        : this.resolveEndpoint(hasImageInput ?? false);
 
     const input: Record<string, unknown> = {
       ...(providerOptions?.fal ?? {}),
     };
 
     if (isLipsync) {
+      // Lipsync: video + audio input
       const videoFile = files?.find((f) =>
         getMediaType(f)?.startsWith("video/"),
       );
@@ -190,21 +208,70 @@ class FalVideoModel implements VideoModelV3 {
       if (audioFile) {
         input.audio_url = await fileToUrl(audioFile);
       }
+    } else if (isMotionControl) {
+      // Motion control: image + reference video input
+      if (prompt) {
+        input.prompt = prompt;
+      }
+
+      const imageFile = files?.find((f) =>
+        getMediaType(f)?.startsWith("image/"),
+      );
+      const videoFile = files?.find((f) =>
+        getMediaType(f)?.startsWith("video/"),
+      );
+
+      if (imageFile) {
+        input.image_url = await fileToUrl(imageFile);
+      }
+      if (videoFile) {
+        input.video_url = await fileToUrl(videoFile);
+      }
+
+      // Default character orientation to 'video' for better motion matching
+      if (!input.character_orientation) {
+        input.character_orientation = "video";
+      }
+
+      // Default to keeping original sound
+      if (input.keep_original_sound === undefined) {
+        input.keep_original_sound = true;
+      }
     } else {
+      // Standard video generation
       input.prompt = prompt;
-      input.duration = duration ?? 5;
+
+      // Duration must be string "5" or "10" for Kling v2.6
+      if (isKlingV26) {
+        input.duration = String(duration ?? 5);
+      } else {
+        input.duration = duration ?? 5;
+      }
 
       if (hasImageInput && files) {
-        const imageFile = files.find((f) =>
+        const imageFiles = files.filter((f) =>
           getMediaType(f)?.startsWith("image/"),
         );
-        if (imageFile) {
-          input.image_url = await fileToUrl(imageFile);
+        if (imageFiles.length > 0) {
+          // First image is start image
+          input.image_url = await fileToUrl(imageFiles[0]!);
+          // Second image (if provided) is end image for Kling v2.6
+          if (isKlingV26 && imageFiles.length > 1) {
+            input.end_image_url = await fileToUrl(imageFiles[1]!);
+          }
         }
       } else {
         input.aspect_ratio = aspectRatio ?? "16:9";
       }
 
+      // Kling v2.6 supports native audio generation
+      if (isKlingV26) {
+        // Default to generating audio unless explicitly disabled
+        if (input.generate_audio === undefined) {
+          input.generate_audio = true;
+        }
+      }
+
       const audioFile = files?.find((f) =>
         getMediaType(f)?.startsWith("audio/"),
       );
@@ -283,6 +350,14 @@ class FalVideoModel implements VideoModelV3 {
 
     return LIPSYNC_MODELS[this.modelId] ?? this.modelId;
   }
+
+  private resolveMotionControlEndpoint(): string {
+    if (this.modelId.startsWith("raw:")) {
+      return this.modelId.slice(4);
+    }
+
+    return MOTION_CONTROL_MODELS[this.modelId] ?? this.modelId;
+  }
 }
 
 class FalImageModel implements ImageModelV3 {

package/src/react/index.ts

@@ -17,7 +17,7 @@ export {
   Title,
   Video,
 } from "./elements";
-export { Grid, SplitLayout } from "./layouts";
+export { Grid, Slot, SplitLayout } from "./layouts";
 export { render, renderStream } from "./render";
 export type {
   CaptionsProps,

package/src/react/layouts/grid.tsx

@@ -21,7 +21,7 @@ export const Grid = ({
       top: `${(Math.floor(i / cols) / rowCount) * 100}%`,
       width: `${(1 / cols) * 100}%`,
       height: `${(1 / rowCount) * 100}%`,
-      resize,
+      resize: (el.props as Record<string, unknown>).resize ?? resize,
     },
   }));
   return <>{positioned}</>;

package/src/react/layouts/index.ts

@@ -1,2 +1,3 @@
 export { Grid } from "./grid";
-export { SplitLayout } from "./split";
+export { Slot } from "./slot";
+export { Split, SplitLayout } from "./split";

package/src/react/layouts/slot.tsx

@@ -0,0 +1,85 @@
+import type {
+  CropPosition,
+  ResizeMode,
+} from "../../ai-sdk/providers/editly/types";
+import type { VargElement } from "../types";
+
+type SlotFit = "cover" | "contain" | "contain-blur" | "fill";
+
+type SlotPosition =
+  | "center"
+  | "top"
+  | "bottom"
+  | "left"
+  | "right"
+  | "top-left"
+  | "top-right"
+  | "bottom-left"
+  | "bottom-right";
+
+interface SlotProps {
+  class?: string;
+  fit?: SlotFit;
+  position?: SlotPosition;
+  children: VargElement;
+}
+
+interface ParsedSlotClass {
+  fit?: SlotFit;
+  position?: SlotPosition;
+}
+
+function parseSlotClass(classString?: string): ParsedSlotClass {
+  if (!classString) return {};
+  const result: ParsedSlotClass = {};
+
+  for (const cls of classString.trim().split(/\s+/)) {
+    if (cls === "fit-cover") result.fit = "cover";
+    else if (cls === "fit-contain") result.fit = "contain";
+    else if (cls === "fit-contain-blur") result.fit = "contain-blur";
+    else if (cls === "fit-fill") result.fit = "fill";
+    else if (cls === "pos-center") result.position = "center";
+    else if (cls === "pos-top") result.position = "top";
+    else if (cls === "pos-bottom") result.position = "bottom";
+    else if (cls === "pos-left") result.position = "left";
+    else if (cls === "pos-right") result.position = "right";
+    else if (cls === "pos-top-left") result.position = "top-left";
+    else if (cls === "pos-top-right") result.position = "top-right";
+    else if (cls === "pos-bottom-left") result.position = "bottom-left";
+    else if (cls === "pos-bottom-right") result.position = "bottom-right";
+  }
+  return result;
+}
+
+function slotFitToResize(fit: SlotFit): ResizeMode {
+  switch (fit) {
+    case "cover":
+      return "cover";
+    case "contain":
+      return "contain";
+    case "contain-blur":
+      return "contain-blur";
+    case "fill":
+      return "stretch";
+  }
+}
+
+export const Slot = ({
+  class: className,
+  fit,
+  position,
+  children,
+}: SlotProps) => {
+  const parsed = parseSlotClass(className);
+  const resolvedFit = fit ?? parsed.fit ?? "cover";
+  const resolvedPosition = position ?? parsed.position ?? "center";
+
+  return {
+    ...children,
+    props: {
+      ...children.props,
+      resize: slotFitToResize(resolvedFit),
+      cropPosition: resolvedPosition as CropPosition,
+    },
+  } as VargElement;
+};

package/src/react/layouts/split.tsx

@@ -18,3 +18,21 @@ export const SplitLayout = ({
     {right}
   </Grid>
 );
+
+export const Split = ({
+  direction = "horizontal",
+  children,
+}: {
+  direction?: "horizontal" | "vertical";
+  children: VargElement[];
+}) => {
+  if (children.length === 0) return null;
+  return (
+    <Grid
+      columns={direction === "horizontal" ? children.length : 1}
+      rows={direction === "vertical" ? children.length : 1}
+    >
+      {children}
+    </Grid>
+  );
+};

package/src/react/renderers/captions.ts

@@ -69,14 +69,14 @@ interface SubtitleStyle {
 const STYLE_PRESETS: Record<string, SubtitleStyle> = {
   tiktok: {
     fontName: "Montserrat",
-    fontSize: 32,
+    fontSize: 72,
     primaryColor: "&HFFFFFF",
     outlineColor: "&H000000",
-    backColor: "&H80000000",
+    backColor: "&H00000000",
     bold: true,
-    outline: 3,
+    outline: 4,
     shadow: 0,
-    marginV: 50,
+    marginV: 480,
     alignment: 2,
   },
   karaoke: {
@@ -164,10 +164,17 @@ function formatAssTime(seconds: number): string {
   return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`;
 }
 
-function convertSrtToAss(srtContent: string, style: SubtitleStyle): string {
+function convertSrtToAss(
+  srtContent: string,
+  style: SubtitleStyle,
+  width: number,
+  height: number,
+): string {
   const assHeader = `[Script Info]
 Title: Generated Subtitles
 ScriptType: v4.00+
+PlayResX: ${width}
+PlayResY: ${height}
 WrapStyle: 0
 ScaledBorderAndShadow: yes
 YCbCr Matrix: TV.601
@@ -193,6 +200,12 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
   return assHeader + assDialogues;
 }
 
+const POSITION_ALIGNMENT: Record<string, number> = {
+  top: 8,
+  center: 5,
+  bottom: 2,
+};
+
 function colorToAss(color: string): string {
   if (color.startsWith("&H")) return color;
 
@@ -280,15 +293,21 @@ export async function renderCaptions(
   const styleName = props.style ?? "tiktok";
   const baseStyle = STYLE_PRESETS[styleName] ?? STYLE_PRESETS.tiktok!;
 
+  const alignment = props.position
+    ? (POSITION_ALIGNMENT[props.position] ?? baseStyle.alignment)
+    : baseStyle.alignment;
+
   const style: SubtitleStyle = {
     ...baseStyle,
     fontSize: props.fontSize ?? baseStyle.fontSize,
     primaryColor: props.color
       ? colorToAss(props.color)
       : baseStyle.primaryColor,
+    alignment,
+    marginV: props.position === "center" ? 0 : baseStyle.marginV,
   };
 
-  const assContent = convertSrtToAss(srtContent, style);
+  const assContent = convertSrtToAss(srtContent, style, ctx.width, ctx.height);
   const assPath = `/tmp/varg-captions-${Date.now()}.ass`;
   writeFileSync(assPath, assContent);
   ctx.tempFiles.push(assPath);

package/src/react/renderers/clip.ts

@@ -10,6 +10,7 @@ import type {
 import type {
   ClipProps,
   ImageProps,
+  MusicProps,
   SpeechProps,
   VargElement,
   VargNode,
@@ -17,6 +18,7 @@ import type {
 } from "../types";
 import type { RenderContext } from "./context";
 import { renderImage } from "./image";
+import { renderMusic } from "./music";
 import { renderPackshot } from "./packshot";
 import { renderSlider } from "./slider";
 import { renderSpeech } from "./speech";
@@ -24,6 +26,7 @@ import { renderSplit } from "./split";
 import { renderSubtitle } from "./subtitle";
 import { renderSwipe } from "./swipe";
 import { renderTitle } from "./title";
+import { resolvePath } from "./utils";
 import { renderVideo } from "./video";
 
 type PendingLayer =
@@ -90,7 +93,7 @@ async function renderClipLayers(
                 type: "video",
                 path,
                 resizeMode: props.resize,
-                // Video-level cutFrom/cutTo take precedence over clip-level
+                cropPosition: props.cropPosition,
                 cutFrom: props.cutFrom ?? clipOptions?.cutFrom,
                 cutTo: props.cutTo ?? clipOptions?.cutTo,
                 mixVolume: props.keepAudio ? (props.volume ?? 1) : 0,
@@ -136,6 +139,35 @@ async function renderClipLayers(
         break;
       }
 
+      case "music": {
+        const props = element.props as MusicProps;
+        pending.push({
+          type: "async",
+          promise: (async () => {
+            let path: string;
+            if (props.src) {
+              path = resolvePath(props.src);
+            } else if (props.prompt) {
+              const result = await renderMusic(
+                element as VargElement<"music">,
+                ctx,
+              );
+              path = result.path;
+            } else {
+              throw new Error("Music requires either src or prompt");
+            }
+            return {
+              type: "audio",
+              path,
+              mixVolume: props.volume ?? 1,
+              cutFrom: props.cutFrom,
+              cutTo: props.cutTo,
+            } as AudioLayer;
+          })(),
+        });
+        break;
+      }
+
       case "split": {
         pending.push({
           type: "async",

package/src/react/renderers/music.ts

@@ -40,6 +40,11 @@ export async function renderMusic(
     const cached = await ctx.cache.get(cacheKey);
     if (cached) {
       audioData = cached as Uint8Array;
+      // Signal cache hit to progress tracker
+      if (taskId && ctx.progress) {
+        startTask(ctx.progress, taskId);
+        completeTask(ctx.progress, taskId);
+      }
     } else {
       if (taskId && ctx.progress) startTask(ctx.progress, taskId);
       audioData = await generateFn();