vargai 0.3.2 → 0.4.0-alpha2

package/src/react/renderers/progress.ts

@@ -0,0 +1,173 @@
+export type GenerationType =
+  | "image"
+  | "video"
+  | "animate"
+  | "speech"
+  | "music"
+  | "editly"
+  | "captions"
+  | "transcribe";
+
+export const TIME_ESTIMATES: Record<GenerationType, number> = {
+  image: 30,
+  video: 120,
+  animate: 90,
+  speech: 5,
+  music: 45,
+  editly: 15,
+  captions: 10,
+  transcribe: 15,
+};
+
+export const MODEL_TIME_ESTIMATES: Record<string, number> = {
+  // image models - use partial matching
+  "flux/schnell": 3,
+  "flux/dev": 8,
+  "flux-pro": 12,
+  recraft: 10,
+  ideogram: 8,
+  // video models
+  kling: 180,
+  "kling-v2": 180,
+  "kling-v2.5": 180,
+  minimax: 90,
+  luma: 90,
+  runway: 45,
+  veo: 120,
+  // speech models
+  elevenlabs: 5,
+  eleven: 5,
+  // music models
+  "stable-audio": 30,
+  musicgen: 45,
+};
+
+export interface ProgressTask {
+  id: string;
+  type: GenerationType;
+  model: string;
+  status: "pending" | "running" | "done";
+  startedAt?: number;
+  completedAt?: number;
+}
+
+export interface ProgressTracker {
+  tasks: ProgressTask[];
+  onUpdate?: (tracker: ProgressTracker) => void;
+  quiet: boolean;
+}
+
+export function createProgressTracker(quiet = false): ProgressTracker {
+  return {
+    tasks: [],
+    quiet,
+  };
+}
+
+export function addTask(
+  tracker: ProgressTracker,
+  type: GenerationType,
+  model: string,
+): string {
+  const id = `${type}-${tracker.tasks.length}`;
+  tracker.tasks.push({ id, type, model, status: "pending" });
+  return id;
+}
+
+export function startTask(tracker: ProgressTracker, id: string): void {
+  const task = tracker.tasks.find((t) => t.id === id);
+  if (task) {
+    task.status = "running";
+    task.startedAt = Date.now();
+    if (!tracker.quiet) {
+      logTaskStart(task);
+    }
+  }
+  tracker.onUpdate?.(tracker);
+}
+
+const CACHE_THRESHOLD_MS = 1000;
+
+export function completeTask(tracker: ProgressTracker, id: string): void {
+  const task = tracker.tasks.find((t) => t.id === id);
+  if (task) {
+    task.status = "done";
+    task.completedAt = Date.now();
+    const duration = task.startedAt ? task.completedAt - task.startedAt : 0;
+    const wasCached = duration < CACHE_THRESHOLD_MS;
+    if (!tracker.quiet) {
+      logTaskComplete(task, wasCached);
+    }
+  }
+  tracker.onUpdate?.(tracker);
+}
+
+function getEstimate(task: ProgressTask): number {
+  const modelLower = task.model.toLowerCase();
+  for (const [key, estimate] of Object.entries(MODEL_TIME_ESTIMATES)) {
+    if (modelLower.includes(key.toLowerCase())) {
+      return estimate;
+    }
+  }
+  return TIME_ESTIMATES[task.type];
+}
+
+function logTaskStart(task: ProgressTask): void {
+  const estimate = getEstimate(task);
+  console.log(`⏳ generating ${task.type} with ${task.model} (~${estimate}s)`);
+}
+
+function logTaskComplete(task: ProgressTask, cached: boolean): void {
+  const duration =
+    task.completedAt && task.startedAt
+      ? ((task.completedAt - task.startedAt) / 1000).toFixed(1)
+      : "?";
+  if (cached) {
+    console.log(`⚡ ${task.type} from cache`);
+  } else {
+    console.log(`✓ ${task.type} done (${duration}s)`);
+  }
+}
+
+export function getTotalEstimate(tracker: ProgressTracker): number {
+  return tracker.tasks.reduce((sum, task) => sum + getEstimate(task), 0);
+}
+
+export function getElapsedTime(tracker: ProgressTracker): number {
+  const completed = tracker.tasks.filter((t) => t.status === "done");
+  const running = tracker.tasks.find((t) => t.status === "running");
+
+  let elapsed = 0;
+  for (const task of completed) {
+    if (task.startedAt && task.completedAt) {
+      elapsed += (task.completedAt - task.startedAt) / 1000;
+    }
+  }
+  if (running?.startedAt) {
+    elapsed += (Date.now() - running.startedAt) / 1000;
+  }
+  return elapsed;
+}
+
+export function getProgress(tracker: ProgressTracker): number {
+  const total = tracker.tasks.length;
+  if (total === 0) return 0;
+  const done = tracker.tasks.filter((t) => t.status === "done").length;
+  return Math.round((done / total) * 100);
+}
+
+export function renderProgressBar(tracker: ProgressTracker): string {
+  const progress = getProgress(tracker);
+  const total = getTotalEstimate(tracker);
+  const elapsed = getElapsedTime(tracker);
+
+  const width = 30;
+  const filled = Math.round((progress / 100) * width);
+  const empty = width - filled;
+  const bar = "█".repeat(filled) + "░".repeat(empty);
+
+  const done = tracker.tasks.filter((t) => t.status === "done").length;
+  const totalTasks = tracker.tasks.length;
+
+  return `[${bar}] ${progress}% (${done}/${totalTasks} tasks, ~${Math.round(total - elapsed)}s remaining)`;
+}

package/src/react/renderers/render.ts

@@ -0,0 +1,319 @@
+import { generateImage, wrapImageModel } from "ai";
+import { withCache } from "../../ai-sdk/cache";
+import { fileCache } from "../../ai-sdk/file-cache";
+import { generateVideo } from "../../ai-sdk/generate-video";
+import {
+  imagePlaceholderFallbackMiddleware,
+  placeholderFallbackMiddleware,
+  wrapVideoModel,
+} from "../../ai-sdk/middleware";
+import { editly } from "../../ai-sdk/providers/editly";
+import type {
+  AudioTrack,
+  Clip,
+  Layer,
+  VideoLayer,
+} from "../../ai-sdk/providers/editly/types";
+import type {
+  CaptionsProps,
+  ClipProps,
+  MusicProps,
+  OverlayProps,
+  RenderMode,
+  RenderOptions,
+  RenderProps,
+  SpeechProps,
+  VargElement,
+} from "../types";
+import { renderAnimate } from "./animate";
+import { renderCaptions } from "./captions";
+import { renderClip } from "./clip";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderMusic } from "./music";
+import {
+  addTask,
+  completeTask,
+  createProgressTracker,
+  startTask,
+} from "./progress";
+import { renderSpeech } from "./speech";
+import { resolvePath } from "./utils";
+import { renderVideo } from "./video";
+
+interface RenderedOverlay {
+  path: string;
+  props: OverlayProps;
+  isVideo: boolean;
+}
+
+export async function renderRoot(
+  element: VargElement<"render">,
+  options: RenderOptions,
+): Promise<Uint8Array> {
+  const props = element.props as RenderProps;
+  const progress = createProgressTracker(options.quiet ?? false);
+
+  const mode: RenderMode = options.mode ?? "default";
+  const placeholderCount = { images: 0, videos: 0, total: 0 };
+
+  const onFallback = (error: Error, prompt: string) => {
+    if (!options.quiet) {
+      console.warn(
+        `\x1b[33m⚠ provider failed: ${error.message} → placeholder\x1b[0m`,
+      );
+    }
+  };
+
+  const trackPlaceholder = (type: "image" | "video") => {
+    placeholderCount[type === "image" ? "images" : "videos"]++;
+    placeholderCount.total++;
+  };
+
+  const wrapGenerateImage: typeof generateImage = async (opts) => {
+    if (
+      typeof opts.model === "string" ||
+      opts.model.specificationVersion !== "v3"
+    ) {
+      return generateImage(opts);
+    }
+    const wrappedModel = wrapImageModel({
+      model: opts.model,
+      middleware: imagePlaceholderFallbackMiddleware({
+        mode,
+        onFallback: (error, prompt) => {
+          trackPlaceholder("image");
+          onFallback(error, prompt);
+        },
+      }),
+    });
+    const result = await generateImage({ ...opts, model: wrappedModel });
+    if (mode === "preview") trackPlaceholder("image");
+    return result;
+  };
+
+  const wrapGenerateVideo: typeof generateVideo = async (opts) => {
+    const wrappedModel = wrapVideoModel({
+      model: opts.model,
+      middleware: placeholderFallbackMiddleware({
+        mode,
+        onFallback: (error, prompt) => {
+          trackPlaceholder("video");
+          onFallback(error, prompt);
+        },
+      }),
+    });
+    const result = await generateVideo({ ...opts, model: wrappedModel });
+    if (mode === "preview") trackPlaceholder("video");
+    return result;
+  };
+
+  const ctx: RenderContext = {
+    width: props.width ?? 1920,
+    height: props.height ?? 1080,
+    fps: props.fps ?? 30,
+    cache: options.cache ? fileCache({ dir: options.cache }) : undefined,
+    generateImage: options.cache
+      ? withCache(wrapGenerateImage, {
+          storage: fileCache({ dir: options.cache }),
+        })
+      : wrapGenerateImage,
+    generateVideo: options.cache
+      ? withCache(wrapGenerateVideo, {
+          storage: fileCache({ dir: options.cache }),
+        })
+      : wrapGenerateVideo,
+    tempFiles: [],
+    progress,
+    pending: new Map(),
+  };
+
+  const clipElements: VargElement<"clip">[] = [];
+  const overlayElements: VargElement<"overlay">[] = [];
+  const musicElements: VargElement<"music">[] = [];
+  const audioTracks: AudioTrack[] = [];
+  let captionsResult: Awaited<ReturnType<typeof renderCaptions>> | undefined;
+
+  for (const child of element.children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+
+    const childElement = child as VargElement;
+
+    if (childElement.type === "clip") {
+      clipElements.push(childElement as VargElement<"clip">);
+    } else if (childElement.type === "overlay") {
+      overlayElements.push(childElement as VargElement<"overlay">);
+    } else if (childElement.type === "captions") {
+      captionsResult = await renderCaptions(
+        childElement as VargElement<"captions">,
+        ctx,
+      );
+      if (captionsResult.audioPath) {
+        audioTracks.push({
+          path: captionsResult.audioPath,
+          mixVolume: 1,
+        });
+      }
+    } else if (childElement.type === "speech") {
+      const result = await renderSpeech(
+        childElement as VargElement<"speech">,
+        ctx,
+      );
+      const speechProps = childElement.props as SpeechProps;
+      audioTracks.push({
+        path: result.path,
+        mixVolume: speechProps.volume ?? 1,
+      });
+    } else if (childElement.type === "music") {
+      musicElements.push(childElement as VargElement<"music">);
+    }
+  }
+
+  const renderedOverlays: RenderedOverlay[] = [];
+  for (const overlay of overlayElements) {
+    const overlayProps = overlay.props as OverlayProps;
+    for (const child of overlay.children) {
+      if (!child || typeof child !== "object" || !("type" in child)) continue;
+      const childElement = child as VargElement;
+
+      let path: string | undefined;
+      const isVideo =
+        childElement.type === "video" || childElement.type === "animate";
+
+      if (childElement.type === "video") {
+        path = await renderVideo(childElement as VargElement<"video">, ctx);
+      } else if (childElement.type === "animate") {
+        path = await renderAnimate(childElement as VargElement<"animate">, ctx);
+      } else if (childElement.type === "image") {
+        path = await renderImage(childElement as VargElement<"image">, ctx);
+      }
+
+      if (path) {
+        renderedOverlays.push({ path, props: overlayProps, isVideo });
+
+        if (isVideo && overlayProps.keepAudio) {
+          audioTracks.push({
+            path,
+            mixVolume: overlayProps.volume ?? 1,
+          });
+        }
+      }
+    }
+  }
+
+  const renderedClips = await Promise.all(
+    clipElements.map((clipElement) => renderClip(clipElement, ctx)),
+  );
+
+  const clips: Clip[] = [];
+  let currentTime = 0;
+
+  for (let i = 0; i < clipElements.length; i++) {
+    const clipElement = clipElements[i];
+    const clip = renderedClips[i];
+    if (!clipElement || !clip) {
+      throw new Error(`Missing clip data at index ${i}`);
+    }
+    const clipProps = clipElement.props as ClipProps;
+    const clipDuration =
+      typeof clipProps.duration === "number" ? clipProps.duration : 3;
+
+    for (const overlay of renderedOverlays) {
+      const overlayLayer: VideoLayer = {
+        type: "video",
+        path: overlay.path,
+        cutFrom: currentTime,
+        cutTo: currentTime + clipDuration,
+        left: overlay.props.left,
+        top: overlay.props.top,
+        width: overlay.props.width,
+        height: overlay.props.height,
+      };
+      clip.layers.push(overlayLayer as Layer);
+    }
+
+    clips.push(clip);
+
+    currentTime += clipDuration;
+    if (i < clipElements.length - 1 && clip.transition) {
+      currentTime -= clip.transition.duration ?? 0;
+    }
+  }
+
+  const totalDuration = currentTime;
+
+  // process music after clips so we know total duration for auto-trim
+  for (const musicElement of musicElements) {
+    const musicProps = musicElement.props as MusicProps;
+    const cutFrom = musicProps.cutFrom ?? 0;
+    const cutTo =
+      musicProps.cutTo ??
+      (musicProps.duration !== undefined
+        ? cutFrom + musicProps.duration
+        : totalDuration); // auto-trim to video length
+
+    let path: string;
+    if (musicProps.src) {
+      path = resolvePath(musicProps.src);
+    } else if (musicProps.prompt && musicProps.model) {
+      const result = await renderMusic(musicElement, ctx);
+      path = result.path;
+    } else {
+      throw new Error("Music requires either src or prompt+model");
+    }
+
+    audioTracks.push({
+      path,
+      mixVolume: musicProps.volume ?? 1,
+      cutFrom,
+      cutTo,
+    });
+  }
+
+  const hasCaptions = captionsResult !== undefined;
+
+  const tempOutPath = hasCaptions
+    ? `/tmp/varg-pre-captions-${Date.now()}.mp4`
+    : (options.output ?? `output/varg-${Date.now()}.mp4`);
+  const finalOutPath = options.output ?? `output/varg-${Date.now()}.mp4`;
+
+  const editlyTaskId = addTask(progress, "editly", "ffmpeg");
+  startTask(progress, editlyTaskId);
+
+  await editly({
+    outPath: tempOutPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips,
+    audioTracks: audioTracks.length > 0 ? audioTracks : undefined,
+  });
+
+  completeTask(progress, editlyTaskId);
+
+  if (hasCaptions && captionsResult) {
+    const captionsTaskId = addTask(progress, "captions", "ffmpeg");
+    startTask(progress, captionsTaskId);
+
+    const { $ } = await import("bun");
+    await $`ffmpeg -y -i ${tempOutPath} -vf "ass=${captionsResult.assPath}" -c:a copy ${finalOutPath}`.quiet();
+
+    ctx.tempFiles.push(tempOutPath);
+    completeTask(progress, captionsTaskId);
+  }
+
+  if (!options.quiet && placeholderCount.total > 0) {
+    if (mode === "preview") {
+      console.log(
+        `\x1b[36mℹ preview mode: ${placeholderCount.total} placeholders used (${placeholderCount.images} images, ${placeholderCount.videos} videos)\x1b[0m`,
+      );
+    } else {
+      console.warn(
+        `\x1b[33m⚠ ${placeholderCount.total} elements used placeholders - run with --strict for production\x1b[0m`,
+      );
+    }
+  }
+
+  const result = await Bun.file(finalOutPath).arrayBuffer();
+  return new Uint8Array(result);
+}

package/src/react/renderers/slider.ts

@@ -0,0 +1,69 @@
+import { editly } from "../../ai-sdk/providers/editly";
+import type { Clip } from "../../ai-sdk/providers/editly/types";
+import type { SliderProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderVideo } from "./video";
+
+export async function renderSlider(
+  element: VargElement<"slider">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as SliderProps;
+  const direction = props.direction ?? "horizontal";
+
+  const childPaths: string[] = [];
+
+  for (const child of element.children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+    const childElement = child as VargElement;
+
+    if (childElement.type === "image") {
+      const path = await renderImage(childElement as VargElement<"image">, ctx);
+      childPaths.push(path);
+    } else if (childElement.type === "video") {
+      const path = await renderVideo(childElement as VargElement<"video">, ctx);
+      childPaths.push(path);
+    }
+  }
+
+  if (childPaths.length === 0) {
+    throw new Error(
+      "Slider element requires at least one image or video child",
+    );
+  }
+
+  if (childPaths.length === 1) {
+    return childPaths[0]!;
+  }
+
+  const transitionName = direction === "horizontal" ? "slideleft" : "slideup";
+
+  const clips: Clip[] = childPaths.map((path, i) => {
+    const isVideo = path.endsWith(".mp4") || path.endsWith(".webm");
+    const isLast = i === childPaths.length - 1;
+
+    return {
+      layers: [
+        isVideo
+          ? { type: "video" as const, path, resizeMode: "cover" as const }
+          : { type: "image" as const, path, resizeMode: "cover" as const },
+      ],
+      duration: 3,
+      transition: isLast ? null : { name: transitionName, duration: 0.5 },
+    };
+  });
+
+  const outPath = `/tmp/varg-slider-${Date.now()}.mp4`;
+
+  await editly({
+    outPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips,
+  });
+
+  ctx.tempFiles.push(outPath);
+  return outPath;
+}

package/src/react/renderers/speech.ts

@@ -0,0 +1,53 @@
+import { experimental_generateSpeech as generateSpeech } from "ai";
+import { File } from "../../ai-sdk/file";
+import type { SpeechProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { addTask, completeTask, startTask } from "./progress";
+import { computeCacheKey, getTextContent } from "./utils";
+
+export interface SpeechResult {
+  path: string;
+  duration?: number;
+}
+
+export async function renderSpeech(
+  element: VargElement<"speech">,
+  ctx: RenderContext,
+): Promise<SpeechResult> {
+  const props = element.props as SpeechProps;
+  const text = getTextContent(element.children);
+
+  if (!text) {
+    throw new Error("Speech element requires text content");
+  }
+
+  const model = props.model;
+  if (!model) {
+    throw new Error("Speech element requires 'model' prop");
+  }
+
+  const cacheKey = computeCacheKey(element);
+
+  const modelId = typeof model === "string" ? model : model.modelId;
+  const taskId = ctx.progress ? addTask(ctx.progress, "speech", modelId) : null;
+  if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+
+  const { audio } = await generateSpeech({
+    model,
+    text,
+    voice: props.voice ?? "adam",
+    cacheKey,
+  } as Parameters<typeof generateSpeech>[0]);
+
+  if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+
+  const tempPath = await File.toTemp({
+    uint8Array: audio.uint8Array,
+    mimeType: "audio/mpeg",
+  });
+  ctx.tempFiles.push(tempPath);
+
+  return {
+    path: tempPath,
+  };
+}

package/src/react/renderers/split.ts

@@ -0,0 +1,91 @@
+import { editly } from "../../ai-sdk/providers/editly";
+import type { Clip, Layer } from "../../ai-sdk/providers/editly/types";
+import type { SplitProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderVideo } from "./video";
+
+export async function renderSplit(
+  element: VargElement<"split">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as SplitProps;
+  const direction = props.direction ?? "horizontal";
+
+  const childPaths: string[] = [];
+
+  for (const child of element.children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+    const childElement = child as VargElement;
+
+    if (childElement.type === "image") {
+      const path = await renderImage(childElement as VargElement<"image">, ctx);
+      childPaths.push(path);
+    } else if (childElement.type === "video") {
+      const path = await renderVideo(childElement as VargElement<"video">, ctx);
+      childPaths.push(path);
+    }
+  }
+
+  if (childPaths.length === 0) {
+    throw new Error("Split element requires at least one image or video child");
+  }
+
+  if (childPaths.length === 1) {
+    return childPaths[0]!;
+  }
+
+  const numChildren = childPaths.length;
+  const cellWidth =
+    direction === "horizontal"
+      ? Math.floor(ctx.width / numChildren)
+      : ctx.width;
+  const cellHeight =
+    direction === "vertical"
+      ? Math.floor(ctx.height / numChildren)
+      : ctx.height;
+
+  const layers: Layer[] = childPaths.map((path, i) => {
+    const isVideo = path.endsWith(".mp4") || path.endsWith(".webm");
+    const left = direction === "horizontal" ? cellWidth * i : 0;
+    const top = direction === "vertical" ? cellHeight * i : 0;
+
+    if (isVideo) {
+      return {
+        type: "video" as const,
+        path,
+        left,
+        top,
+        width: cellWidth,
+        height: cellHeight,
+      };
+    }
+    return {
+      type: "image-overlay" as const,
+      path,
+      position: { x: left, y: top },
+      width: cellWidth,
+      height: cellHeight,
+    };
+  });
+
+  layers.unshift({ type: "fill-color" as const, color: "#000000" });
+
+  const clip: Clip = {
+    layers,
+    duration: 5,
+  };
+
+  const outPath = `/tmp/varg-split-${Date.now()}.mp4`;
+
+  await editly({
+    outPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips: [clip],
+  });
+
+  ctx.tempFiles.push(outPath);
+  return outPath;
+}