vargai 0.3.1 → 0.4.0-alpha.1

package/src/react/renderers/clip.ts

@@ -0,0 +1,248 @@
+import type {
+  AudioLayer,
+  Clip,
+  FillColorLayer,
+  ImageLayer,
+  ImageOverlayLayer,
+  Layer,
+  VideoLayer,
+} from "../../ai-sdk/providers/editly/types";
+import type {
+  AnimateProps,
+  ClipProps,
+  ImageProps,
+  SpeechProps,
+  VargElement,
+  VargNode,
+  VideoProps,
+} from "../types";
+import { renderAnimate } from "./animate";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderPackshot } from "./packshot";
+import { renderSlider } from "./slider";
+import { renderSpeech } from "./speech";
+import { renderSplit } from "./split";
+import { renderSubtitle } from "./subtitle";
+import { renderSwipe } from "./swipe";
+import { renderTitle } from "./title";
+import { renderVideo } from "./video";
+
+type PendingLayer =
+  | { type: "sync"; layer: Layer }
+  | { type: "async"; promise: Promise<Layer> };
+
+async function renderClipLayers(
+  children: VargNode[],
+  ctx: RenderContext,
+): Promise<Layer[]> {
+  const pending: PendingLayer[] = [];
+
+  for (const child of children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+
+    const element = child as VargElement;
+
+    switch (element.type) {
+      case "image": {
+        const props = element.props as ImageProps;
+        const hasPosition =
+          props.left !== undefined ||
+          props.top !== undefined ||
+          props.width !== undefined ||
+          props.height !== undefined;
+
+        pending.push({
+          type: "async",
+          promise: renderImage(element as VargElement<"image">, ctx).then(
+            (path) =>
+              hasPosition
+                ? ({
+                    type: "image-overlay",
+                    path,
+                    zoomDirection: props.zoom,
+                    width: props.width,
+                    height: props.height,
+                    position: { x: props.left ?? 0, y: props.top ?? 0 },
+                  } as ImageOverlayLayer)
+                : ({
+                    type: "image",
+                    path,
+                    resizeMode: props.resize,
+                    zoomDirection: props.zoom,
+                  } as ImageLayer),
+          ),
+        });
+        break;
+      }
+
+      case "video": {
+        const props = element.props as VideoProps;
+        pending.push({
+          type: "async",
+          promise: renderVideo(element as VargElement<"video">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+                resizeMode: props.resize,
+                cutFrom: props.cutFrom,
+                cutTo: props.cutTo,
+                mixVolume: props.keepAudio ? (props.volume ?? 1) : 0,
+                left: props.left,
+                top: props.top,
+                width: props.width,
+                height: props.height,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "animate": {
+        const props = element.props as AnimateProps;
+        pending.push({
+          type: "async",
+          promise: renderAnimate(element as VargElement<"animate">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+                left: props.left,
+                top: props.top,
+                width: props.width,
+                height: props.height,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "title": {
+        pending.push({
+          type: "sync",
+          layer: renderTitle(element as VargElement<"title">),
+        });
+        break;
+      }
+
+      case "subtitle": {
+        pending.push({
+          type: "sync",
+          layer: renderSubtitle(element as VargElement<"subtitle">),
+        });
+        break;
+      }
+
+      case "speech": {
+        const props = element.props as SpeechProps;
+        pending.push({
+          type: "async",
+          promise: renderSpeech(element as VargElement<"speech">, ctx).then(
+            (result) =>
+              ({
+                type: "audio",
+                path: result.path,
+                mixVolume: props.volume ?? 1,
+              }) as AudioLayer,
+          ),
+        });
+        break;
+      }
+
+      case "split": {
+        pending.push({
+          type: "async",
+          promise: renderSplit(element as VargElement<"split">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "slider": {
+        pending.push({
+          type: "async",
+          promise: renderSlider(element as VargElement<"slider">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "swipe": {
+        pending.push({
+          type: "async",
+          promise: renderSwipe(element as VargElement<"swipe">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "packshot": {
+        pending.push({
+          type: "async",
+          promise: renderPackshot(element as VargElement<"packshot">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+    }
+  }
+
+  const layers = await Promise.all(
+    pending.map((p) => (p.type === "sync" ? p.layer : p.promise)),
+  );
+
+  return layers;
+}
+
+export async function renderClip(
+  element: VargElement<"clip">,
+  ctx: RenderContext,
+): Promise<Clip> {
+  const props = element.props as ClipProps;
+  const layers = await renderClipLayers(element.children, ctx);
+
+  const isOverlayVideo = (l: Layer) =>
+    l.type === "video" &&
+    ((l as VideoLayer).left !== undefined ||
+      (l as VideoLayer).top !== undefined ||
+      (l as VideoLayer).width !== undefined ||
+      (l as VideoLayer).height !== undefined);
+
+  const hasBaseLayer = layers.some(
+    (l) =>
+      l.type === "image" ||
+      l.type === "fill-color" ||
+      (l.type === "video" && !isOverlayVideo(l)),
+  );
+
+  if (!hasBaseLayer && layers.length > 0) {
+    layers.unshift({ type: "fill-color", color: "#000000" } as FillColorLayer);
+  }
+
+  return {
+    layers,
+    duration: typeof props.duration === "number" ? props.duration : undefined,
+    transition: props.transition ?? null,
+  };
+}

package/src/react/renderers/context.ts

@@ -0,0 +1,17 @@
+import type { generateImage } from "ai";
+import type { fileCache } from "../../ai-sdk/file-cache";
+import type { generateVideo } from "../../ai-sdk/generate-video";
+import type { ProgressTracker } from "./progress";
+
+export interface RenderContext {
+  width: number;
+  height: number;
+  fps: number;
+  cache?: ReturnType<typeof fileCache>;
+  generateImage: typeof generateImage;
+  generateVideo: typeof generateVideo;
+  tempFiles: string[];
+  progress?: ProgressTracker;
+  /** In-memory deduplication for concurrent renders of the same element */
+  pending: Map<string, Promise<string>>;
+}

package/src/react/renderers/image.ts

@@ -0,0 +1,109 @@
+import type { generateImage } from "ai";
+import { File } from "../../ai-sdk/file";
+import type {
+  ImageInput,
+  ImagePrompt,
+  ImageProps,
+  VargElement,
+} from "../types";
+import type { RenderContext } from "./context";
+import { addTask, completeTask, startTask } from "./progress";
+import { computeCacheKey, toFileUrl } from "./utils";
+
+async function resolveImageInput(
+  input: ImageInput,
+  ctx: RenderContext,
+): Promise<Uint8Array> {
+  if (input instanceof Uint8Array) {
+    return input;
+  }
+  if (typeof input === "string") {
+    const response = await fetch(toFileUrl(input));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  const path = await renderImage(input, ctx);
+  const response = await fetch(toFileUrl(path));
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function resolvePrompt(
+  prompt: ImagePrompt,
+  ctx: RenderContext,
+): Promise<string | { text?: string; images: Uint8Array[] }> {
+  if (typeof prompt === "string") {
+    return prompt;
+  }
+  const resolvedImages = await Promise.all(
+    prompt.images.map((img) => resolveImageInput(img, ctx)),
+  );
+  return { text: prompt.text, images: resolvedImages };
+}
+
+export async function renderImage(
+  element: VargElement<"image">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as ImageProps;
+
+  if (props.src) {
+    return props.src;
+  }
+
+  const prompt = props.prompt;
+  if (!prompt) {
+    throw new Error("Image element requires either 'prompt' or 'src'");
+  }
+
+  const model = props.model;
+  if (!model) {
+    throw new Error("Image element requires 'model' prop when using prompt");
+  }
+
+  // Compute cache key for deduplication
+  const cacheKey = computeCacheKey(element);
+  const cacheKeyStr = JSON.stringify(cacheKey);
+
+  // Check if this element is already being rendered (deduplication)
+  const pendingRender = ctx.pending.get(cacheKeyStr);
+  if (pendingRender) {
+    return pendingRender;
+  }
+
+  // Create the render promise and store it for deduplication
+  const renderPromise = (async () => {
+    const resolvedPrompt = await resolvePrompt(prompt, ctx);
+
+    const modelId = typeof model === "string" ? model : model.modelId;
+    const taskId = ctx.progress
+      ? addTask(ctx.progress, "image", modelId)
+      : null;
+    if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+
+    const { images } = await ctx.generateImage({
+      model,
+      prompt: resolvedPrompt,
+      aspectRatio: props.aspectRatio,
+      n: 1,
+      cacheKey,
+    } as Parameters<typeof generateImage>[0]);
+
+    if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+
+    const firstImage = images[0];
+    if (!firstImage?.uint8Array) {
+      throw new Error("Image generation returned no image data");
+    }
+    const imageData = firstImage.uint8Array;
+    const tempPath = await File.toTemp({
+      uint8Array: imageData,
+      mimeType: "image/png",
+    });
+    ctx.tempFiles.push(tempPath);
+
+    return tempPath;
+  })();
+
+  ctx.pending.set(cacheKeyStr, renderPromise);
+
+  return renderPromise;
+}

package/src/react/renderers/index.ts

@@ -0,0 +1,22 @@
+export { renderAnimate } from "./animate";
+export { renderCaptions } from "./captions";
+export { renderClip } from "./clip";
+export type { RenderContext } from "./context";
+export { renderImage } from "./image";
+export { renderPackshot } from "./packshot";
+export {
+  createProgressTracker,
+  type GenerationType,
+  type ProgressTask,
+  type ProgressTracker,
+  TIME_ESTIMATES,
+} from "./progress";
+export { renderRoot } from "./render";
+export { renderSlider } from "./slider";
+export type { SpeechResult } from "./speech";
+export { renderSpeech } from "./speech";
+export { renderSplit } from "./split";
+export { renderSwipe } from "./swipe";
+export { renderTitle } from "./title";
+export { computeCacheKey, getTextContent } from "./utils";
+export { renderVideo } from "./video";

package/src/react/renderers/music.ts

@@ -0,0 +1,60 @@
+import { generateMusic } from "../../ai-sdk/generate-music";
+import type { MusicProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { addTask, completeTask, startTask } from "./progress";
+
+export async function renderMusic(
+  element: VargElement<"music">,
+  ctx: RenderContext,
+): Promise<{ path: string }> {
+  const props = element.props as MusicProps;
+
+  const prompt = props.prompt;
+  const model = props.model;
+  if (!prompt || !model) {
+    throw new Error("Music generation requires both prompt and model");
+  }
+
+  const cacheKey = JSON.stringify({
+    type: "music",
+    prompt,
+    model: model.modelId,
+    duration: props.duration,
+  });
+
+  const modelId = model.modelId;
+  const taskId = ctx.progress ? addTask(ctx.progress, "music", modelId) : null;
+
+  const generateFn = async () => {
+    const result = await generateMusic({
+      model,
+      prompt,
+      duration: props.duration,
+    });
+    return result.audio.uint8Array;
+  };
+
+  let audioData: Uint8Array;
+
+  if (ctx.cache) {
+    const cached = await ctx.cache.get(cacheKey);
+    if (cached) {
+      audioData = cached as Uint8Array;
+    } else {
+      if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+      audioData = await generateFn();
+      if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+      await ctx.cache.set(cacheKey, audioData);
+    }
+  } else {
+    if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+    audioData = await generateFn();
+    if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+  }
+
+  const tempPath = `/tmp/varg-music-${Date.now()}.mp3`;
+  await Bun.write(tempPath, audioData);
+  ctx.tempFiles.push(tempPath);
+
+  return { path: tempPath };
+}

package/src/react/renderers/packshot.ts

@@ -0,0 +1,84 @@
+import { editly } from "../../ai-sdk/providers/editly";
+import type {
+  Clip,
+  ImageOverlayLayer,
+  Layer,
+  Position,
+  TitleLayer,
+} from "../../ai-sdk/providers/editly/types";
+import type { PackshotProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+
+function resolvePosition(pos: Position | undefined): Position {
+  return pos ?? "center";
+}
+
+export async function renderPackshot(
+  element: VargElement<"packshot">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as PackshotProps;
+  const duration = props.duration ?? 3;
+
+  const layers: Layer[] = [];
+
+  if (props.background) {
+    if (typeof props.background === "string") {
+      layers.push({
+        type: "fill-color" as const,
+        color: props.background,
+      });
+    } else {
+      const bgPath = await renderImage(props.background, ctx);
+      layers.push({
+        type: "image" as const,
+        path: bgPath,
+        resizeMode: "cover" as const,
+      });
+    }
+  } else {
+    layers.push({
+      type: "fill-color" as const,
+      color: "#000000",
+    });
+  }
+
+  if (props.logo) {
+    const logoLayer: ImageOverlayLayer = {
+      type: "image-overlay",
+      path: props.logo,
+      position: resolvePosition(props.logoPosition),
+      width: props.logoSize ?? "30%",
+    };
+    layers.push(logoLayer);
+  }
+
+  if (props.cta) {
+    const ctaLayer: TitleLayer = {
+      type: "title",
+      text: props.cta,
+      textColor: props.ctaColor ?? "white",
+      position: resolvePosition(props.ctaPosition ?? "bottom"),
+    };
+    layers.push(ctaLayer);
+  }
+
+  const clip: Clip = {
+    layers,
+    duration,
+  };
+
+  const outPath = `/tmp/varg-packshot-${Date.now()}.mp4`;
+
+  await editly({
+    outPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips: [clip],
+  });
+
+  ctx.tempFiles.push(outPath);
+  return outPath;
+}