vargai 0.3.2 → 0.4.0-alpha2

package/src/react/react.test.ts

@@ -0,0 +1,309 @@
+import { describe, expect, test } from "bun:test";
+import { existsSync, unlinkSync } from "node:fs";
+import { fal } from "../ai-sdk/providers/fal";
+import {
+  Animate,
+  Captions,
+  Clip,
+  Image,
+  Packshot,
+  Render,
+  render,
+  Slider,
+  Split,
+  Swipe,
+  Title,
+  Video,
+} from "./index";
+
+describe("varg-react elements", () => {
+  test("Render creates correct element structure", () => {
+    const element = Render({
+      width: 1280,
+      height: 720,
+      fps: 30,
+      children: [],
+    });
+
+    expect(element.type).toBe("render");
+    expect(element.props.width).toBe(1280);
+    expect(element.props.height).toBe(720);
+    expect(element.props.fps).toBe(30);
+  });
+
+  test("Video creates correct element structure", () => {
+    const element = Video({
+      prompt: "ocean waves",
+      model: fal.videoModel("wan-2.5"),
+    });
+
+    expect(element.type).toBe("video");
+    expect(element.props.prompt).toBe("ocean waves");
+  });
+
+  test("Clip creates correct element structure", () => {
+    const element = Clip({
+      duration: 5,
+      transition: { name: "fade", duration: 0.5 },
+      children: [],
+    });
+
+    expect(element.type).toBe("clip");
+    expect(element.props.duration).toBe(5);
+    expect(element.props.transition).toEqual({ name: "fade", duration: 0.5 });
+  });
+
+  test("Image creates correct element structure", () => {
+    const element = Image({
+      prompt: "fat tiger on couch",
+      model: fal.imageModel("flux-schnell"),
+      aspectRatio: "16:9",
+      zoom: "in",
+    });
+
+    expect(element.type).toBe("image");
+    expect(element.props.prompt).toBe("fat tiger on couch");
+    expect(element.props.aspectRatio).toBe("16:9");
+    expect(element.props.zoom).toBe("in");
+  });
+
+  test("Title creates correct element with text children", () => {
+    const element = Title({
+      position: "bottom",
+      color: "#ffffff",
+      children: "I'M IN DANGER",
+    });
+
+    expect(element.type).toBe("title");
+    expect(element.props.position).toBe("bottom");
+    expect(element.props.color).toBe("#ffffff");
+    expect(element.children).toContain("I'M IN DANGER");
+  });
+
+  test("Animate creates correct element with nested image", () => {
+    const image = Image({ prompt: "luigi in wheelchair" });
+    const element = Animate({
+      image,
+      model: fal.videoModel("wan-2.5"),
+      motion: "wheels spinning fast",
+      duration: 5,
+    });
+
+    expect(element.type).toBe("animate");
+    expect(element.props.image).toBe(image);
+    expect(element.props.motion).toBe("wheels spinning fast");
+    expect(element.props.duration).toBe(5);
+  });
+
+  test("nested composition builds correct tree", () => {
+    const root = Render({
+      width: 1080,
+      height: 1920,
+      children: [
+        Clip({
+          duration: 5,
+          children: [
+            Image({
+              prompt: "ralph wiggum",
+              model: fal.imageModel("flux-schnell"),
+            }),
+            Title({ children: "HELLO" }),
+          ],
+        }),
+        Clip({
+          duration: 3,
+          transition: { name: "fade", duration: 0.3 },
+          children: [
+            Image({
+              prompt: "fat tiger",
+              model: fal.imageModel("flux-schnell"),
+            }),
+          ],
+        }),
+      ],
+    });
+
+    expect(root.type).toBe("render");
+    expect(root.children.length).toBe(2);
+
+    const clip1 = root.children[0] as ReturnType<typeof Clip>;
+    expect(clip1.type).toBe("clip");
+    expect(clip1.children.length).toBe(2);
+
+    const clip2 = root.children[1] as ReturnType<typeof Clip>;
+    expect(clip2.type).toBe("clip");
+    expect(clip2.props.transition).toEqual({ name: "fade", duration: 0.3 });
+  });
+});
+
+describe("varg-react render", () => {
+  test("render throws on non-render root", async () => {
+    const clip = Clip({ duration: 5, children: [] });
+
+    expect(render(clip)).rejects.toThrow("Root element must be <Render>");
+  });
+
+  test("render requires model prop for image with prompt", async () => {
+    const root = Render({
+      width: 720,
+      height: 720,
+      children: [
+        Clip({
+          duration: 3,
+          children: [Image({ prompt: "test image without model" })],
+        }),
+      ],
+    });
+
+    expect(render(root)).rejects.toThrow("model");
+  });
+});
+
+describe("layout renderers", () => {
+  const testImage1 = "media/cyberpunk-street.png";
+  const testImage2 = "media/fal-coffee-shop.png";
+  const outPath = "output/layout-test.mp4";
+
+  test("Split renders side-by-side images", async () => {
+    const root = Render({
+      width: 1280,
+      height: 720,
+      children: [
+        Clip({
+          duration: 2,
+          children: [
+            Split({
+              direction: "horizontal",
+              children: [
+                Image({ src: testImage1 }),
+                Image({ src: testImage2 }),
+              ],
+            }),
+          ],
+        }),
+      ],
+    });
+
+    const result = await render(root, { output: outPath, quiet: true });
+    expect(result).toBeInstanceOf(Uint8Array);
+    expect(result.length).toBeGreaterThan(0);
+    expect(existsSync(outPath)).toBe(true);
+    unlinkSync(outPath);
+  });
+
+  test(
+    "Slider renders with slide transitions",
+    async () => {
+      const root = Render({
+        width: 1280,
+        height: 720,
+        children: [
+          Clip({
+            duration: 4,
+            children: [
+              Slider({
+                direction: "horizontal",
+                children: [
+                  Image({ src: testImage1 }),
+                  Image({ src: testImage2 }),
+                ],
+              }),
+            ],
+          }),
+        ],
+      });
+
+      const result = await render(root, { output: outPath, quiet: true });
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(existsSync(outPath)).toBe(true);
+      unlinkSync(outPath);
+    },
+    { timeout: 30000 },
+  );
+
+  test(
+    "Swipe renders with swipe animation",
+    async () => {
+      const root = Render({
+        width: 1280,
+        height: 720,
+        children: [
+          Clip({
+            duration: 4,
+            children: [
+              Swipe({
+                direction: "left",
+                interval: 2,
+                children: [
+                  Image({ src: testImage1 }),
+                  Image({ src: testImage2 }),
+                ],
+              }),
+            ],
+          }),
+        ],
+      });
+
+      const result = await render(root, { output: outPath, quiet: true });
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(existsSync(outPath)).toBe(true);
+      unlinkSync(outPath);
+    },
+    { timeout: 30000 },
+  );
+
+  test("Packshot renders end card with logo and cta", async () => {
+    const root = Render({
+      width: 1280,
+      height: 720,
+      children: [
+        Clip({
+          duration: 3,
+          children: [
+            Packshot({
+              background: "#1a1a2e",
+              logo: testImage1,
+              logoPosition: "center",
+              logoSize: "40%",
+              cta: "Subscribe Now!",
+              ctaPosition: "bottom",
+              ctaColor: "#FFD700",
+              duration: 3,
+            }),
+          ],
+        }),
+      ],
+    });
+
+    const result = await render(root, { output: outPath, quiet: true });
+    expect(result).toBeInstanceOf(Uint8Array);
+    expect(existsSync(outPath)).toBe(true);
+    unlinkSync(outPath);
+  });
+
+  test(
+    "Captions burns subtitles from SRT file",
+    async () => {
+      const root = Render({
+        width: 1280,
+        height: 720,
+        children: [
+          Clip({
+            duration: 3,
+            children: [Image({ src: testImage1 })],
+          }),
+          Captions({
+            srt: "media/dora-test.srt",
+            style: "tiktok",
+          }),
+        ],
+      });
+
+      const result = await render(root, { output: outPath, quiet: true });
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(existsSync(outPath)).toBe(true);
+      unlinkSync(outPath);
+    },
+    { timeout: 30000 },
+  );
+});

package/src/react/render.ts

@@ -0,0 +1,21 @@
+import { renderRoot } from "./renderers";
+import type { RenderOptions, VargElement } from "./types";
+
+export async function render(
+  element: VargElement,
+  options: RenderOptions = {},
+): Promise<Uint8Array> {
+  if (element.type !== "render") {
+    throw new Error("Root element must be <Render>");
+  }
+
+  return renderRoot(element as VargElement<"render">, options);
+}
+
+export const renderStream = {
+  async *stream(element: VargElement, options: RenderOptions = {}) {
+    yield { type: "start", progress: 0 };
+    const result = await render(element, options);
+    yield { type: "complete", progress: 100, result };
+  },
+};

package/src/react/renderers/animate.ts

@@ -0,0 +1,59 @@
+import { File } from "../../ai-sdk/file";
+import type { generateVideo } from "../../ai-sdk/generate-video";
+import type { AnimateProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { addTask, completeTask, startTask } from "./progress";
+import { computeCacheKey, resolvePath } from "./utils";
+
+export async function renderAnimate(
+  element: VargElement<"animate">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as AnimateProps;
+
+  let imagePath: string;
+  if (props.src) {
+    imagePath = props.src;
+  } else if (props.image) {
+    if (props.image.type !== "image") {
+      throw new Error(
+        `Animate 'image' prop must be an <Image /> element, got <${props.image.type} />`,
+      );
+    }
+    imagePath = await renderImage(props.image as VargElement<"image">, ctx);
+  } else {
+    throw new Error("Animate element requires either 'src' or 'image' prop");
+  }
+
+  const model = props.model;
+  if (!model) {
+    throw new Error("Animate element requires 'model' prop");
+  }
+
+  const imageData = await Bun.file(resolvePath(imagePath)).arrayBuffer();
+  const cacheKey = computeCacheKey(element);
+
+  const modelId = typeof model === "string" ? model : model.modelId;
+  const taskId = ctx.progress
+    ? addTask(ctx.progress, "animate", modelId)
+    : null;
+  if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+
+  const { video } = await ctx.generateVideo({
+    model,
+    prompt: {
+      text: props.motion ?? "",
+      images: [new Uint8Array(imageData)],
+    },
+    duration: props.duration ?? 5,
+    cacheKey,
+  } as Parameters<typeof generateVideo>[0]);
+
+  if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+
+  const tempPath = await File.toTemp(video);
+  ctx.tempFiles.push(tempPath);
+
+  return tempPath;
+}

package/src/react/renderers/captions.ts

@@ -0,0 +1,297 @@
+import { writeFileSync } from "node:fs";
+import { groq } from "@ai-sdk/groq";
+import { experimental_transcribe as transcribe } from "ai";
+import { z } from "zod";
+import type { CaptionsProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { addTask, completeTask, startTask } from "./progress";
+import { renderSpeech } from "./speech";
+
+const groqWordSchema = z.object({
+  word: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+
+const groqResponseSchema = z.object({
+  words: z.array(groqWordSchema).optional(),
+});
+
+type GroqWord = z.infer<typeof groqWordSchema>;
+
+// Helper function to convert words to SRT format
+function formatTime(seconds: number): string {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+  const millis = Math.floor((seconds % 1) * 1000);
+
+  return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")},${String(millis).padStart(3, "0")}`;
+}
+
+export function convertToSRT(words: GroqWord[]): string {
+  let srt = "";
+  let index = 1;
+
+  for (const word of words) {
+    const startTime = formatTime(word.start);
+    const endTime = formatTime(word.end);
+
+    srt += `${index}\n`;
+    srt += `${startTime} --> ${endTime}\n`;
+    srt += `${word.word.trim()}\n\n`;
+    index++;
+  }
+
+  return srt;
+}
+
+interface SrtEntry {
+  index: number;
+  start: number;
+  end: number;
+  text: string;
+}
+
+interface SubtitleStyle {
+  fontName: string;
+  fontSize: number;
+  primaryColor: string;
+  outlineColor: string;
+  backColor: string;
+  bold: boolean;
+  outline: number;
+  shadow: number;
+  marginV: number;
+  alignment: number;
+}
+
+const STYLE_PRESETS: Record<string, SubtitleStyle> = {
+  tiktok: {
+    fontName: "Montserrat",
+    fontSize: 32,
+    primaryColor: "&HFFFFFF",
+    outlineColor: "&H000000",
+    backColor: "&H80000000",
+    bold: true,
+    outline: 3,
+    shadow: 0,
+    marginV: 50,
+    alignment: 2,
+  },
+  karaoke: {
+    fontName: "Arial",
+    fontSize: 28,
+    primaryColor: "&H00FFFF",
+    outlineColor: "&H000000",
+    backColor: "&H00000000",
+    bold: true,
+    outline: 2,
+    shadow: 1,
+    marginV: 40,
+    alignment: 2,
+  },
+  bounce: {
+    fontName: "Impact",
+    fontSize: 36,
+    primaryColor: "&HFFFFFF",
+    outlineColor: "&H000000",
+    backColor: "&H00000000",
+    bold: false,
+    outline: 4,
+    shadow: 2,
+    marginV: 60,
+    alignment: 2,
+  },
+  typewriter: {
+    fontName: "Courier New",
+    fontSize: 24,
+    primaryColor: "&H00FF00",
+    outlineColor: "&H000000",
+    backColor: "&H80000000",
+    bold: false,
+    outline: 1,
+    shadow: 0,
+    marginV: 30,
+    alignment: 2,
+  },
+};
+
+function parseSrt(content: string): SrtEntry[] {
+  const entries: SrtEntry[] = [];
+  const blocks = content.trim().split(/\n\n+/);
+
+  for (const block of blocks) {
+    const lines = block.split("\n");
+    if (lines.length < 3) continue;
+
+    const index = Number.parseInt(lines[0] || "0", 10);
+    const timeLine = lines[1] || "";
+    const timeMatch = timeLine.match(
+      /(\d{2}):(\d{2}):(\d{2})[,.](\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2})[,.](\d{3})/,
+    );
+
+    if (!timeMatch) continue;
+
+    const [, h1, m1, s1, ms1, h2, m2, s2, ms2] = timeMatch;
+    if (!h1 || !m1 || !s1 || !ms1 || !h2 || !m2 || !s2 || !ms2) continue;
+
+    const start =
+      Number.parseInt(h1, 10) * 3600 +
+      Number.parseInt(m1, 10) * 60 +
+      Number.parseInt(s1, 10) +
+      Number.parseInt(ms1, 10) / 1000;
+
+    const end =
+      Number.parseInt(h2, 10) * 3600 +
+      Number.parseInt(m2, 10) * 60 +
+      Number.parseInt(s2, 10) +
+      Number.parseInt(ms2, 10) / 1000;
+
+    const text = lines.slice(2).join("\n");
+    entries.push({ index, start, end, text });
+  }
+
+  return entries;
+}
+
+function formatAssTime(seconds: number): string {
+  const h = Math.floor(seconds / 3600);
+  const m = Math.floor((seconds % 3600) / 60);
+  const s = Math.floor(seconds % 60);
+  const cs = Math.floor((seconds % 1) * 100);
+
+  return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`;
+}
+
+function convertSrtToAss(srtContent: string, style: SubtitleStyle): string {
+  const assHeader = `[Script Info]
+Title: Generated Subtitles
+ScriptType: v4.00+
+WrapStyle: 0
+ScaledBorderAndShadow: yes
+YCbCr Matrix: TV.601
+
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+Style: Default,${style.fontName},${style.fontSize},${style.primaryColor},&H000000FF,${style.outlineColor},${style.backColor},${style.bold ? -1 : 0},0,0,0,100,100,0,0,1,${style.outline},${style.shadow},${style.alignment},10,10,${style.marginV},1
+
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+`;
+
+  const entries = parseSrt(srtContent);
+  const assDialogues = entries
+    .map((entry) => {
+      const start = formatAssTime(entry.start);
+      const end = formatAssTime(entry.end);
+      const text = entry.text.replace(/\n/g, "\\N");
+      return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`;
+    })
+    .join("\n");
+
+  return assHeader + assDialogues;
+}
+
+function colorToAss(color: string): string {
+  if (color.startsWith("&H")) return color;
+
+  const hex = color.replace("#", "");
+  if (hex.length === 6) {
+    const r = hex.substring(0, 2);
+    const g = hex.substring(2, 4);
+    const b = hex.substring(4, 6);
+    return `&H${b}${g}${r}`.toUpperCase();
+  }
+  return "&HFFFFFF";
+}
+
+export interface CaptionsResult {
+  assPath: string;
+  srtPath?: string;
+  audioPath?: string;
+}
+
+export async function renderCaptions(
+  element: VargElement<"captions">,
+  ctx: RenderContext,
+): Promise<CaptionsResult> {
+  const props = element.props as CaptionsProps;
+
+  let srtContent: string;
+  let srtPath: string | undefined;
+  let audioPath: string | undefined;
+
+  if (props.srt) {
+    srtContent = await Bun.file(props.srt).text();
+    srtPath = props.srt;
+  } else if (props.src) {
+    if (typeof props.src === "string") {
+      srtContent = await Bun.file(props.src).text();
+      srtPath = props.src;
+    } else if (props.src.type === "speech") {
+      const speechResult = await renderSpeech(props.src, ctx);
+      audioPath = speechResult.path;
+
+      const transcribeTaskId = ctx.progress
+        ? addTask(ctx.progress, "transcribe", "groq-whisper")
+        : null;
+      if (transcribeTaskId && ctx.progress)
+        startTask(ctx.progress, transcribeTaskId);
+
+      const audioData = await Bun.file(speechResult.path).arrayBuffer();
+
+      const result = await transcribe({
+        model: groq.transcription("whisper-large-v3"),
+        audio: new Uint8Array(audioData),
+        providerOptions: {
+          groq: {
+            responseFormat: "verbose_json",
+            timestampGranularities: ["word"],
+          },
+        },
+      });
+
+      if (transcribeTaskId && ctx.progress)
+        completeTask(ctx.progress, transcribeTaskId);
+
+      const rawBody = (result.responses[0] as { body?: unknown })?.body;
+      const parsed = groqResponseSchema.safeParse(rawBody);
+      const words = parsed.success ? parsed.data.words : undefined;
+
+      if (!words || words.length === 0) {
+        srtContent = `1\n00:00:00,000 --> 00:00:05,000\n${result.text}\n`;
+      } else {
+        srtContent = convertToSRT(words);
+      }
+
+      srtPath = `/tmp/varg-captions-${Date.now()}.srt`;
+      writeFileSync(srtPath, srtContent);
+      ctx.tempFiles.push(srtPath);
+    } else {
+      throw new Error(
+        "Captions src must be a path to SRT file or Speech element",
+      );
+    }
+  } else {
+    throw new Error("Captions element requires either 'srt' or 'src' prop");
+  }
+
+  const styleName = props.style ?? "tiktok";
+  const baseStyle = STYLE_PRESETS[styleName] ?? STYLE_PRESETS.tiktok!;
+
+  const style: SubtitleStyle = {
+    ...baseStyle,
+    fontSize: props.fontSize ?? baseStyle.fontSize,
+    primaryColor: props.color
+      ? colorToAss(props.color)
+      : baseStyle.primaryColor,
+  };
+
+  const assContent = convertSrtToAss(srtContent, style);
+  const assPath = `/tmp/varg-captions-${Date.now()}.ass`;
+  writeFileSync(assPath, assContent);
+  ctx.tempFiles.push(assPath);
+
+  return { assPath, srtPath, audioPath };
+}