varg.ai-sdk 0.1.0 → 0.4.0-alpha.1

package/src/react/renderers/captions.ts

@@ -0,0 +1,297 @@
+import { writeFileSync } from "node:fs";
+import { groq } from "@ai-sdk/groq";
+import { experimental_transcribe as transcribe } from "ai";
+import { z } from "zod";
+import type { CaptionsProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { addTask, completeTask, startTask } from "./progress";
+import { renderSpeech } from "./speech";
+
+const groqWordSchema = z.object({
+  word: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+
+const groqResponseSchema = z.object({
+  words: z.array(groqWordSchema).optional(),
+});
+
+type GroqWord = z.infer<typeof groqWordSchema>;
+
+// Helper function to convert words to SRT format
+function formatTime(seconds: number): string {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+  const millis = Math.floor((seconds % 1) * 1000);
+
+  return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")},${String(millis).padStart(3, "0")}`;
+}
+
+export function convertToSRT(words: GroqWord[]): string {
+  let srt = "";
+  let index = 1;
+
+  for (const word of words) {
+    const startTime = formatTime(word.start);
+    const endTime = formatTime(word.end);
+
+    srt += `${index}\n`;
+    srt += `${startTime} --> ${endTime}\n`;
+    srt += `${word.word.trim()}\n\n`;
+    index++;
+  }
+
+  return srt;
+}
+
+interface SrtEntry {
+  index: number;
+  start: number;
+  end: number;
+  text: string;
+}
+
+interface SubtitleStyle {
+  fontName: string;
+  fontSize: number;
+  primaryColor: string;
+  outlineColor: string;
+  backColor: string;
+  bold: boolean;
+  outline: number;
+  shadow: number;
+  marginV: number;
+  alignment: number;
+}
+
+const STYLE_PRESETS: Record<string, SubtitleStyle> = {
+  tiktok: {
+    fontName: "Montserrat",
+    fontSize: 32,
+    primaryColor: "&HFFFFFF",
+    outlineColor: "&H000000",
+    backColor: "&H80000000",
+    bold: true,
+    outline: 3,
+    shadow: 0,
+    marginV: 50,
+    alignment: 2,
+  },
+  karaoke: {
+    fontName: "Arial",
+    fontSize: 28,
+    primaryColor: "&H00FFFF",
+    outlineColor: "&H000000",
+    backColor: "&H00000000",
+    bold: true,
+    outline: 2,
+    shadow: 1,
+    marginV: 40,
+    alignment: 2,
+  },
+  bounce: {
+    fontName: "Impact",
+    fontSize: 36,
+    primaryColor: "&HFFFFFF",
+    outlineColor: "&H000000",
+    backColor: "&H00000000",
+    bold: false,
+    outline: 4,
+    shadow: 2,
+    marginV: 60,
+    alignment: 2,
+  },
+  typewriter: {
+    fontName: "Courier New",
+    fontSize: 24,
+    primaryColor: "&H00FF00",
+    outlineColor: "&H000000",
+    backColor: "&H80000000",
+    bold: false,
+    outline: 1,
+    shadow: 0,
+    marginV: 30,
+    alignment: 2,
+  },
+};
+
+function parseSrt(content: string): SrtEntry[] {
+  const entries: SrtEntry[] = [];
+  const blocks = content.trim().split(/\n\n+/);
+
+  for (const block of blocks) {
+    const lines = block.split("\n");
+    if (lines.length < 3) continue;
+
+    const index = Number.parseInt(lines[0] || "0", 10);
+    const timeLine = lines[1] || "";
+    const timeMatch = timeLine.match(
+      /(\d{2}):(\d{2}):(\d{2})[,.](\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2})[,.](\d{3})/,
+    );
+
+    if (!timeMatch) continue;
+
+    const [, h1, m1, s1, ms1, h2, m2, s2, ms2] = timeMatch;
+    if (!h1 || !m1 || !s1 || !ms1 || !h2 || !m2 || !s2 || !ms2) continue;
+
+    const start =
+      Number.parseInt(h1, 10) * 3600 +
+      Number.parseInt(m1, 10) * 60 +
+      Number.parseInt(s1, 10) +
+      Number.parseInt(ms1, 10) / 1000;
+
+    const end =
+      Number.parseInt(h2, 10) * 3600 +
+      Number.parseInt(m2, 10) * 60 +
+      Number.parseInt(s2, 10) +
+      Number.parseInt(ms2, 10) / 1000;
+
+    const text = lines.slice(2).join("\n");
+    entries.push({ index, start, end, text });
+  }
+
+  return entries;
+}
+
+function formatAssTime(seconds: number): string {
+  const h = Math.floor(seconds / 3600);
+  const m = Math.floor((seconds % 3600) / 60);
+  const s = Math.floor(seconds % 60);
+  const cs = Math.floor((seconds % 1) * 100);
+
+  return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`;
+}
+
+function convertSrtToAss(srtContent: string, style: SubtitleStyle): string {
+  const assHeader = `[Script Info]
+Title: Generated Subtitles
+ScriptType: v4.00+
+WrapStyle: 0
+ScaledBorderAndShadow: yes
+YCbCr Matrix: TV.601
+
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+Style: Default,${style.fontName},${style.fontSize},${style.primaryColor},&H000000FF,${style.outlineColor},${style.backColor},${style.bold ? -1 : 0},0,0,0,100,100,0,0,1,${style.outline},${style.shadow},${style.alignment},10,10,${style.marginV},1
+
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+`;
+
+  const entries = parseSrt(srtContent);
+  const assDialogues = entries
+    .map((entry) => {
+      const start = formatAssTime(entry.start);
+      const end = formatAssTime(entry.end);
+      const text = entry.text.replace(/\n/g, "\\N");
+      return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`;
+    })
+    .join("\n");
+
+  return assHeader + assDialogues;
+}
+
+function colorToAss(color: string): string {
+  if (color.startsWith("&H")) return color;
+
+  const hex = color.replace("#", "");
+  if (hex.length === 6) {
+    const r = hex.substring(0, 2);
+    const g = hex.substring(2, 4);
+    const b = hex.substring(4, 6);
+    return `&H${b}${g}${r}`.toUpperCase();
+  }
+  return "&HFFFFFF";
+}
+
+export interface CaptionsResult {
+  assPath: string;
+  srtPath?: string;
+  audioPath?: string;
+}
+
+export async function renderCaptions(
+  element: VargElement<"captions">,
+  ctx: RenderContext,
+): Promise<CaptionsResult> {
+  const props = element.props as CaptionsProps;
+
+  let srtContent: string;
+  let srtPath: string | undefined;
+  let audioPath: string | undefined;
+
+  if (props.srt) {
+    srtContent = await Bun.file(props.srt).text();
+    srtPath = props.srt;
+  } else if (props.src) {
+    if (typeof props.src === "string") {
+      srtContent = await Bun.file(props.src).text();
+      srtPath = props.src;
+    } else if (props.src.type === "speech") {
+      const speechResult = await renderSpeech(props.src, ctx);
+      audioPath = speechResult.path;
+
+      const transcribeTaskId = ctx.progress
+        ? addTask(ctx.progress, "transcribe", "groq-whisper")
+        : null;
+      if (transcribeTaskId && ctx.progress)
+        startTask(ctx.progress, transcribeTaskId);
+
+      const audioData = await Bun.file(speechResult.path).arrayBuffer();
+
+      const result = await transcribe({
+        model: groq.transcription("whisper-large-v3"),
+        audio: new Uint8Array(audioData),
+        providerOptions: {
+          groq: {
+            responseFormat: "verbose_json",
+            timestampGranularities: ["word"],
+          },
+        },
+      });
+
+      if (transcribeTaskId && ctx.progress)
+        completeTask(ctx.progress, transcribeTaskId);
+
+      const rawBody = (result.responses[0] as { body?: unknown })?.body;
+      const parsed = groqResponseSchema.safeParse(rawBody);
+      const words = parsed.success ? parsed.data.words : undefined;
+
+      if (!words || words.length === 0) {
+        srtContent = `1\n00:00:00,000 --> 00:00:05,000\n${result.text}\n`;
+      } else {
+        srtContent = convertToSRT(words);
+      }
+
+      srtPath = `/tmp/varg-captions-${Date.now()}.srt`;
+      writeFileSync(srtPath, srtContent);
+      ctx.tempFiles.push(srtPath);
+    } else {
+      throw new Error(
+        "Captions src must be a path to SRT file or Speech element",
+      );
+    }
+  } else {
+    throw new Error("Captions element requires either 'srt' or 'src' prop");
+  }
+
+  const styleName = props.style ?? "tiktok";
+  const baseStyle = STYLE_PRESETS[styleName] ?? STYLE_PRESETS.tiktok!;
+
+  const style: SubtitleStyle = {
+    ...baseStyle,
+    fontSize: props.fontSize ?? baseStyle.fontSize,
+    primaryColor: props.color
+      ? colorToAss(props.color)
+      : baseStyle.primaryColor,
+  };
+
+  const assContent = convertSrtToAss(srtContent, style);
+  const assPath = `/tmp/varg-captions-${Date.now()}.ass`;
+  writeFileSync(assPath, assContent);
+  ctx.tempFiles.push(assPath);
+
+  return { assPath, srtPath, audioPath };
+}

package/src/react/renderers/clip.ts

@@ -0,0 +1,248 @@
+import type {
+  AudioLayer,
+  Clip,
+  FillColorLayer,
+  ImageLayer,
+  ImageOverlayLayer,
+  Layer,
+  VideoLayer,
+} from "../../ai-sdk/providers/editly/types";
+import type {
+  AnimateProps,
+  ClipProps,
+  ImageProps,
+  SpeechProps,
+  VargElement,
+  VargNode,
+  VideoProps,
+} from "../types";
+import { renderAnimate } from "./animate";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderPackshot } from "./packshot";
+import { renderSlider } from "./slider";
+import { renderSpeech } from "./speech";
+import { renderSplit } from "./split";
+import { renderSubtitle } from "./subtitle";
+import { renderSwipe } from "./swipe";
+import { renderTitle } from "./title";
+import { renderVideo } from "./video";
+
+type PendingLayer =
+  | { type: "sync"; layer: Layer }
+  | { type: "async"; promise: Promise<Layer> };
+
+async function renderClipLayers(
+  children: VargNode[],
+  ctx: RenderContext,
+): Promise<Layer[]> {
+  const pending: PendingLayer[] = [];
+
+  for (const child of children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+
+    const element = child as VargElement;
+
+    switch (element.type) {
+      case "image": {
+        const props = element.props as ImageProps;
+        const hasPosition =
+          props.left !== undefined ||
+          props.top !== undefined ||
+          props.width !== undefined ||
+          props.height !== undefined;
+
+        pending.push({
+          type: "async",
+          promise: renderImage(element as VargElement<"image">, ctx).then(
+            (path) =>
+              hasPosition
+                ? ({
+                    type: "image-overlay",
+                    path,
+                    zoomDirection: props.zoom,
+                    width: props.width,
+                    height: props.height,
+                    position: { x: props.left ?? 0, y: props.top ?? 0 },
+                  } as ImageOverlayLayer)
+                : ({
+                    type: "image",
+                    path,
+                    resizeMode: props.resize,
+                    zoomDirection: props.zoom,
+                  } as ImageLayer),
+          ),
+        });
+        break;
+      }
+
+      case "video": {
+        const props = element.props as VideoProps;
+        pending.push({
+          type: "async",
+          promise: renderVideo(element as VargElement<"video">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+                resizeMode: props.resize,
+                cutFrom: props.cutFrom,
+                cutTo: props.cutTo,
+                mixVolume: props.keepAudio ? (props.volume ?? 1) : 0,
+                left: props.left,
+                top: props.top,
+                width: props.width,
+                height: props.height,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "animate": {
+        const props = element.props as AnimateProps;
+        pending.push({
+          type: "async",
+          promise: renderAnimate(element as VargElement<"animate">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+                left: props.left,
+                top: props.top,
+                width: props.width,
+                height: props.height,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "title": {
+        pending.push({
+          type: "sync",
+          layer: renderTitle(element as VargElement<"title">),
+        });
+        break;
+      }
+
+      case "subtitle": {
+        pending.push({
+          type: "sync",
+          layer: renderSubtitle(element as VargElement<"subtitle">),
+        });
+        break;
+      }
+
+      case "speech": {
+        const props = element.props as SpeechProps;
+        pending.push({
+          type: "async",
+          promise: renderSpeech(element as VargElement<"speech">, ctx).then(
+            (result) =>
+              ({
+                type: "audio",
+                path: result.path,
+                mixVolume: props.volume ?? 1,
+              }) as AudioLayer,
+          ),
+        });
+        break;
+      }
+
+      case "split": {
+        pending.push({
+          type: "async",
+          promise: renderSplit(element as VargElement<"split">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "slider": {
+        pending.push({
+          type: "async",
+          promise: renderSlider(element as VargElement<"slider">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "swipe": {
+        pending.push({
+          type: "async",
+          promise: renderSwipe(element as VargElement<"swipe">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+
+      case "packshot": {
+        pending.push({
+          type: "async",
+          promise: renderPackshot(element as VargElement<"packshot">, ctx).then(
+            (path) =>
+              ({
+                type: "video",
+                path,
+              }) as VideoLayer,
+          ),
+        });
+        break;
+      }
+    }
+  }
+
+  const layers = await Promise.all(
+    pending.map((p) => (p.type === "sync" ? p.layer : p.promise)),
+  );
+
+  return layers;
+}
+
+export async function renderClip(
+  element: VargElement<"clip">,
+  ctx: RenderContext,
+): Promise<Clip> {
+  const props = element.props as ClipProps;
+  const layers = await renderClipLayers(element.children, ctx);
+
+  const isOverlayVideo = (l: Layer) =>
+    l.type === "video" &&
+    ((l as VideoLayer).left !== undefined ||
+      (l as VideoLayer).top !== undefined ||
+      (l as VideoLayer).width !== undefined ||
+      (l as VideoLayer).height !== undefined);
+
+  const hasBaseLayer = layers.some(
+    (l) =>
+      l.type === "image" ||
+      l.type === "fill-color" ||
+      (l.type === "video" && !isOverlayVideo(l)),
+  );
+
+  if (!hasBaseLayer && layers.length > 0) {
+    layers.unshift({ type: "fill-color", color: "#000000" } as FillColorLayer);
+  }
+
+  return {
+    layers,
+    duration: typeof props.duration === "number" ? props.duration : undefined,
+    transition: props.transition ?? null,
+  };
+}

package/src/react/renderers/context.ts

@@ -0,0 +1,17 @@
+import type { generateImage } from "ai";
+import type { fileCache } from "../../ai-sdk/file-cache";
+import type { generateVideo } from "../../ai-sdk/generate-video";
+import type { ProgressTracker } from "./progress";
+
+export interface RenderContext {
+  width: number;
+  height: number;
+  fps: number;
+  cache?: ReturnType<typeof fileCache>;
+  generateImage: typeof generateImage;
+  generateVideo: typeof generateVideo;
+  tempFiles: string[];
+  progress?: ProgressTracker;
+  /** In-memory deduplication for concurrent renders of the same element */
+  pending: Map<string, Promise<string>>;
+}

package/src/react/renderers/image.ts

@@ -0,0 +1,109 @@
+import type { generateImage } from "ai";
+import { File } from "../../ai-sdk/file";
+import type {
+  ImageInput,
+  ImagePrompt,
+  ImageProps,
+  VargElement,
+} from "../types";
+import type { RenderContext } from "./context";
+import { addTask, completeTask, startTask } from "./progress";
+import { computeCacheKey, toFileUrl } from "./utils";
+
+async function resolveImageInput(
+  input: ImageInput,
+  ctx: RenderContext,
+): Promise<Uint8Array> {
+  if (input instanceof Uint8Array) {
+    return input;
+  }
+  if (typeof input === "string") {
+    const response = await fetch(toFileUrl(input));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  const path = await renderImage(input, ctx);
+  const response = await fetch(toFileUrl(path));
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function resolvePrompt(
+  prompt: ImagePrompt,
+  ctx: RenderContext,
+): Promise<string | { text?: string; images: Uint8Array[] }> {
+  if (typeof prompt === "string") {
+    return prompt;
+  }
+  const resolvedImages = await Promise.all(
+    prompt.images.map((img) => resolveImageInput(img, ctx)),
+  );
+  return { text: prompt.text, images: resolvedImages };
+}
+
+export async function renderImage(
+  element: VargElement<"image">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as ImageProps;
+
+  if (props.src) {
+    return props.src;
+  }
+
+  const prompt = props.prompt;
+  if (!prompt) {
+    throw new Error("Image element requires either 'prompt' or 'src'");
+  }
+
+  const model = props.model;
+  if (!model) {
+    throw new Error("Image element requires 'model' prop when using prompt");
+  }
+
+  // Compute cache key for deduplication
+  const cacheKey = computeCacheKey(element);
+  const cacheKeyStr = JSON.stringify(cacheKey);
+
+  // Check if this element is already being rendered (deduplication)
+  const pendingRender = ctx.pending.get(cacheKeyStr);
+  if (pendingRender) {
+    return pendingRender;
+  }
+
+  // Create the render promise and store it for deduplication
+  const renderPromise = (async () => {
+    const resolvedPrompt = await resolvePrompt(prompt, ctx);
+
+    const modelId = typeof model === "string" ? model : model.modelId;
+    const taskId = ctx.progress
+      ? addTask(ctx.progress, "image", modelId)
+      : null;
+    if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+
+    const { images } = await ctx.generateImage({
+      model,
+      prompt: resolvedPrompt,
+      aspectRatio: props.aspectRatio,
+      n: 1,
+      cacheKey,
+    } as Parameters<typeof generateImage>[0]);
+
+    if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+
+    const firstImage = images[0];
+    if (!firstImage?.uint8Array) {
+      throw new Error("Image generation returned no image data");
+    }
+    const imageData = firstImage.uint8Array;
+    const tempPath = await File.toTemp({
+      uint8Array: imageData,
+      mimeType: "image/png",
+    });
+    ctx.tempFiles.push(tempPath);
+
+    return tempPath;
+  })();
+
+  ctx.pending.set(cacheKeyStr, renderPromise);
+
+  return renderPromise;
+}

package/src/react/renderers/index.ts

@@ -0,0 +1,22 @@
+export { renderAnimate } from "./animate";
+export { renderCaptions } from "./captions";
+export { renderClip } from "./clip";
+export type { RenderContext } from "./context";
+export { renderImage } from "./image";
+export { renderPackshot } from "./packshot";
+export {
+  createProgressTracker,
+  type GenerationType,
+  type ProgressTask,
+  type ProgressTracker,
+  TIME_ESTIMATES,
+} from "./progress";
+export { renderRoot } from "./render";
+export { renderSlider } from "./slider";
+export type { SpeechResult } from "./speech";
+export { renderSpeech } from "./speech";
+export { renderSplit } from "./split";
+export { renderSwipe } from "./swipe";
+export { renderTitle } from "./title";
+export { computeCacheKey, getTextContent } from "./utils";
+export { renderVideo } from "./video";