varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/src/react/renderers/split.ts

@@ -0,0 +1,91 @@
+import { editly } from "../../ai-sdk/providers/editly";
+import type { Clip, Layer } from "../../ai-sdk/providers/editly/types";
+import type { SplitProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderVideo } from "./video";
+
+export async function renderSplit(
+  element: VargElement<"split">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as SplitProps;
+  const direction = props.direction ?? "horizontal";
+
+  const childPaths: string[] = [];
+
+  for (const child of element.children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+    const childElement = child as VargElement;
+
+    if (childElement.type === "image") {
+      const path = await renderImage(childElement as VargElement<"image">, ctx);
+      childPaths.push(path);
+    } else if (childElement.type === "video") {
+      const path = await renderVideo(childElement as VargElement<"video">, ctx);
+      childPaths.push(path);
+    }
+  }
+
+  if (childPaths.length === 0) {
+    throw new Error("Split element requires at least one image or video child");
+  }
+
+  if (childPaths.length === 1) {
+    return childPaths[0]!;
+  }
+
+  const numChildren = childPaths.length;
+  const cellWidth =
+    direction === "horizontal"
+      ? Math.floor(ctx.width / numChildren)
+      : ctx.width;
+  const cellHeight =
+    direction === "vertical"
+      ? Math.floor(ctx.height / numChildren)
+      : ctx.height;
+
+  const layers: Layer[] = childPaths.map((path, i) => {
+    const isVideo = path.endsWith(".mp4") || path.endsWith(".webm");
+    const left = direction === "horizontal" ? cellWidth * i : 0;
+    const top = direction === "vertical" ? cellHeight * i : 0;
+
+    if (isVideo) {
+      return {
+        type: "video" as const,
+        path,
+        left,
+        top,
+        width: cellWidth,
+        height: cellHeight,
+      };
+    }
+    return {
+      type: "image-overlay" as const,
+      path,
+      position: { x: left, y: top },
+      width: cellWidth,
+      height: cellHeight,
+    };
+  });
+
+  layers.unshift({ type: "fill-color" as const, color: "#000000" });
+
+  const clip: Clip = {
+    layers,
+    duration: 5,
+  };
+
+  const outPath = `/tmp/varg-split-${Date.now()}.mp4`;
+
+  await editly({
+    outPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips: [clip],
+  });
+
+  ctx.tempFiles.push(outPath);
+  return outPath;
+}

package/src/react/renderers/subtitle.ts

@@ -0,0 +1,16 @@
+import type { SubtitleLayer } from "../../ai-sdk/providers/editly/types";
+import type { SubtitleProps, VargElement } from "../types";
+import { getTextContent } from "./utils";
+
+export function renderSubtitle(
+  element: VargElement<"subtitle">,
+): SubtitleLayer {
+  const props = element.props as SubtitleProps;
+  const text = getTextContent(element.children);
+
+  return {
+    type: "subtitle",
+    text,
+    backgroundColor: props.backgroundColor,
+  };
+}

package/src/react/renderers/swipe.ts

@@ -0,0 +1,75 @@
+import { editly } from "../../ai-sdk/providers/editly";
+import type { Clip } from "../../ai-sdk/providers/editly/types";
+import type { SwipeProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderVideo } from "./video";
+
+const SWIPE_TRANSITION_MAP = {
+  left: "slideleft",
+  right: "slideright",
+  up: "slideup",
+  down: "slidedown",
+} as const;
+
+export async function renderSwipe(
+  element: VargElement<"swipe">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as SwipeProps;
+  const direction = props.direction ?? "left";
+  const interval = props.interval ?? 3;
+
+  const childPaths: string[] = [];
+
+  for (const child of element.children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+    const childElement = child as VargElement;
+
+    if (childElement.type === "image") {
+      const path = await renderImage(childElement as VargElement<"image">, ctx);
+      childPaths.push(path);
+    } else if (childElement.type === "video") {
+      const path = await renderVideo(childElement as VargElement<"video">, ctx);
+      childPaths.push(path);
+    }
+  }
+
+  if (childPaths.length === 0) {
+    throw new Error("Swipe element requires at least one image or video child");
+  }
+
+  if (childPaths.length === 1) {
+    return childPaths[0]!;
+  }
+
+  const transitionName = SWIPE_TRANSITION_MAP[direction];
+
+  const clips: Clip[] = childPaths.map((path, i) => {
+    const isVideo = path.endsWith(".mp4") || path.endsWith(".webm");
+    const isLast = i === childPaths.length - 1;
+
+    return {
+      layers: [
+        isVideo
+          ? { type: "video" as const, path, resizeMode: "cover" as const }
+          : { type: "image" as const, path, resizeMode: "cover" as const },
+      ],
+      duration: interval,
+      transition: isLast ? null : { name: transitionName, duration: 0.5 },
+    };
+  });
+
+  const outPath = `/tmp/varg-swipe-${Date.now()}.mp4`;
+
+  await editly({
+    outPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips,
+  });
+
+  ctx.tempFiles.push(outPath);
+  return outPath;
+}

package/src/react/renderers/title.ts

@@ -0,0 +1,17 @@
+import type { TitleLayer } from "../../ai-sdk/providers/editly/types";
+import type { TitleProps, VargElement } from "../types";
+import { getTextContent } from "./utils";
+
+export function renderTitle(element: VargElement<"title">): TitleLayer {
+  const props = element.props as TitleProps;
+  const text = getTextContent(element.children);
+
+  return {
+    type: "title",
+    text,
+    textColor: props.color,
+    position: props.position,
+    start: props.start,
+    stop: props.end,
+  };
+}

package/src/react/renderers/utils.ts

@@ -0,0 +1,124 @@
+import { existsSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+import type { VargElement, VargNode } from "../types";
+
+export function resolvePath(path: string): string {
+  if (path.startsWith("http://") || path.startsWith("https://")) {
+    return path;
+  }
+  return resolve(process.cwd(), path);
+}
+
+export function toFileUrl(path: string): string {
+  if (path.startsWith("http://") || path.startsWith("https://")) {
+    return path;
+  }
+  return `file://${resolvePath(path)}`;
+}
+
+type CacheKeyPart = string | number | boolean | null | undefined;
+
+function isVargElement(v: unknown): v is VargElement {
+  return (
+    typeof v === "object" &&
+    v !== null &&
+    "type" in v &&
+    "props" in v &&
+    "children" in v
+  );
+}
+
+function isLocalFilePath(v: string): boolean {
+  if (v.startsWith("http://") || v.startsWith("https://")) return false;
+  if (v.startsWith("data:")) return false;
+  const resolved = resolvePath(v);
+  return existsSync(resolved);
+}
+
+function getFileFingerprint(path: string): string {
+  const resolved = resolvePath(path);
+  const stat = statSync(resolved);
+  return `${path}:${stat.mtimeMs}:${stat.size}`;
+}
+
+function serializeValue(v: unknown): string {
+  if (typeof v === "string") {
+    if (isLocalFilePath(v)) {
+      return getFileFingerprint(v);
+    }
+    return v;
+  }
+  if (v instanceof Uint8Array) {
+    return Buffer.from(v).toString("base64");
+  }
+  if (Array.isArray(v)) {
+    return `[${v.map(serializeValue).join(",")}]`;
+  }
+  if (v && typeof v === "object") {
+    const entries = Object.entries(v)
+      .map(([key, val]) => `${key}:${serializeValue(val)}`)
+      .join(",");
+    return `{${entries}}`;
+  }
+  return String(v);
+}
+
+export function computeCacheKey(element: VargElement): CacheKeyPart[] {
+  const key: CacheKeyPart[] = [element.type];
+
+  for (const [k, v] of Object.entries(element.props)) {
+    if (k === "children") continue;
+    if (k === "model" && v && typeof v === "object" && "modelId" in v) {
+      const model = v as {
+        provider?: string;
+        modelId: string;
+        settings?: Record<string, unknown>;
+      };
+      key.push("model", model.provider ?? "", model.modelId);
+      // Include model settings in cache key (e.g., higgsfield styleId, quality)
+      if (model.settings) {
+        key.push("modelSettings", serializeValue(model.settings));
+      }
+      continue;
+    }
+    if (typeof v === "string") {
+      if (isLocalFilePath(v)) {
+        key.push(k, getFileFingerprint(v));
+      } else {
+        key.push(k, v);
+      }
+    } else if (typeof v === "number" || typeof v === "boolean") {
+      key.push(k, v);
+    } else if (v === null || v === undefined) {
+      key.push(k, v);
+    } else if (v instanceof Uint8Array) {
+      key.push(k, Buffer.from(v).toString("base64"));
+    } else if (isVargElement(v)) {
+      key.push(k, ...computeCacheKey(v));
+    } else if (Array.isArray(v) || typeof v === "object") {
+      key.push(k, serializeValue(v));
+    }
+  }
+
+  for (const child of element.children) {
+    if (typeof child === "string") {
+      key.push("text", child);
+    } else if (typeof child === "number") {
+      key.push("num", child);
+    } else if (isVargElement(child)) {
+      key.push("child", ...computeCacheKey(child));
+    }
+  }
+
+  return key;
+}
+
+export function getTextContent(node: VargNode): string {
+  if (typeof node === "string") return node;
+  if (typeof node === "number") return String(node);
+  if (Array.isArray(node)) return node.map(getTextContent).join("");
+  if (node && typeof node === "object" && "children" in node) {
+    return node.children.map(getTextContent).join("");
+  }
+  return "";
+}

package/src/react/renderers/video.ts

@@ -0,0 +1,127 @@
+import { File } from "../../ai-sdk/file";
+import type { generateVideo } from "../../ai-sdk/generate-video";
+import type {
+  ImageInput,
+  VargElement,
+  VideoPrompt,
+  VideoProps,
+} from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { addTask, completeTask, startTask } from "./progress";
+import { computeCacheKey, toFileUrl } from "./utils";
+
+async function resolveImageInput(
+  input: ImageInput,
+  ctx: RenderContext,
+): Promise<Uint8Array> {
+  if (input instanceof Uint8Array) {
+    return input;
+  }
+  if (typeof input === "string") {
+    const response = await fetch(toFileUrl(input));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  const path = await renderImage(input, ctx);
+  const response = await fetch(toFileUrl(path));
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function resolveMediaInput(
+  input: Uint8Array | string | undefined,
+): Promise<Uint8Array | undefined> {
+  if (!input) return undefined;
+  if (input instanceof Uint8Array) return input;
+  const response = await fetch(toFileUrl(input));
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function resolvePrompt(
+  prompt: VideoPrompt,
+  ctx: RenderContext,
+): Promise<
+  | string
+  | {
+      text?: string;
+      images?: Uint8Array[];
+      audio?: Uint8Array;
+      video?: Uint8Array;
+    }
+> {
+  if (typeof prompt === "string") {
+    return prompt;
+  }
+  const [resolvedImages, resolvedAudio, resolvedVideo] = await Promise.all([
+    prompt.images
+      ? Promise.all(prompt.images.map((img) => resolveImageInput(img, ctx)))
+      : undefined,
+    resolveMediaInput(prompt.audio),
+    resolveMediaInput(prompt.video),
+  ]);
+  return {
+    text: prompt.text,
+    images: resolvedImages,
+    audio: resolvedAudio,
+    video: resolvedVideo,
+  };
+}
+
+export async function renderVideo(
+  element: VargElement<"video">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as VideoProps;
+
+  if (props.src && !props.prompt) {
+    return props.src;
+  }
+
+  const prompt = props.prompt;
+  if (!prompt) {
+    throw new Error("Video element requires either 'prompt' or 'src'");
+  }
+
+  const model = props.model;
+  if (!model) {
+    throw new Error("Video element requires 'model' prop when using prompt");
+  }
+
+  // Compute cache key for deduplication
+  const cacheKey = computeCacheKey(element);
+  const cacheKeyStr = JSON.stringify(cacheKey);
+
+  // Check if this element is already being rendered (deduplication)
+  const pendingRender = ctx.pending.get(cacheKeyStr);
+  if (pendingRender) {
+    return pendingRender;
+  }
+
+  // Create the render promise and store it for deduplication
+  const renderPromise = (async () => {
+    const resolvedPrompt = await resolvePrompt(prompt, ctx);
+
+    const modelId = typeof model === "string" ? model : model.modelId;
+    const taskId = ctx.progress
+      ? addTask(ctx.progress, "video", modelId)
+      : null;
+    if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+
+    const { video } = await ctx.generateVideo({
+      model,
+      prompt: resolvedPrompt,
+      duration: 5,
+      cacheKey,
+    } as Parameters<typeof generateVideo>[0]);
+
+    if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+
+    const tempPath = await File.toTemp(video);
+    ctx.tempFiles.push(tempPath);
+
+    return tempPath;
+  })();
+
+  ctx.pending.set(cacheKeyStr, renderPromise);
+
+  return renderPromise;
+}

package/src/react/runtime/jsx-dev-runtime.ts

@@ -0,0 +1,43 @@
+import type { VargElement, VargNode } from "../types";
+
+type ElementFactory = (props: Record<string, unknown>) => VargElement;
+
+export function jsx(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  const finalProps = { ...props };
+  if (key !== undefined) {
+    finalProps.key = key;
+  }
+  return type(finalProps);
+}
+
+export function jsxs(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  return jsx(type, props, key);
+}
+
+export function jsxDEV(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  return jsx(type, props, key);
+}
+
+export const Fragment = ({ children }: { children?: VargNode }) => children;
+
+export namespace JSX {
+  export type Element = VargElement;
+  // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+  export type IntrinsicElements = {};
+  export interface ElementChildrenAttribute {
+    // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+    children: {};
+  }
+}

package/src/react/runtime/jsx-runtime.ts

@@ -0,0 +1,35 @@
+import type { VargElement, VargNode } from "../types";
+
+type ElementFactory = (props: Record<string, unknown>) => VargElement;
+
+export function jsx(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  const finalProps = { ...props };
+  if (key !== undefined) {
+    finalProps.key = key;
+  }
+  return type(finalProps);
+}
+
+export function jsxs(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  return jsx(type, props, key);
+}
+
+export const Fragment = ({ children }: { children?: VargNode }) => children;
+
+export namespace JSX {
+  export type Element = VargElement;
+  // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+  export type IntrinsicElements = {};
+  export interface ElementChildrenAttribute {
+    // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+    children: {};
+  }
+}