vargai 0.3.2 → 0.4.0-alpha2

package/src/react/renderers/subtitle.ts

@@ -0,0 +1,16 @@
+import type { SubtitleLayer } from "../../ai-sdk/providers/editly/types";
+import type { SubtitleProps, VargElement } from "../types";
+import { getTextContent } from "./utils";
+
+export function renderSubtitle(
+  element: VargElement<"subtitle">,
+): SubtitleLayer {
+  const props = element.props as SubtitleProps;
+  const text = getTextContent(element.children);
+
+  return {
+    type: "subtitle",
+    text,
+    backgroundColor: props.backgroundColor,
+  };
+}

package/src/react/renderers/swipe.ts

@@ -0,0 +1,75 @@
+import { editly } from "../../ai-sdk/providers/editly";
+import type { Clip } from "../../ai-sdk/providers/editly/types";
+import type { SwipeProps, VargElement } from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { renderVideo } from "./video";
+
+const SWIPE_TRANSITION_MAP = {
+  left: "slideleft",
+  right: "slideright",
+  up: "slideup",
+  down: "slidedown",
+} as const;
+
+export async function renderSwipe(
+  element: VargElement<"swipe">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as SwipeProps;
+  const direction = props.direction ?? "left";
+  const interval = props.interval ?? 3;
+
+  const childPaths: string[] = [];
+
+  for (const child of element.children) {
+    if (!child || typeof child !== "object" || !("type" in child)) continue;
+    const childElement = child as VargElement;
+
+    if (childElement.type === "image") {
+      const path = await renderImage(childElement as VargElement<"image">, ctx);
+      childPaths.push(path);
+    } else if (childElement.type === "video") {
+      const path = await renderVideo(childElement as VargElement<"video">, ctx);
+      childPaths.push(path);
+    }
+  }
+
+  if (childPaths.length === 0) {
+    throw new Error("Swipe element requires at least one image or video child");
+  }
+
+  if (childPaths.length === 1) {
+    return childPaths[0]!;
+  }
+
+  const transitionName = SWIPE_TRANSITION_MAP[direction];
+
+  const clips: Clip[] = childPaths.map((path, i) => {
+    const isVideo = path.endsWith(".mp4") || path.endsWith(".webm");
+    const isLast = i === childPaths.length - 1;
+
+    return {
+      layers: [
+        isVideo
+          ? { type: "video" as const, path, resizeMode: "cover" as const }
+          : { type: "image" as const, path, resizeMode: "cover" as const },
+      ],
+      duration: interval,
+      transition: isLast ? null : { name: transitionName, duration: 0.5 },
+    };
+  });
+
+  const outPath = `/tmp/varg-swipe-${Date.now()}.mp4`;
+
+  await editly({
+    outPath,
+    width: ctx.width,
+    height: ctx.height,
+    fps: ctx.fps,
+    clips,
+  });
+
+  ctx.tempFiles.push(outPath);
+  return outPath;
+}

package/src/react/renderers/title.ts

@@ -0,0 +1,17 @@
+import type { TitleLayer } from "../../ai-sdk/providers/editly/types";
+import type { TitleProps, VargElement } from "../types";
+import { getTextContent } from "./utils";
+
+export function renderTitle(element: VargElement<"title">): TitleLayer {
+  const props = element.props as TitleProps;
+  const text = getTextContent(element.children);
+
+  return {
+    type: "title",
+    text,
+    textColor: props.color,
+    position: props.position,
+    start: props.start,
+    stop: props.end,
+  };
+}

package/src/react/renderers/utils.ts

@@ -0,0 +1,124 @@
+import { existsSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+import type { VargElement, VargNode } from "../types";
+
+export function resolvePath(path: string): string {
+  if (path.startsWith("http://") || path.startsWith("https://")) {
+    return path;
+  }
+  return resolve(process.cwd(), path);
+}
+
+export function toFileUrl(path: string): string {
+  if (path.startsWith("http://") || path.startsWith("https://")) {
+    return path;
+  }
+  return `file://${resolvePath(path)}`;
+}
+
+type CacheKeyPart = string | number | boolean | null | undefined;
+
+function isVargElement(v: unknown): v is VargElement {
+  return (
+    typeof v === "object" &&
+    v !== null &&
+    "type" in v &&
+    "props" in v &&
+    "children" in v
+  );
+}
+
+function isLocalFilePath(v: string): boolean {
+  if (v.startsWith("http://") || v.startsWith("https://")) return false;
+  if (v.startsWith("data:")) return false;
+  const resolved = resolvePath(v);
+  return existsSync(resolved);
+}
+
+function getFileFingerprint(path: string): string {
+  const resolved = resolvePath(path);
+  const stat = statSync(resolved);
+  return `${path}:${stat.mtimeMs}:${stat.size}`;
+}
+
+function serializeValue(v: unknown): string {
+  if (typeof v === "string") {
+    if (isLocalFilePath(v)) {
+      return getFileFingerprint(v);
+    }
+    return v;
+  }
+  if (v instanceof Uint8Array) {
+    return Buffer.from(v).toString("base64");
+  }
+  if (Array.isArray(v)) {
+    return `[${v.map(serializeValue).join(",")}]`;
+  }
+  if (v && typeof v === "object") {
+    const entries = Object.entries(v)
+      .map(([key, val]) => `${key}:${serializeValue(val)}`)
+      .join(",");
+    return `{${entries}}`;
+  }
+  return String(v);
+}
+
+export function computeCacheKey(element: VargElement): CacheKeyPart[] {
+  const key: CacheKeyPart[] = [element.type];
+
+  for (const [k, v] of Object.entries(element.props)) {
+    if (k === "children") continue;
+    if (k === "model" && v && typeof v === "object" && "modelId" in v) {
+      const model = v as {
+        provider?: string;
+        modelId: string;
+        settings?: Record<string, unknown>;
+      };
+      key.push("model", model.provider ?? "", model.modelId);
+      // Include model settings in cache key (e.g., higgsfield styleId, quality)
+      if (model.settings) {
+        key.push("modelSettings", serializeValue(model.settings));
+      }
+      continue;
+    }
+    if (typeof v === "string") {
+      if (isLocalFilePath(v)) {
+        key.push(k, getFileFingerprint(v));
+      } else {
+        key.push(k, v);
+      }
+    } else if (typeof v === "number" || typeof v === "boolean") {
+      key.push(k, v);
+    } else if (v === null || v === undefined) {
+      key.push(k, v);
+    } else if (v instanceof Uint8Array) {
+      key.push(k, Buffer.from(v).toString("base64"));
+    } else if (isVargElement(v)) {
+      key.push(k, ...computeCacheKey(v));
+    } else if (Array.isArray(v) || typeof v === "object") {
+      key.push(k, serializeValue(v));
+    }
+  }
+
+  for (const child of element.children) {
+    if (typeof child === "string") {
+      key.push("text", child);
+    } else if (typeof child === "number") {
+      key.push("num", child);
+    } else if (isVargElement(child)) {
+      key.push("child", ...computeCacheKey(child));
+    }
+  }
+
+  return key;
+}
+
+export function getTextContent(node: VargNode): string {
+  if (typeof node === "string") return node;
+  if (typeof node === "number") return String(node);
+  if (Array.isArray(node)) return node.map(getTextContent).join("");
+  if (node && typeof node === "object" && "children" in node) {
+    return node.children.map(getTextContent).join("");
+  }
+  return "";
+}

package/src/react/renderers/video.ts

@@ -0,0 +1,127 @@
+import { File } from "../../ai-sdk/file";
+import type { generateVideo } from "../../ai-sdk/generate-video";
+import type {
+  ImageInput,
+  VargElement,
+  VideoPrompt,
+  VideoProps,
+} from "../types";
+import type { RenderContext } from "./context";
+import { renderImage } from "./image";
+import { addTask, completeTask, startTask } from "./progress";
+import { computeCacheKey, toFileUrl } from "./utils";
+
+async function resolveImageInput(
+  input: ImageInput,
+  ctx: RenderContext,
+): Promise<Uint8Array> {
+  if (input instanceof Uint8Array) {
+    return input;
+  }
+  if (typeof input === "string") {
+    const response = await fetch(toFileUrl(input));
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  const path = await renderImage(input, ctx);
+  const response = await fetch(toFileUrl(path));
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function resolveMediaInput(
+  input: Uint8Array | string | undefined,
+): Promise<Uint8Array | undefined> {
+  if (!input) return undefined;
+  if (input instanceof Uint8Array) return input;
+  const response = await fetch(toFileUrl(input));
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+async function resolvePrompt(
+  prompt: VideoPrompt,
+  ctx: RenderContext,
+): Promise<
+  | string
+  | {
+      text?: string;
+      images?: Uint8Array[];
+      audio?: Uint8Array;
+      video?: Uint8Array;
+    }
+> {
+  if (typeof prompt === "string") {
+    return prompt;
+  }
+  const [resolvedImages, resolvedAudio, resolvedVideo] = await Promise.all([
+    prompt.images
+      ? Promise.all(prompt.images.map((img) => resolveImageInput(img, ctx)))
+      : undefined,
+    resolveMediaInput(prompt.audio),
+    resolveMediaInput(prompt.video),
+  ]);
+  return {
+    text: prompt.text,
+    images: resolvedImages,
+    audio: resolvedAudio,
+    video: resolvedVideo,
+  };
+}
+
+export async function renderVideo(
+  element: VargElement<"video">,
+  ctx: RenderContext,
+): Promise<string> {
+  const props = element.props as VideoProps;
+
+  if (props.src && !props.prompt) {
+    return props.src;
+  }
+
+  const prompt = props.prompt;
+  if (!prompt) {
+    throw new Error("Video element requires either 'prompt' or 'src'");
+  }
+
+  const model = props.model;
+  if (!model) {
+    throw new Error("Video element requires 'model' prop when using prompt");
+  }
+
+  // Compute cache key for deduplication
+  const cacheKey = computeCacheKey(element);
+  const cacheKeyStr = JSON.stringify(cacheKey);
+
+  // Check if this element is already being rendered (deduplication)
+  const pendingRender = ctx.pending.get(cacheKeyStr);
+  if (pendingRender) {
+    return pendingRender;
+  }
+
+  // Create the render promise and store it for deduplication
+  const renderPromise = (async () => {
+    const resolvedPrompt = await resolvePrompt(prompt, ctx);
+
+    const modelId = typeof model === "string" ? model : model.modelId;
+    const taskId = ctx.progress
+      ? addTask(ctx.progress, "video", modelId)
+      : null;
+    if (taskId && ctx.progress) startTask(ctx.progress, taskId);
+
+    const { video } = await ctx.generateVideo({
+      model,
+      prompt: resolvedPrompt,
+      duration: 5,
+      cacheKey,
+    } as Parameters<typeof generateVideo>[0]);
+
+    if (taskId && ctx.progress) completeTask(ctx.progress, taskId);
+
+    const tempPath = await File.toTemp(video);
+    ctx.tempFiles.push(tempPath);
+
+    return tempPath;
+  })();
+
+  ctx.pending.set(cacheKeyStr, renderPromise);
+
+  return renderPromise;
+}

package/src/react/runtime/jsx-dev-runtime.ts

@@ -0,0 +1,43 @@
+import type { VargElement, VargNode } from "../types";
+
+type ElementFactory = (props: Record<string, unknown>) => VargElement;
+
+export function jsx(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  const finalProps = { ...props };
+  if (key !== undefined) {
+    finalProps.key = key;
+  }
+  return type(finalProps);
+}
+
+export function jsxs(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  return jsx(type, props, key);
+}
+
+export function jsxDEV(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  return jsx(type, props, key);
+}
+
+export const Fragment = ({ children }: { children?: VargNode }) => children;
+
+export namespace JSX {
+  export type Element = VargElement;
+  // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+  export type IntrinsicElements = {};
+  export interface ElementChildrenAttribute {
+    // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+    children: {};
+  }
+}

package/src/react/runtime/jsx-runtime.ts

@@ -0,0 +1,35 @@
+import type { VargElement, VargNode } from "../types";
+
+type ElementFactory = (props: Record<string, unknown>) => VargElement;
+
+export function jsx(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  const finalProps = { ...props };
+  if (key !== undefined) {
+    finalProps.key = key;
+  }
+  return type(finalProps);
+}
+
+export function jsxs(
+  type: ElementFactory,
+  props: Record<string, unknown> | null,
+  key?: string,
+): VargElement {
+  return jsx(type, props, key);
+}
+
+export const Fragment = ({ children }: { children?: VargNode }) => children;
+
+export namespace JSX {
+  export type Element = VargElement;
+  // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+  export type IntrinsicElements = {};
+  export interface ElementChildrenAttribute {
+    // biome-ignore lint/complexity/noBannedTypes: required for JSX namespace
+    children: {};
+  }
+}

package/src/react/types.ts

@@ -0,0 +1,235 @@
+import type { ImageModelV3, SpeechModelV3 } from "@ai-sdk/provider";
+import type { MusicModelV3 } from "../ai-sdk/music-model";
+import type {
+  Position,
+  ResizeMode,
+  SizeValue,
+  TransitionOptions,
+} from "../ai-sdk/providers/editly/types";
+import type { VideoModelV3 } from "../ai-sdk/video-model";
+
+export type VargElementType =
+  | "render"
+  | "clip"
+  | "overlay"
+  | "image"
+  | "video"
+  | "animate"
+  | "speech"
+  | "talking-head"
+  | "title"
+  | "subtitle"
+  | "music"
+  | "captions"
+  | "split"
+  | "slider"
+  | "swipe"
+  | "packshot";
+
+export interface VargElement<T extends VargElementType = VargElementType> {
+  type: T;
+  props: Record<string, unknown>;
+  children: VargNode[];
+}
+
+export type VargNode =
+  | VargElement
+  | string
+  | number
+  | null
+  | undefined
+  | VargNode[];
+
+export interface BaseProps {
+  key?: string | number;
+}
+
+export interface PositionProps {
+  left?: SizeValue;
+  top?: SizeValue;
+  width?: SizeValue;
+  height?: SizeValue;
+}
+
+export interface VolumeProps {
+  volume?: number;
+}
+
+export interface AudioProps extends VolumeProps {
+  keepAudio?: boolean;
+}
+
+export type TrimProps =
+  | { cutFrom?: number; cutTo?: number; duration?: never }
+  | { cutFrom?: number; cutTo?: never; duration?: number };
+
+// Root container - sets dimensions, fps, contains clips
+export interface RenderProps extends BaseProps {
+  width?: number;
+  height?: number;
+  fps?: number;
+  normalize?: boolean;
+  children?: VargNode;
+}
+
+export interface ClipProps extends BaseProps {
+  duration?: number | "auto";
+  transition?: TransitionOptions;
+  children?: VargNode;
+}
+
+export interface OverlayProps extends BaseProps, PositionProps, AudioProps {
+  children?: VargNode;
+}
+
+export type ImageInput = Uint8Array | string | VargElement<"image">;
+export type ImagePrompt = string | { text?: string; images: ImageInput[] };
+
+export interface ImageProps extends BaseProps, PositionProps {
+  prompt?: ImagePrompt;
+  src?: string;
+  model?: ImageModelV3;
+  aspectRatio?: `${number}:${number}`;
+  zoom?: "in" | "out" | "left" | "right";
+  resize?: ResizeMode;
+  position?: Position;
+  size?: { width: string; height: string };
+  removeBackground?: boolean;
+}
+
+export type VideoPrompt =
+  | string
+  | {
+      text?: string;
+      images?: ImageInput[];
+      audio?: Uint8Array | string;
+      video?: Uint8Array | string;
+    };
+
+export type VideoProps = BaseProps &
+  PositionProps &
+  AudioProps &
+  TrimProps & {
+    prompt?: VideoPrompt;
+    src?: string;
+    model?: VideoModelV3;
+    resize?: ResizeMode;
+  };
+
+// Image-to-video animation
+export interface AnimateProps extends BaseProps, PositionProps {
+  image?: VargElement<"image">;
+  src?: string;
+  model?: VideoModelV3;
+  motion?: string;
+  duration?: number;
+}
+
+export interface SpeechProps extends BaseProps, VolumeProps {
+  voice?: string;
+  model?: SpeechModelV3;
+  id?: string;
+  children?: string;
+}
+
+export interface TalkingHeadProps extends BaseProps {
+  character?: string;
+  src?: string;
+  voice?: string;
+  model?: VideoModelV3;
+  lipsyncModel?: VideoModelV3;
+  position?:
+    | Position
+    | { left?: string; right?: string; top?: string; bottom?: string };
+  size?: { width: string; height: string };
+  children?: string;
+}
+
+export interface TitleProps extends BaseProps {
+  position?: Position;
+  color?: string;
+  start?: number;
+  end?: number;
+  children?: string;
+}
+
+export interface SubtitleProps extends BaseProps {
+  backgroundColor?: string;
+  children?: string;
+}
+
+export type MusicProps = BaseProps &
+  VolumeProps &
+  TrimProps & {
+    prompt?: string;
+    model?: MusicModelV3;
+    src?: string;
+    loop?: boolean;
+    ducking?: boolean;
+  };
+
+export interface CaptionsProps extends BaseProps {
+  src?: string | VargElement<"speech">;
+  srt?: string;
+  style?: "tiktok" | "karaoke" | "bounce" | "typewriter";
+  color?: string;
+  activeColor?: string;
+  fontSize?: number;
+}
+
+export interface SplitProps extends BaseProps {
+  direction?: "horizontal" | "vertical";
+  children?: VargNode;
+}
+
+export interface SliderProps extends BaseProps {
+  direction?: "horizontal" | "vertical";
+  children?: VargNode;
+}
+
+export interface SwipeProps extends BaseProps {
+  direction?: "left" | "right" | "up" | "down";
+  interval?: number;
+  children?: VargNode;
+}
+
+export interface PackshotProps extends BaseProps {
+  background?: VargElement<"image"> | string;
+  logo?: string;
+  logoPosition?: Position;
+  logoSize?: SizeValue;
+  cta?: string;
+  ctaPosition?: Position;
+  ctaColor?: string;
+  ctaSize?: number;
+  blinkCta?: boolean;
+  duration?: number;
+}
+
+export type RenderMode = "strict" | "default" | "preview";
+
+export interface RenderOptions {
+  output?: string;
+  cache?: string;
+  quiet?: boolean;
+  mode?: RenderMode;
+}
+
+export interface ElementPropsMap {
+  render: RenderProps;
+  clip: ClipProps;
+  overlay: OverlayProps;
+  image: ImageProps;
+  video: VideoProps;
+  animate: AnimateProps;
+  speech: SpeechProps;
+  "talking-head": TalkingHeadProps;
+  title: TitleProps;
+  subtitle: SubtitleProps;
+  music: MusicProps;
+  captions: CaptionsProps;
+  split: SplitProps;
+  slider: SliderProps;
+  swipe: SwipeProps;
+  packshot: PackshotProps;
+}

package/src/studio/index.ts

@@ -0,0 +1,26 @@
+import { resolve } from "node:path";
+import { createStudioServer } from "./server";
+
+// Parse arguments: bun run studio [file.tsx] [--cache=.cache/ai]
+let initialFile: string | undefined;
+let cacheDir = ".cache/ai";
+
+for (const arg of process.argv.slice(2)) {
+  if (arg.startsWith("--cache=")) {
+    cacheDir = arg.replace("--cache=", "");
+  } else if (arg.endsWith(".tsx") || arg.endsWith(".ts")) {
+    initialFile = resolve(arg);
+  }
+}
+
+console.log("varg studio starting...");
+console.log(`cache folder: ${cacheDir}`);
+if (initialFile) {
+  console.log(`initial file: ${initialFile}`);
+}
+
+const { port } = createStudioServer({ cacheDir, initialFile });
+
+console.log(`\nopen http://localhost:${port}`);
+console.log("  /editor - code editor");
+console.log("  /cache  - cache viewer");