vargai 0.4.0-alpha105 → 0.4.0-alpha106

package/package.json

@@ -28,6 +28,7 @@
     "@commitlint/config-conventional": "^20.0.0",
     "@size-limit/preset-small-lib": "^11.2.0",
     "@types/bun": "latest",
+    "@types/opentype.js": "^1.3.9",
     "@types/react": "^19.2.7",
     "husky": "^9.1.7",
     "lint-staged": "^16.2.7"
@@ -58,9 +59,11 @@
     "ai": "^6.0.26",
     "apify-client": "^2.20.0",
     "citty": "^0.1.6",
+    "fflate": "^0.8.2",
     "fluent-ffmpeg": "^2.1.3",
     "groq-sdk": "^0.36.0",
     "ink": "^6.5.1",
+    "opentype.js": "^1.3.4",
     "p-limit": "^6.2.0",
     "p-map": "^7.0.4",
     "react": "^19.2.0",
@@ -104,7 +107,7 @@
   "license": "Apache-2.0",
   "author": "varg.ai <hello@varg.ai> (https://varg.ai)",
   "sideEffects": false,
-  "version": "0.4.0-alpha105",
+  "version": "0.4.0-alpha106",
   "exports": {
     ".": "./src/index.ts",
     "./ai": "./src/ai-sdk/index.ts",

package/src/ai-sdk/providers/editly/backends/types.ts

@@ -47,6 +47,11 @@ export interface FFmpegRunOptions {
   verbose?: boolean;
   /** Max execution time in seconds (used by cloud backends like Rendi, ignored by local) */
   timeoutSeconds?: number;
+  /** Extra files (e.g. fonts, ASS subtitles) to include alongside inputs.
+   *  When present, cloud backends like Rendi use compressed folder mode
+   *  (input_compressed_folder) to bundle all files together.
+   *  Each entry provides either a `url` to download or raw `data` bytes. */
+  auxiliaryFiles?: { url?: string; data?: Uint8Array; fileName: string }[];
 }
 
 export type FFmpegOutput =

package/src/ai-sdk/providers/editly/rendi/index.ts

@@ -1,3 +1,4 @@
+import { zipSync } from "fflate";
 import sharp from "sharp";
 import { File } from "../../../file";
 import type { StorageProvider } from "../../../storage/types";
@@ -128,6 +129,11 @@ export class RendiBackend implements FFmpegBackend {
   }
 
   async run(options: FFmpegRunOptions): Promise<FFmpegRunResult> {
+    // When auxiliary files (e.g. fonts) are present, use compressed folder mode
+    if (options.auxiliaryFiles && options.auxiliaryFiles.length > 0) {
+      return this.runWithCompressedFolder(options);
+    }
+
     let {
       inputs,
       filterComplex,
@@ -287,6 +293,204 @@ export class RendiBackend implements FFmpegBackend {
     throw new Error("Rendi command timed out");
   }
 
+  /**
+   * Run an FFmpeg command using Rendi's input_compressed_folder mode.
+   *
+   * Used when auxiliary files (e.g. fonts for subtitle rendering) need to be
+   * bundled alongside regular inputs. Creates a ZIP containing all input files
+   * and auxiliary files, uploads it to storage, and submits to Rendi with
+   * `input_compressed_folder` instead of `input_files`.
+   *
+   * Inside the ZIP, all files are at the root level. The ffmpeg command
+   * references files by their bare filenames (not placeholders).
+   */
+  private async runWithCompressedFolder(
+    options: FFmpegRunOptions,
+  ): Promise<FFmpegRunResult> {
+    const {
+      inputs,
+      videoFilter,
+      filterComplex,
+      outputArgs = [],
+      outputPath,
+      verbose,
+      auxiliaryFiles = [],
+    } = options;
+
+    // 1. Resolve all input files to URLs
+    const inputEntries: { fileName: string; url: string }[] = [];
+    for (const input of inputs ?? []) {
+      const path = this.getInputPath(input);
+      const url = await this.resolvePath(path);
+      // Extract filename from URL or path
+      const fileName =
+        url.split("/").pop()?.split("?")[0] ?? `input_${inputEntries.length}`;
+      inputEntries.push({ fileName, url });
+    }
+
+    // 2. Download all files (inputs + auxiliary) into memory
+    const zipContents: Record<string, Uint8Array> = {};
+
+    const downloadTasks = [
+      ...inputEntries.map(async (entry) => {
+        const res = await fetch(entry.url);
+        if (!res.ok)
+          throw new Error(
+            `Failed to download input ${entry.fileName}: ${res.status}`,
+          );
+        zipContents[entry.fileName] = new Uint8Array(await res.arrayBuffer());
+      }),
+      ...auxiliaryFiles.map(async (file) => {
+        if (file.data) {
+          // Inline data — no download needed
+          zipContents[file.fileName] = file.data;
+          return;
+        }
+        if (!file.url) {
+          throw new Error(
+            `Auxiliary file ${file.fileName} has neither url nor data`,
+          );
+        }
+        const res = await fetch(file.url);
+        if (!res.ok)
+          throw new Error(
+            `Failed to download auxiliary file ${file.fileName}: ${res.status}`,
+          );
+        zipContents[file.fileName] = new Uint8Array(await res.arrayBuffer());
+      }),
+    ];
+
+    await Promise.all(downloadTasks);
+
+    if (verbose) {
+      const totalSize = Object.values(zipContents).reduce(
+        (sum, buf) => sum + buf.length,
+        0,
+      );
+      console.log(
+        `[rendi] creating ZIP with ${Object.keys(zipContents).length} files (${(totalSize / 1024 / 1024).toFixed(1)} MB)`,
+      );
+    }
+
+    // 3. Create ZIP
+    const zipData = zipSync(zipContents, { level: 1 }); // fast compression
+
+    // 4. Upload ZIP to storage
+    const zipKey = `internal/rendi-compressed-${Date.now()}.zip`;
+    const zipUrl = await this.storage.upload(
+      zipData,
+      zipKey,
+      "application/zip",
+    );
+
+    if (verbose) {
+      console.log(
+        `[rendi] uploaded ZIP (${(zipData.length / 1024 / 1024).toFixed(1)} MB) -> ${zipUrl}`,
+      );
+    }
+
+    // 5. Build ffmpeg command using bare filenames (not {{in_X}} placeholders)
+    const inputArgs: string[] = [];
+    for (const [i, input] of (inputs ?? []).entries()) {
+      if (typeof input !== "string" && "options" in input && input.options) {
+        inputArgs.push(...input.options);
+      }
+      inputArgs.push("-i", inputEntries[i]!.fileName);
+    }
+
+    const filterArgs: string[] = [];
+    if (filterComplex) {
+      filterArgs.push("-filter_complex", filterComplex);
+    }
+    if (videoFilter) {
+      // For compressed folder mode, the video filter references files by
+      // their bare filenames (already resolved in the working directory)
+      filterArgs.push("-vf", videoFilter);
+    }
+
+    const processedOutputArgs = outputArgs.filter((arg) => arg !== "-y");
+
+    const commandParts = [
+      ...inputArgs,
+      ...filterArgs,
+      ...processedOutputArgs,
+      "{{out_1}}",
+    ];
+    const ffmpegCommand = this.buildCommandString(commandParts);
+    const outputFilename = outputPath?.split("/").pop() ?? "output.mp4";
+
+    if (verbose) {
+      console.log("[rendi] input_compressed_folder:", zipUrl);
+      console.log("[rendi] ffmpeg_command:", ffmpegCommand);
+    }
+
+    // 6. Submit to Rendi with input_compressed_folder
+    const submitResponse = await fetch(`${RENDI_API_BASE}/run-ffmpeg-command`, {
+      method: "POST",
+      headers: {
+        "X-API-KEY": this.apiKey,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        input_compressed_folder: zipUrl,
+        output_files: { out_1: outputFilename },
+        ffmpeg_command: ffmpegCommand,
+        max_command_run_seconds:
+          options.timeoutSeconds ?? this.maxCommandRunSeconds,
+      }),
+    });
+
+    if (!submitResponse.ok) {
+      const errorText = await submitResponse.text();
+      throw new Error(
+        `Rendi submit failed: ${submitResponse.status} - ${errorText}`,
+      );
+    }
+
+    const { command_id } =
+      (await submitResponse.json()) as RendiCommandResponse;
+
+    if (verbose) {
+      console.log("[rendi] command_id:", command_id);
+    }
+
+    // 7. Poll for completion (same as standard run)
+    let attempts = 0;
+    while (attempts < MAX_POLL_ATTEMPTS) {
+      const statusResponse = await fetch(
+        `${RENDI_API_BASE}/commands/${command_id}`,
+        {
+          headers: { "X-API-KEY": this.apiKey },
+        },
+      );
+
+      if (!statusResponse.ok) {
+        throw new Error(`Rendi poll failed: ${statusResponse.status}`);
+      }
+
+      const status = (await statusResponse.json()) as RendiStatusResponse;
+
+      if (status.status === "SUCCESS") {
+        const outputFile = status.output_files?.out_1;
+        if (!outputFile?.storage_url) {
+          throw new Error("Rendi completed but no output URL");
+        }
+        return { output: { type: "url", url: outputFile.storage_url } };
+      }
+
+      if (status.status === "FAILED") {
+        throw new Error(
+          `Rendi command failed: ${status.error_message ?? "unknown error"}`,
+        );
+      }
+
+      await this.sleep(POLL_INTERVAL_MS);
+      attempts++;
+    }
+
+    throw new Error("Rendi command timed out");
+  }
+
   async resolvePath(input: FilePath): Promise<string> {
     if (input instanceof File) {
       return input.upload(this.storage);

package/src/react/renderers/burn-captions.ts

@@ -1,8 +1,11 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { localBackend } from "@/ai-sdk/providers/editly";
 import type {
   FFmpegBackend,
+  FFmpegInput,
   FFmpegOutput,
 } from "../../ai-sdk/providers/editly/backends/types";
+import { type EmojiOverlay, buildEmojiFilterComplex } from "./emoji";
 
 /**
  * Resolves an FFmpegOutput to a string path/URL via the backend.
@@ -16,12 +19,85 @@ async function resolveInputPathMaybeUpload(
   return backend.resolvePath(input.path);
 }
 
+/** Font file descriptor for caption rendering. */
+export interface CaptionFontFile {
+  /** Public URL of the font file (e.g. https://s3.varg.ai/fonts/Montserrat-Bold.ttf) */
+  url: string;
+  /** Filename (e.g. "Montserrat-Bold.ttf") */
+  fileName: string;
+}
+
 export interface CaptionOverlayOptions {
   video: FFmpegOutput;
   assPath: string;
   outputPath: string;
   backend?: FFmpegBackend;
   verbose?: boolean;
+  /** Font files to include for subtitle rendering. When provided, fontsdir is set. */
+  fontFiles?: CaptionFontFile[];
+  /** Emoji overlay data for color emoji rendering via image overlay. */
+  emojiOverlays?: EmojiOverlay[];
+}
+
+/** Local cache directory for fonts and emoji PNGs. */
+const LOCAL_FONTS_DIR = "/tmp/varg-caption-fonts";
+const LOCAL_EMOJI_DIR = "/tmp/varg-caption-fonts/emoji";
+
+/**
+ * Download font files to a local directory for the local FFmpeg backend.
+ * Fonts are cached by filename — only downloaded once per process lifetime.
+ * Returns the directory path where fonts were saved.
+ */
+export async function ensureLocalFonts(fontFiles: CaptionFontFile[]): Promise<string> {
+  if (!existsSync(LOCAL_FONTS_DIR)) {
+    mkdirSync(LOCAL_FONTS_DIR, { recursive: true });
+  }
+  await Promise.all(
+    fontFiles.map(async (font) => {
+      const localPath = `${LOCAL_FONTS_DIR}/${font.fileName}`;
+      if (existsSync(localPath)) return;
+      const res = await fetch(font.url);
+      if (!res.ok)
+        throw new Error(
+          `Failed to download font ${font.fileName}: ${res.status}`,
+        );
+      writeFileSync(localPath, new Uint8Array(await res.arrayBuffer()));
+    }),
+  );
+  return LOCAL_FONTS_DIR;
+}
+
+/**
+ * Download emoji PNG files to a local directory for the local FFmpeg backend.
+ * Returns an array of local file paths in the same order as the input overlays.
+ */
+async function ensureLocalEmoji(
+  overlays: EmojiOverlay[],
+): Promise<string[]> {
+  if (!existsSync(LOCAL_EMOJI_DIR)) {
+    mkdirSync(LOCAL_EMOJI_DIR, { recursive: true });
+  }
+
+  // Deduplicate by fileName (same emoji may appear multiple times)
+  const unique = new Map<string, EmojiOverlay>();
+  for (const o of overlays) {
+    if (!unique.has(o.fileName)) unique.set(o.fileName, o);
+  }
+
+  await Promise.all(
+    Array.from(unique.values()).map(async (overlay) => {
+      const localPath = `${LOCAL_EMOJI_DIR}/${overlay.fileName}`;
+      if (existsSync(localPath)) return;
+      const res = await fetch(overlay.url);
+      if (!res.ok)
+        throw new Error(
+          `Failed to download emoji ${overlay.fileName}: ${res.status}`,
+        );
+      writeFileSync(localPath, new Uint8Array(await res.arrayBuffer()));
+    }),
+  );
+
+  return overlays.map((o) => `${LOCAL_EMOJI_DIR}/${o.fileName}`);
 }
 
 /**
@@ -32,51 +108,169 @@ export interface CaptionOverlayOptions {
  * FFmpeg backends - when using a cloud backend, input files are automatically
  * uploaded to storage.
  *
- * @param options - Configuration for the caption burn operation
- * @param options.video - Source video as {@link FFmpegOutput} (file path or URL)
- * @param options.assPath - Path to the ASS subtitle file to burn
- * @param options.outputPath - Destination path for the output video (defaults to "output.mp4")
- * @param options.backend - Optional {@link FFmpegBackend} for cloud processing; uses local FFmpeg if omitted
- * @param options.verbose - Enable verbose FFmpeg logging
- *
- * @returns Promise resolving to {@link FFmpegOutput} containing the path or URL of the captioned video
+ * When `fontFiles` are provided:
+ * - **Local backend**: fonts are downloaded to a temp directory, and `fontsdir=`
+ *   is appended to the subtitles filter so FFmpeg/libass can find them.
+ * - **Cloud backend (Rendi)**: font files are passed as `auxiliaryFiles` in the
+ *   run options. The backend bundles them into a compressed input folder and
+ *   uses `fontsdir=.` so FFmpeg finds them in the working directory.
  *
- * @throws May throw if FFmpeg execution fails, input files are missing, or upload fails for cloud backends
+ * When `emojiOverlays` are provided, the function switches from a simple `-vf`
+ * filter to a `filter_complex` that first burns subtitles, then overlays each
+ * emoji PNG at the calculated position with timing. Emoji PNGs are added as
+ * additional FFmpeg inputs referenced by stream index in the filter graph.
  *
- * @example
- * ```ts
- * const result = await burnCaptions({
- *   video: { type: "file", path: "input.mp4" },
- *   assPath: "captions.ass",
- *   outputPath: "output-with-captions.mp4",
- * });
- * ```
+ * @param options - Configuration for the caption burn operation
+ * @returns Promise resolving to {@link FFmpegOutput} containing the path or URL of the captioned video
  */
 export async function burnCaptions(
   options: CaptionOverlayOptions,
 ): Promise<FFmpegOutput> {
-  const { video, assPath, outputPath = "output.mp4", verbose } = options;
-  const captions: FFmpegOutput = { type: "file", path: assPath };
-
+  const {
+    video,
+    assPath,
+    outputPath = "output.mp4",
+    verbose,
+    fontFiles,
+    emojiOverlays,
+  } = options;
   const backend = options.backend ?? localBackend;
 
+  const isCloud = backend.name !== "local";
+  const hasFonts = fontFiles && fontFiles.length > 0;
+  const hasEmoji = emojiOverlays && emojiOverlays.length > 0;
+
   const videoInput = await resolveInputPathMaybeUpload(video, backend);
-  const assInput = await resolveInputPathMaybeUpload(captions, backend);
 
-  // For cloud backends (Rendi): pass raw URL so replaceWithPlaceholders() can match
-  // and replace with {{in_X}} placeholder. Rendi downloads inputs and provides local paths.
-  // For local backend: escape for FFmpeg filter syntax (backslashes and colons)
-  const isCloud = backend.name !== "local";
-  const subtitlesPath = isCloud
-    ? assInput
-    : assInput.replace(/\\/g, "\\\\").replace(/:/g, "\\:");
+  // For local backends, resolve the ASS path via the backend (may return
+  // a local path or uploaded URL). For cloud backends, the ASS file is
+  // bundled into the compressed folder as raw bytes — no upload needed.
+  // (fal.storage rejects .ass files as "unsupported file format")
+  const assInput = isCloud
+    ? assPath
+    : await resolveInputPathMaybeUpload({ type: "file", path: assPath }, backend);
+
+  // Build the subtitles filter string (used in both simple and complex modes)
+  //
+  // Cloud backends (Rendi) always use compressed folder mode for captions.
+  // The subtitles= filter needs the ASS file accessible by filename in the
+  // working directory — Rendi's standard input_files mode uses {{in_N}}
+  // placeholders which libass can't resolve as file paths. By always passing
+  // the ASS file as an auxiliaryFile, we force runWithCompressedFolder() which
+  // extracts all files into the working directory.
+  let subtitlesFilter: string;
+  // For cloud backends, derive the bare ASS filename for the compressed folder.
+  const assFileName = isCloud
+    ? (assPath.split("/").pop() ?? "captions.ass")
+    : undefined;
+
+  if (isCloud) {
+    // Cloud: bare ASS filename, fontsdir=. if fonts are present
+    subtitlesFilter = hasFonts
+      ? `subtitles=${assFileName}:fontsdir=.`
+      : `subtitles=${assFileName}`;
+  } else if (hasFonts) {
+    // Local with fonts: download fonts, escape paths for FFmpeg filter syntax
+    const fontsDir = await ensureLocalFonts(fontFiles);
+    const escapedFontsDir = fontsDir
+      .replace(/\\/g, "\\\\")
+      .replace(/:/g, "\\:");
+    const escapedAssPath = assInput
+      .replace(/\\/g, "\\\\")
+      .replace(/:/g, "\\:");
+    subtitlesFilter = `subtitles=${escapedAssPath}:fontsdir=${escapedFontsDir}`;
+  } else {
+    // Local without fonts: just the ASS path
+    const escapedAssPath = assInput
+      .replace(/\\/g, "\\\\")
+      .replace(/:/g, "\\:");
+    subtitlesFilter = `subtitles=${escapedAssPath}`;
+  }
+
+  // Build auxiliary files for cloud backends.
+  // Always include the ASS file so we force compressed folder mode (the ASS
+  // file must be in the working directory for the subtitles= filter to find it).
+  // The ASS file is passed as raw bytes (data) to avoid uploading to fal storage
+  // which rejects .ass files as "unsupported file format".
+  // Font files are passed as URLs (they're on S3 and download fine).
+  const auxiliaryFiles = isCloud
+    ? [
+        // ASS file — read from disk as raw bytes, no upload needed
+        { data: new Uint8Array(readFileSync(assPath)), fileName: assFileName! },
+        ...(hasFonts
+          ? fontFiles.map((f) => ({ url: f.url, fileName: f.fileName }))
+          : []),
+      ]
+    : undefined;
+
+  if (hasEmoji) {
+    // ---- Emoji overlay mode: use filter_complex ----
+    //
+    // Cloud backends: ASS file is in the compressed folder (via auxiliaryFiles),
+    // NOT as an -i input. The subtitles= filter reads it by filename from the
+    // working directory. Emoji PNGs are added as inputs starting at [1].
+    //
+    // Local backend: ASS is an -i input at [1] (needed for some ffmpeg builds
+    // that require it). Emoji PNGs start at [2].
+    const inputs: FFmpegInput[] = isCloud ? [videoInput] : [videoInput, assInput];
+    const emojiInputOffset = isCloud ? 1 : 2;
+
+    if (isCloud) {
+      // For Rendi: emoji PNGs are added as inputs (they'll be in the ZIP)
+      for (const overlay of emojiOverlays) {
+        inputs.push(overlay.url);
+      }
+    } else {
+      // For local: download emoji PNGs and use local paths
+      const localPaths = await ensureLocalEmoji(emojiOverlays);
+      for (const localPath of localPaths) {
+        inputs.push(localPath);
+      }
+    }
+
+    const filterComplex = buildEmojiFilterComplex(
+      subtitlesFilter,
+      emojiOverlays,
+      emojiInputOffset,
+    );
+
+    if (verbose) {
+      console.log(
+        `[burn-captions] emoji overlay mode: ${emojiOverlays.length} emoji, filter_complex length=${filterComplex.length}`,
+      );
+    }
+
+    const result = await backend.run({
+      inputs,
+      filterComplex,
+      outputArgs: [
+        "-map", "[vout]",
+        "-map", "0:a?",
+        "-crf", "18",
+        "-preset", "fast",
+      ],
+      outputPath,
+      verbose,
+      auxiliaryFiles,
+    });
+
+    return result.output;
+  }
 
+  // ---- Simple mode: no emoji, just subtitles filter ----
+  //
+  // Cloud backends: only the video is an -i input. The ASS file is in the
+  // compressed folder (via auxiliaryFiles) and referenced by the subtitles=
+  // filter by bare filename.
+  //
+  // Local backend: both video and ASS are -i inputs.
   const result = await backend.run({
-    inputs: [videoInput, assInput],
-    videoFilter: `subtitles=${subtitlesPath}`,
+    inputs: isCloud ? [videoInput] : [videoInput, assInput],
+    videoFilter: subtitlesFilter,
     outputArgs: ["-crf", "18", "-preset", "fast", "-c:a", "copy"],
     outputPath,
     verbose,
+    auxiliaryFiles,
   });
 
   return result.output;