vargai 0.4.0-alpha4 → 0.4.0-alpha40

package/src/ai-sdk/providers/editly/rendi/editly-with-rendi-backend.test.ts

@@ -0,0 +1,335 @@
+/**
+ * Rendi backend tests - same as editly.test.ts but uses cloud ffmpeg
+ *
+ * NOTE: Free tier has 4 commands/min rate limit. Run tests individually:
+ *   bun test src/ai-sdk/providers/editly/rendi/editly-with-rendi-backend.test.ts -t "merges two"
+ */
+
+import { describe, expect, test } from "bun:test";
+import { $ } from "bun";
+import { editly } from "../index";
+import { createRendiBackend } from ".";
+
+const shouldRunRendiTests =
+  !!process.env.RENDI_INTEGRATION_TESTS && !!process.env.RENDI_API_KEY;
+
+const VIDEO_1 = "https://s3.varg.ai/test-media/sora-landscape.mp4";
+const VIDEO_2 = "https://s3.varg.ai/test-media/simpsons-scene.mp4";
+const VIDEO_TALKING =
+  "https://s3.varg.ai/test-media/workflow-talking-synced.mp4";
+const IMAGE_SQUARE = "https://s3.varg.ai/test-media/replicate-forest.png";
+
+const rendi = shouldRunRendiTests ? createRendiBackend() : (null as never);
+
+async function saveResult(
+  result: {
+    output: { type: "url"; url: string } | { type: "file"; path: string };
+  },
+  outPath: string,
+) {
+  expect(result.output.type).toBe("url");
+  if (result.output.type === "url") {
+    expect(result.output.url).toMatch(/^https:\/\//);
+    const res = await fetch(result.output.url);
+    if (!res.ok) throw new Error(`Failed to download: ${res.status}`);
+
+    const dir = outPath.split("/").slice(0, -1).join("/");
+    await $`mkdir -p ${dir}`.quiet();
+
+    const bytes = await res.arrayBuffer();
+    await Bun.write(outPath, bytes);
+
+    const written = Bun.file(outPath);
+    if (!(await written.exists()) || written.size === 0) {
+      throw new Error(`Failed to write output file: ${outPath}`);
+    }
+    console.log(`Output: ${outPath}`);
+  }
+}
+
+describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
+  test("merges two videos with fade transition", async () => {
+    const outPath = "output/rendi/merge.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      clips: [
+        {
+          layers: [{ type: "video", path: VIDEO_1 }],
+          transition: { name: "fade", duration: 0.5 },
+        },
+        {
+          layers: [{ type: "video", path: VIDEO_2 }],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("picture-in-picture (pip)", async () => {
+    const outPath = "output/rendi/pip.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "video", path: VIDEO_1 },
+            {
+              type: "video",
+              path: VIDEO_2,
+              width: "30%",
+              height: "30%",
+              left: "68%",
+              top: "2%",
+            },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("image ken burns preserves aspect ratio", async () => {
+    const outPath = "output/rendi/ken-burns.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            {
+              type: "image",
+              path: IMAGE_SQUARE,
+              zoomDirection: "in",
+              zoomAmount: 0.1,
+              resizeMode: "contain",
+            },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("subtitle layer", async () => {
+    const outPath = "output/rendi/subtitle.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "video", path: VIDEO_1 },
+            {
+              type: "subtitle",
+              text: "This is a subtitle at the bottom",
+            },
+          ],
+        },
+        {
+          duration: 3,
+          layers: [
+            { type: "video", path: VIDEO_2 },
+            {
+              type: "subtitle",
+              text: "Another subtitle with custom colors",
+              textColor: "yellow",
+              backgroundColor: "blue@0.8",
+            },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("news-title layer", async () => {
+    const outPath = "output/rendi/news-title.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "video", path: VIDEO_1 },
+            {
+              type: "news-title",
+              text: "BREAKING NEWS: Something important happened",
+              backgroundColor: "red",
+            },
+          ],
+        },
+        {
+          duration: 3,
+          layers: [
+            { type: "video", path: VIDEO_2 },
+            {
+              type: "news-title",
+              text: "TOP STORY",
+              backgroundColor: "blue",
+              position: "top",
+            },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("keepSourceAudio preserves original video audio", async () => {
+    const outPath = "output/rendi/keep-audio.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      keepSourceAudio: true,
+      clips: [
+        {
+          layers: [
+            { type: "video", path: VIDEO_TALKING },
+            { type: "subtitle", text: "Original audio should play" },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("keepSourceAudio with cutFrom stays in sync", async () => {
+    const outPath = "output/rendi/keep-audio-cut.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1280,
+      height: 720,
+      fps: 30,
+      keepSourceAudio: true,
+      clips: [
+        {
+          layers: [
+            { type: "video", path: VIDEO_TALKING, cutFrom: 2, cutTo: 6 },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("contain-blur resize mode for video", async () => {
+    const outPath = "output/rendi/contain-blur.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1080,
+      height: 1920,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "video", path: VIDEO_1, resizeMode: "contain-blur" },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("video overlay with cropPosition", async () => {
+    const outPath = "output/rendi/crop-position.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1080,
+      height: 1920,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "fill-color", color: "#000000" },
+            {
+              type: "video",
+              path: VIDEO_1,
+              width: 1080,
+              height: 960,
+              left: 0,
+              top: 0,
+              resizeMode: "cover",
+              cropPosition: "top",
+            },
+            {
+              type: "video",
+              path: VIDEO_2,
+              width: 1080,
+              height: 960,
+              left: 0,
+              top: 960,
+              resizeMode: "cover",
+              cropPosition: "bottom",
+            },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+
+  test("portrait 9:16 image with zoompan cover mode", async () => {
+    const outPath = "output/rendi/portrait-zoompan.mp4";
+    const result = await editly({
+      outPath,
+      backend: rendi,
+      width: 1080,
+      height: 1920,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            {
+              type: "image",
+              path: IMAGE_SQUARE,
+              zoomDirection: "in",
+              zoomAmount: 0.1,
+              resizeMode: "cover",
+            },
+          ],
+        },
+      ],
+    });
+
+    await saveResult(result, outPath);
+  }, 120000);
+});

package/src/ai-sdk/providers/editly/rendi/index.ts

@@ -0,0 +1,289 @@
+import type {
+  FFmpegBackend,
+  FFmpegInput,
+  FFmpegRunOptions,
+  FFmpegRunResult,
+  VideoInfo,
+} from "../backends/types";
+
+const RENDI_API_BASE = "https://api.rendi.dev/v1";
+const POLL_INTERVAL_MS = 2000;
+const MAX_POLL_ATTEMPTS = 300;
+const DEFAULT_MAX_COMMAND_SECONDS = 60;
+
+interface RendiCommandResponse {
+  command_id: string;
+}
+
+interface RendiStoredFile {
+  file_id: string;
+  storage_url: string | null;
+  status: string;
+  duration?: number;
+  width?: number;
+  height?: number;
+  frame_rate?: number;
+}
+
+interface RendiStatusResponse {
+  command_id: string;
+  status: "QUEUED" | "PROCESSING" | "SUCCESS" | "FAILED";
+  error_message?: string;
+  output_files?: Record<string, RendiStoredFile>;
+}
+
+export class RendiBackend implements FFmpegBackend {
+  readonly name = "rendi";
+  private apiKey: string;
+
+  constructor(apiKey?: string) {
+    this.apiKey = apiKey ?? process.env.RENDI_API_KEY ?? "";
+    if (!this.apiKey) {
+      throw new Error("RENDI_API_KEY is required for Rendi backend");
+    }
+  }
+
+  async ffprobe(input: string): Promise<VideoInfo> {
+    const inputUrl = this.ensureUrl(input);
+
+    const submitResponse = await fetch(`${RENDI_API_BASE}/run-ffmpeg-command`, {
+      method: "POST",
+      headers: {
+        "X-API-KEY": this.apiKey,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        input_files: { in_1: inputUrl },
+        output_files: { out_1: "probe.mp4" },
+        ffmpeg_command: "-i {{in_1}} -c copy {{out_1}}",
+        max_command_run_seconds: DEFAULT_MAX_COMMAND_SECONDS,
+      }),
+    });
+
+    if (!submitResponse.ok) {
+      throw new Error(`Rendi ffprobe failed: ${submitResponse.status}`);
+    }
+
+    const { command_id } =
+      (await submitResponse.json()) as RendiCommandResponse;
+
+    let attempts = 0;
+    while (attempts < MAX_POLL_ATTEMPTS) {
+      const statusResponse = await fetch(
+        `${RENDI_API_BASE}/commands/${command_id}`,
+        { headers: { "X-API-KEY": this.apiKey } },
+      );
+
+      if (!statusResponse.ok) {
+        throw new Error(`Rendi ffprobe poll failed: ${statusResponse.status}`);
+      }
+
+      const status = (await statusResponse.json()) as RendiStatusResponse;
+
+      if (status.status === "SUCCESS") {
+        const output = status.output_files?.out_1;
+        if (!output) {
+          throw new Error("rendi ffprobe completed but no output metadata");
+        }
+        return {
+          duration: output.duration ?? 0,
+          width: output.width,
+          height: output.height,
+          fps: output.frame_rate,
+        };
+      }
+
+      if (status.status === "FAILED") {
+        throw new Error(`Rendi ffprobe failed: ${status.error_message}`);
+      }
+
+      await this.sleep(POLL_INTERVAL_MS);
+      attempts++;
+    }
+
+    throw new Error("Rendi ffprobe timed out");
+  }
+
+  private getInputPath(input: FFmpegInput): string {
+    if (typeof input === "string") return input;
+    if ("raw" in input) throw new Error("raw inputs not supported in Rendi");
+    return input.path;
+  }
+
+  async run(options: FFmpegRunOptions): Promise<FFmpegRunResult> {
+    const {
+      inputs,
+      filterComplex,
+      videoFilter,
+      outputArgs = [],
+      outputPath,
+      verbose,
+    } = options;
+
+    const inputFiles: Record<string, string> = {};
+    const pathToPlaceholder = new Map<string, string>();
+
+    for (let i = 0; i < inputs.length; i++) {
+      const input = inputs[i]!;
+      const path = this.getInputPath(input);
+      const url = this.ensureUrl(path);
+      const placeholder = `in_${i + 1}`;
+      inputFiles[placeholder] = url;
+      pathToPlaceholder.set(path, `{{${placeholder}}}`);
+    }
+
+    const replaceWithPlaceholders = (str: string): string => {
+      let result = str;
+      const sortedEntries = [...pathToPlaceholder.entries()].sort(
+        (a, b) => b[0].length - a[0].length,
+      );
+      for (const [url, ph] of sortedEntries) {
+        if (result.includes(url)) {
+          result = result.replaceAll(url, ph);
+        }
+      }
+      return result;
+    };
+
+    const inputArgs: string[] = [];
+    for (let i = 0; i < inputs.length; i++) {
+      const input = inputs[i]!;
+      if (typeof input !== "string" && "options" in input && input.options) {
+        inputArgs.push(...input.options);
+      }
+      inputArgs.push("-i", `{{in_${i + 1}}}`);
+    }
+
+    const filterArgs: string[] = [];
+    if (filterComplex) {
+      filterArgs.push(
+        "-filter_complex",
+        replaceWithPlaceholders(filterComplex),
+      );
+    }
+    if (videoFilter) {
+      filterArgs.push("-vf", replaceWithPlaceholders(videoFilter));
+    }
+
+    const processedOutputArgs = outputArgs
+      .filter((arg) => arg !== "-y")
+      .map((arg) => replaceWithPlaceholders(arg));
+
+    const commandParts = [
+      ...inputArgs,
+      ...filterArgs,
+      ...processedOutputArgs,
+      "{{out_1}}",
+    ];
+    const ffmpegCommand = this.buildCommandString(commandParts);
+    const outputFilename = outputPath?.split("/").pop() ?? "output.mp4";
+
+    if (verbose) {
+      console.log("[rendi] input_files:", inputFiles);
+      console.log("[rendi] ffmpeg_command:", ffmpegCommand);
+    }
+
+    const submitResponse = await fetch(`${RENDI_API_BASE}/run-ffmpeg-command`, {
+      method: "POST",
+      headers: {
+        "X-API-KEY": this.apiKey,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        input_files: inputFiles,
+        output_files: { out_1: outputFilename },
+        ffmpeg_command: ffmpegCommand,
+        max_command_run_seconds: DEFAULT_MAX_COMMAND_SECONDS,
+      }),
+    });
+
+    if (!submitResponse.ok) {
+      const errorText = await submitResponse.text();
+      throw new Error(
+        `Rendi submit failed: ${submitResponse.status} - ${errorText}`,
+      );
+    }
+
+    const { command_id } =
+      (await submitResponse.json()) as RendiCommandResponse;
+
+    if (verbose) {
+      console.log("[rendi] command_id:", command_id);
+    }
+
+    let attempts = 0;
+    while (attempts < MAX_POLL_ATTEMPTS) {
+      const statusResponse = await fetch(
+        `${RENDI_API_BASE}/commands/${command_id}`,
+        {
+          headers: { "X-API-KEY": this.apiKey },
+        },
+      );
+
+      if (!statusResponse.ok) {
+        throw new Error(`Rendi poll failed: ${statusResponse.status}`);
+      }
+
+      const status = (await statusResponse.json()) as RendiStatusResponse;
+
+      if (verbose && attempts % 5 === 0) {
+        console.log("[rendi] status:", status.status);
+      }
+
+      if (status.status === "SUCCESS") {
+        const outputFile = status.output_files?.out_1;
+        if (!outputFile?.storage_url) {
+          throw new Error("Rendi completed but no output URL found");
+        }
+
+        if (verbose) {
+          console.log("[rendi] output url:", outputFile.storage_url);
+        }
+
+        return { output: { type: "url", url: outputFile.storage_url } };
+      }
+
+      if (status.status === "FAILED") {
+        throw new Error(
+          `Rendi command failed: ${status.error_message ?? "Unknown error"}`,
+        );
+      }
+
+      await this.sleep(POLL_INTERVAL_MS);
+      attempts++;
+    }
+
+    throw new Error("Rendi command timed out");
+  }
+
+  private ensureUrl(input: string): string {
+    if (input.startsWith("http://") || input.startsWith("https://")) {
+      return input;
+    }
+    throw new Error(`Rendi backend requires URLs, got local path: ${input}`);
+  }
+
+  private buildCommandString(args: string[]): string {
+    return args
+      .map((arg) => {
+        if (arg.startsWith("-") || arg.startsWith("{{")) {
+          return arg;
+        }
+        if (arg.includes(" ") || arg.includes(":") || arg.includes("'")) {
+          return `"${arg.replace(/"/g, '\\"')}"`;
+        }
+        return arg;
+      })
+      .join(" ");
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+
+export function createRendiBackend(apiKey?: string): RendiBackend {
+  return new RendiBackend(apiKey);
+}
+
+export type { FFmpegBackend } from "../backends/types";

package/src/ai-sdk/providers/editly/rendi/rendi.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from "bun:test";
+import { createRendiBackend } from ".";
+
+const hasRendiKey = !!process.env.RENDI_API_KEY;
+
+describe.skipIf(!hasRendiKey)("rendi backend", () => {
+  test("ffprobe remote file", async () => {
+    const backend = createRendiBackend();
+    const info = await backend.ffprobe(
+      "https://storage.rendi.dev/sample/big_buck_bunny_720p_5sec_intro.mp4",
+    );
+
+    expect(info.duration).toBeGreaterThan(0);
+    expect(info.width).toBe(1280);
+    expect(info.height).toBe(720);
+  }, 30000);
+
+  test("run simple ffmpeg command", async () => {
+    const backend = createRendiBackend();
+
+    const result = await backend.run({
+      inputs: [
+        "https://storage.rendi.dev/sample/big_buck_bunny_720p_5sec_intro.mp4",
+      ],
+      outputArgs: ["-t", "2", "-c:v", "libx264", "-preset", "ultrafast"],
+      outputPath: "output.mp4",
+      verbose: true,
+    });
+
+    expect(result.output.type).toBe("url");
+    if (result.output.type === "url") {
+      expect(result.output.url).toMatch(/^https:\/\//);
+    }
+  }, 120000);
+});

package/src/ai-sdk/providers/editly/types.ts

@@ -1,6 +1,8 @@
 // Types from original editly (https://github.com/mifi/editly)
 // Adapted for pure ffmpeg implementation (no fabric/canvas/gl dependencies)
 
+import type { FFmpegBackend } from "./backends";
+
 export type OriginX = "left" | "center" | "right";
 export type OriginY = "top" | "center" | "bottom";
 export type SizeValue = number | `${number}%` | `${number}px`;
@@ -87,6 +89,21 @@ export interface TextLayer extends BaseLayer {
   fontFamily?: string;
 }
 
+/**
+ * Crop position anchor for cover mode.
+ * NOTE: This is a varg extension to editly, not in the original.
+ */
+export type CropPosition =
+  | "center"
+  | "top"
+  | "bottom"
+  | "left"
+  | "right"
+  | "top-left"
+  | "top-right"
+  | "bottom-left"
+  | "bottom-right";
+
 /**
  * For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`.
  * If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`.
@@ -95,6 +112,7 @@ export interface VideoLayer extends BaseLayer {
   type: "video";
   path: string;
   resizeMode?: ResizeMode;
+  cropPosition?: CropPosition;
   cutFrom?: number;
   cutTo?: number;
   width?: SizeValue;
@@ -309,6 +327,18 @@ export interface EditlyConfig {
   audioNorm?: AudioNormalizationOptions;
   verbose?: boolean;
   enableFfmpegLog?: boolean;
+  /** End output when shortest stream ends (video or audio) */
+  shortest?: boolean;
+  /** FFmpeg backend for execution (defaults to local ffmpeg) */
+  backend?: FFmpegBackend;
+}
+
+export type EditlyOutput =
+  | { type: "file"; path: string }
+  | { type: "url"; url: string };
+
+export interface EditlyResult {
+  output: EditlyOutput;
 }
 
 // Internal types used by our implementation