vargai 0.4.0-alpha45 → 0.4.0-alpha47

package/package.json

@@ -69,7 +69,7 @@
     "sharp": "^0.34.5",
     "zod": "^4.2.1"
   },
-  "version": "0.4.0-alpha45",
+  "version": "0.4.0-alpha47",
   "exports": {
     ".": "./src/index.ts",
     "./ai": "./src/ai-sdk/index.ts",

package/src/ai-sdk/file.ts

@@ -1,4 +1,5 @@
 import type { ImageModelV3File } from "@ai-sdk/provider";
+import type { StorageProvider } from "./storage/types";
 
 export class File {
   private _data: Uint8Array | null = null;
@@ -8,13 +9,14 @@ export class File {
 
   private constructor(
     options:
-      | { data: Uint8Array; mediaType: string }
+      | { data: Uint8Array; mediaType: string; url?: string }
       | { url: string; mediaType?: string }
       | { loader: () => Promise<Uint8Array>; mediaType: string },
   ) {
     if ("data" in options) {
       this._data = options.data;
       this._mediaType = options.mediaType;
+      this._url = options.url ?? null;
     } else if ("url" in options) {
       this._url = options.url;
       this._mediaType = options.mediaType ?? inferMediaType(options.url);
@@ -46,10 +48,12 @@ export class File {
   static fromGenerated(generated: {
     uint8Array: Uint8Array;
     mediaType: string;
+    url?: string;
   }): File {
     return new File({
       data: generated.uint8Array,
       mediaType: generated.mediaType,
+      url: generated.url,
     });
   }
 
@@ -119,6 +123,10 @@ export class File {
     return this._mediaType.startsWith("video/");
   }
 
+  get url(): string | null {
+    return this._url;
+  }
+
   async data(): Promise<Uint8Array> {
     if (this._data) return this._data;
     if (this._loader) {
@@ -142,18 +150,17 @@ export class File {
     return new Blob([data], { type: this._mediaType });
   }
 
-  async url(uploader?: (blob: Blob) => Promise<string>): Promise<string> {
-    if (this._url && !this._data && !this._loader) {
-      return this._url;
-    }
-    const blob = await this.blob();
-    if (uploader) return uploader(blob);
-    try {
-      const { fal } = await import("@fal-ai/client");
-      return fal.storage.upload(blob);
-    } catch {
-      throw new Error("No uploader provided and @fal-ai/client not available.");
-    }
+  /**
+   * Upload file to storage and return the URL. Returns cached URL if already uploaded.
+   * @param storage - Storage provider to use for upload
+   * @returns URL of the uploaded file
+   */
+  async upload(storage: StorageProvider): Promise<string> {
+    if (this._url) return this._url;
+    const data = await this.data();
+    const key = `varg/${Date.now()}-${Math.random().toString(36).slice(2)}${this.extensionFromMediaType()}`;
+    this._url = await storage.upload(data, key, this._mediaType);
+    return this._url;
   }
 
   async base64(): Promise<string> {
@@ -166,14 +173,18 @@ export class File {
   }
 
   async toInput(): Promise<ImageModelV3File> {
-    if (this._url && !this._data && !this._loader) {
+    if (this._url) {
       return { type: "url", url: this._url };
     }
     const data = await this.arrayBuffer();
     return { type: "file", mediaType: this._mediaType, data };
   }
 
-  async toTemp(): Promise<string> {
+  /**
+   * Write file data to a temporary file and return the path.
+   * @returns Path to the temporary file
+   */
+  async toTempFile(): Promise<string> {
     const data = await this.data();
     const ext = this.extensionFromMediaType();
     const tmpDir = process.env.TMPDIR ?? "/tmp";
@@ -189,10 +200,10 @@ export class File {
       | File,
   ): Promise<string> {
     if (file instanceof File) {
-      return file.toTemp();
+      return file.toTempFile();
     }
     const f = File.from(file);
-    return f.toTemp();
+    return f.toTempFile();
   }
 
   private extensionFromMediaType(): string {

package/src/ai-sdk/index.ts

@@ -54,8 +54,14 @@ export {
   type Clip as EditlyClip,
   type EditlyConfig,
   editly,
+  type FFmpegBackend,
   type Layer as EditlyLayer,
+  localBackend,
 } from "./providers/editly";
+export {
+  createRendiBackend,
+  type RendiBackendOptions,
+} from "./providers/editly/rendi";
 export {
   createElevenLabs,
   type ElevenLabsProvider,
@@ -92,6 +98,12 @@ export {
   createTogetherProvider,
   together,
 } from "./providers/together";
+export {
+  falStorage,
+  type R2StorageOptions,
+  r2Storage,
+  type StorageProvider,
+} from "./storage";
 export type {
   VideoModelV3,
   VideoModelV3CallOptions,

package/src/ai-sdk/providers/editly/backends/index.ts

@@ -4,5 +4,6 @@ export type {
   FFmpegInput,
   FFmpegRunOptions,
   FFmpegRunResult,
+  FilePath,
   VideoInfo,
 } from "./types";

package/src/ai-sdk/providers/editly/backends/local.ts

@@ -1,9 +1,11 @@
 import { $ } from "bun";
+import { File } from "../../../file";
 import type {
   FFmpegBackend,
   FFmpegInput,
   FFmpegRunOptions,
   FFmpegRunResult,
+  FilePath,
   VideoInfo,
 } from "./types";
 
@@ -38,16 +40,23 @@ export class LocalBackend implements FFmpegBackend {
     };
   }
 
-  private buildInputArgs(inputs: FFmpegInput[]): string[] {
+  async resolvePath(path: FilePath): Promise<string> {
+    if (typeof path === "string") return path;
+    return path.url ?? (await path.toTempFile());
+  }
+
+  private async buildInputArgs(inputs: FFmpegInput[]): Promise<string[]> {
     const args: string[] = [];
     for (const input of inputs) {
-      if (typeof input === "string") {
+      if (input instanceof File) {
+        args.push("-i", await this.resolvePath(input));
+      } else if (typeof input === "string") {
         args.push("-i", input);
       } else if ("raw" in input) {
         args.push(...input.raw);
       } else {
         if (input.options) args.push(...input.options);
-        args.push("-i", input.path);
+        args.push("-i", await this.resolvePath(input.path));
       }
     }
     return args;
@@ -63,7 +72,7 @@ export class LocalBackend implements FFmpegBackend {
       verbose,
     } = options;
 
-    const inputArgs = this.buildInputArgs(inputs);
+    const inputArgs = await this.buildInputArgs(inputs);
 
     const ffmpegArgs = [
       "-hide_banner",

package/src/ai-sdk/providers/editly/backends/types.ts

@@ -3,6 +3,7 @@
  * Allows switching between local ffmpeg and cloud services like Rendi
  */
 
+import type { File } from "../../../file";
 import type { VideoInfo } from "../types";
 
 /**
@@ -10,14 +11,16 @@ import type { VideoInfo } from "../types";
  */
 export type { VideoInfo };
 
+export type FilePath = File | string;
+
 /**
- * Represents an input to ffmpeg - can be a simple path/URL or structured with options
+ * Represents an input to ffmpeg - can be a simple path/URL, File, or structured with options
  */
 export type FFmpegInput =
-  | string
+  | FilePath
   | {
-      /** Path or URL to the input file */
-      path: string;
+      /** Path, URL, or File for the input */
+      path: FilePath;
       /** Options to apply BEFORE the -i flag (e.g. -ss 5 for seeking) */
       options?: string[];
     }
@@ -65,6 +68,13 @@ export interface FFmpegBackend {
    */
   ffprobe(input: string): Promise<VideoInfo>;
 
+  /**
+   * Resolve a FilePath to a string path/URL for ffmpeg
+   * Local backend: returns URL if available, otherwise writes to temp
+   * Rendi backend: uploads local files, returns URL
+   */
+  resolvePath(path: FilePath): Promise<string>;
+
   /**
    * Run ffmpeg command
    * @param options - Execution options including args, inputs, and output path

package/src/ai-sdk/providers/editly/rendi/editly-with-rendi-backend.test.ts

@@ -7,6 +7,7 @@
 
 import { describe, expect, test } from "bun:test";
 import { $ } from "bun";
+import type { StorageProvider } from "../../../storage/types";
 import { editly } from "../index";
 import { createRendiBackend } from ".";
 
@@ -19,7 +20,15 @@ const VIDEO_TALKING =
   "https://s3.varg.ai/test-media/workflow-talking-synced.mp4";
 const IMAGE_SQUARE = "https://s3.varg.ai/test-media/replicate-forest.png";
 
-const rendi = shouldRunRendiTests ? createRendiBackend() : (null as never);
+const mockStorage: StorageProvider = {
+  async upload() {
+    throw new Error("Mock storage - upload not expected in this test");
+  },
+};
+
+const rendiBackend = shouldRunRendiTests
+  ? createRendiBackend({ storage: mockStorage })
+  : (null as never);
 
 async function saveResult(
   result: {
@@ -52,7 +61,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/merge.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -74,7 +83,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/pip.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -103,7 +112,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/ken-burns.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -130,7 +139,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/subtitle.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -167,7 +176,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/news-title.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -205,7 +214,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/keep-audio.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -227,7 +236,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/keep-audio-cut.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1280,
       height: 720,
       fps: 30,
@@ -248,7 +257,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/contain-blur.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1080,
       height: 1920,
       fps: 30,
@@ -269,7 +278,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/crop-position.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1080,
       height: 1920,
       fps: 30,
@@ -310,7 +319,7 @@ describe.skipIf(!shouldRunRendiTests)("editly (rendi backend)", () => {
     const outPath = "output/rendi/portrait-zoompan.mp4";
     const result = await editly({
       outPath,
-      backend: rendi,
+      backend: rendiBackend,
       width: 1080,
       height: 1920,
       fps: 30,

package/src/ai-sdk/providers/editly/rendi/index.ts

@@ -1,8 +1,11 @@
+import { File } from "../../../file";
+import type { StorageProvider } from "../../../storage/types";
 import type {
   FFmpegBackend,
   FFmpegInput,
   FFmpegRunOptions,
   FFmpegRunResult,
+  FilePath,
   VideoInfo,
 } from "../backends/types";
 
@@ -32,19 +35,26 @@ interface RendiStatusResponse {
   output_files?: Record<string, RendiStoredFile>;
 }
 
+export interface RendiBackendOptions {
+  apiKey?: string;
+  storage: StorageProvider;
+}
+
 export class RendiBackend implements FFmpegBackend {
   readonly name = "rendi";
   private apiKey: string;
+  private storage: StorageProvider;
 
-  constructor(apiKey?: string) {
-    this.apiKey = apiKey ?? process.env.RENDI_API_KEY ?? "";
+  constructor(options: RendiBackendOptions) {
+    this.apiKey = options.apiKey ?? process.env.RENDI_API_KEY ?? "";
+    this.storage = options.storage;
     if (!this.apiKey) {
       throw new Error("RENDI_API_KEY is required for Rendi backend");
     }
   }
 
   async ffprobe(input: string): Promise<VideoInfo> {
-    const inputUrl = this.ensureUrl(input);
+    const inputUrl = await this.resolvePath(input);
 
     const submitResponse = await fetch(`${RENDI_API_BASE}/run-ffmpeg-command`, {
       method: "POST",
@@ -104,7 +114,8 @@ export class RendiBackend implements FFmpegBackend {
     throw new Error("Rendi ffprobe timed out");
   }
 
-  private getInputPath(input: FFmpegInput): string {
+  private getInputPath(input: FFmpegInput): FilePath {
+    if (input instanceof File) return input;
     if (typeof input === "string") return input;
     if ("raw" in input) throw new Error("raw inputs not supported in Rendi");
     return input.path;
@@ -125,10 +136,10 @@ export class RendiBackend implements FFmpegBackend {
 
     for (const [i, input] of inputs.entries()) {
       const path = this.getInputPath(input);
-      const url = this.ensureUrl(path);
+      const url = await this.resolvePath(path);
       const placeholder = `in_${i + 1}`;
       inputFiles[placeholder] = url;
-      pathToPlaceholder.set(path, `{{${placeholder}}}`);
+      pathToPlaceholder.set(url, `{{${placeholder}}}`);
     }
 
     const replaceWithPlaceholders = (str: string): string => {
@@ -254,11 +265,15 @@ export class RendiBackend implements FFmpegBackend {
     throw new Error("Rendi command timed out");
   }
 
-  private ensureUrl(input: string): string {
+  async resolvePath(input: FilePath): Promise<string> {
+    if (input instanceof File) {
+      return input.upload(this.storage);
+    }
     if (input.startsWith("http://") || input.startsWith("https://")) {
       return input;
     }
-    throw new Error(`Rendi backend requires URLs, got local path: ${input}`);
+    const file = File.fromPath(input);
+    return file.upload(this.storage);
   }
 
   private buildCommandString(args: string[]): string {
@@ -280,8 +295,8 @@ export class RendiBackend implements FFmpegBackend {
   }
 }
 
-export function createRendiBackend(apiKey?: string): RendiBackend {
-  return new RendiBackend(apiKey);
+export function createRendiBackend(options: RendiBackendOptions): RendiBackend {
+  return new RendiBackend(options);
 }
 
 export type { FFmpegBackend } from "../backends/types";

package/src/ai-sdk/providers/editly/rendi/rendi.test.ts

@@ -1,11 +1,18 @@
 import { describe, expect, test } from "bun:test";
+import type { StorageProvider } from "../../../storage/types";
 import { createRendiBackend } from ".";
 
 const hasRendiKey = !!process.env.RENDI_API_KEY;
 
+const mockStorage: StorageProvider = {
+  async upload() {
+    throw new Error("Mock storage - upload not expected in this test");
+  },
+};
+
 describe.skipIf(!hasRendiKey)("rendi backend", () => {
   test("ffprobe remote file", async () => {
-    const backend = createRendiBackend();
+    const backend = createRendiBackend({ storage: mockStorage });
     const info = await backend.ffprobe(
       "https://storage.rendi.dev/sample/big_buck_bunny_720p_5sec_intro.mp4",
     );
@@ -16,7 +23,7 @@ describe.skipIf(!hasRendiKey)("rendi backend", () => {
   }, 30000);
 
   test("run simple ffmpeg command", async () => {
-    const backend = createRendiBackend();
+    const backend = createRendiBackend({ storage: mockStorage });
 
     const result = await backend.run({
       inputs: [

package/src/ai-sdk/providers/fal.ts

@@ -12,6 +12,7 @@ import {
 } from "@ai-sdk/provider";
 import { fal } from "@fal-ai/client";
 import pMap from "p-map";
+import type { CacheStorage } from "../cache";
 import { fileCache } from "../file-cache";
 import type { VideoModelV3, VideoModelV3CallOptions } from "../video-model";
 
@@ -21,7 +22,54 @@ interface PendingRequest {
   submitted_at: number;
 }
 
-const pendingStorage = fileCache({ dir: ".cache/fal-pending" });
+const memoryStorage = new Map<string, { value: unknown; expires: number }>();
+
+function createMemoryCache(): CacheStorage {
+  return {
+    async get(key: string) {
+      const entry = memoryStorage.get(key);
+      if (!entry) return undefined;
+      if (entry.expires && Date.now() > entry.expires) {
+        memoryStorage.delete(key);
+        return undefined;
+      }
+      return entry.value;
+    },
+    async set(key: string, value: unknown, ttl?: number) {
+      memoryStorage.set(key, { value, expires: ttl ? Date.now() + ttl : 0 });
+    },
+    async delete(key: string) {
+      memoryStorage.delete(key);
+    },
+  };
+}
+
+// TODO: allow passing CacheStorage via providerOptions.fal.cacheStorage for proper serverless support
+function isFilesystemWritable(): boolean {
+  try {
+    const testPath = `.cache/.write-test-${Date.now()}`;
+    Bun.spawnSync(["mkdir", "-p", ".cache"]);
+    const result = Bun.spawnSync(["touch", testPath]);
+    if (result.exitCode === 0) {
+      Bun.spawnSync(["rm", testPath]);
+      return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+const USE_FILE_CACHE = isFilesystemWritable();
+
+function createFalCache(name: string): CacheStorage {
+  if (!USE_FILE_CACHE) {
+    return createMemoryCache();
+  }
+  return fileCache({ dir: `.cache/${name}` });
+}
+
+const pendingStorage = createFalCache("fal-pending");
 
 const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000;
 const FAL_TIMEOUT_MS = (() => {
@@ -183,7 +231,7 @@ function detectImageType(bytes: Uint8Array): string | undefined {
   return undefined;
 }
 
-const uploadCache = fileCache({ dir: ".cache/fal-uploads" });
+const uploadCache = createFalCache("fal-uploads");
 
 async function fileToUrl(file: ImageModelV3File): Promise<string> {
   if (file.type === "url") return file.url;

package/src/ai-sdk/storage/fal.ts

@@ -0,0 +1,11 @@
+import type { StorageProvider } from "./types";
+
+export function falStorage(): StorageProvider {
+  return {
+    async upload(data: Uint8Array, _key: string, mediaType: string) {
+      const { fal } = await import("@fal-ai/client");
+      const blob = new Blob([data], { type: mediaType });
+      return fal.storage.upload(blob);
+    },
+  };
+}

package/src/ai-sdk/storage/index.ts

@@ -0,0 +1,3 @@
+export { falStorage } from "./fal";
+export { type R2StorageOptions, r2Storage } from "./r2";
+export type { StorageProvider } from "./types";

package/src/ai-sdk/storage/r2.ts

@@ -0,0 +1,55 @@
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import type { StorageProvider } from "./types";
+
+export interface R2StorageOptions {
+  endpoint?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  bucket?: string;
+  publicUrl?: string;
+}
+
+export function r2Storage(options?: R2StorageOptions): StorageProvider {
+  const endpoint = options?.endpoint ?? process.env.CLOUDFLARE_R2_API_URL;
+  const accessKeyId =
+    options?.accessKeyId ?? process.env.CLOUDFLARE_ACCESS_KEY_ID;
+  const secretAccessKey =
+    options?.secretAccessKey ?? process.env.CLOUDFLARE_ACCESS_SECRET;
+  const bucket = options?.bucket ?? process.env.CLOUDFLARE_R2_BUCKET ?? "m";
+  const publicUrl = options?.publicUrl ?? "https://s3.varg.ai";
+
+  if (!endpoint || !accessKeyId || !secretAccessKey) {
+    throw new Error(
+      "R2 storage requires endpoint, accessKeyId, and secretAccessKey. " +
+        "Set CLOUDFLARE_R2_API_URL, CLOUDFLARE_ACCESS_KEY_ID, CLOUDFLARE_ACCESS_SECRET env vars " +
+        "or pass options to r2Storage().",
+    );
+  }
+
+  const client = new S3Client({
+    region: "auto",
+    endpoint,
+    credentials: { accessKeyId, secretAccessKey },
+  });
+
+  const getPublicUrl = (objectKey: string): string => {
+    if (endpoint.includes("localhost")) {
+      return `${endpoint}/${objectKey}`;
+    }
+    return `${publicUrl}/${objectKey}`;
+  };
+
+  return {
+    async upload(data: Uint8Array, key: string, mediaType: string) {
+      await client.send(
+        new PutObjectCommand({
+          Bucket: bucket,
+          Key: key,
+          Body: data,
+          ContentType: mediaType,
+        }),
+      );
+      return getPublicUrl(key);
+    },
+  };
+}

package/src/ai-sdk/storage/types.ts

@@ -0,0 +1,3 @@
+export interface StorageProvider {
+  upload(data: Uint8Array, key: string, mediaType: string): Promise<string>;
+}

package/src/react/renderers/cache.test.ts

@@ -4,7 +4,9 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import type { ImageModelV3 } from "@ai-sdk/provider";
 import { withCache } from "../../ai-sdk/cache";
+import type { File } from "../../ai-sdk/file";
 import { fileCache } from "../../ai-sdk/file-cache";
+import { localBackend } from "../../ai-sdk/providers/editly";
 import type { VideoModelV3 } from "../../ai-sdk/video-model";
 import { Image, Video } from "../elements";
 import type { RenderContext } from "./context";
@@ -99,7 +101,8 @@ function createContext(
     generateImage: generateImage as unknown as RenderContext["generateImage"],
     generateVideo: generateVideo as unknown as RenderContext["generateVideo"],
     tempFiles: [],
-    pending: new Map(),
+    pendingFiles: new Map<string, Promise<File>>(),
+    backend: localBackend,
   };
 }
 

package/src/react/renderers/captions.ts

@@ -243,8 +243,8 @@ export async function renderCaptions(
       srtContent = await Bun.file(props.src).text();
       srtPath = props.src;
     } else if (props.src.type === "speech") {
-      const speechResult = await renderSpeech(props.src, ctx);
-      audioPath = speechResult.path;
+      const speechFile = await renderSpeech(props.src, ctx);
+      audioPath = await ctx.backend.resolvePath(speechFile);
 
       const transcribeTaskId = ctx.progress
         ? addTask(ctx.progress, "transcribe", "groq-whisper")
@@ -252,7 +252,10 @@ export async function renderCaptions(
       if (transcribeTaskId && ctx.progress)
         startTask(ctx.progress, transcribeTaskId);
 
-      const audioData = await Bun.file(speechResult.path).arrayBuffer();
+      const audioData =
+        audioPath.startsWith("http://") || audioPath.startsWith("https://")
+          ? await fetch(audioPath).then((res) => res.arrayBuffer())
+          : await Bun.file(audioPath).arrayBuffer();
 
       const result = await transcribe({
         model: groq.transcription("whisper-large-v3"),