vargai 0.4.0-alpha4 → 0.4.0-alpha40

package/src/ai-sdk/providers/CONTRIBUTING.md

@@ -0,0 +1,457 @@
+# Adding Models & Providers
+
+This guide explains how to add new AI models and providers to the varg SDK.
+
+## Overview
+
+Providers in varg extend the [Vercel AI SDK](https://sdk.vercel.ai/) with additional model types for video, music, and other media generation. Each provider implements a consistent interface pattern.
+
+## Architecture
+
+```
+src/ai-sdk/providers/
+├── fal.ts           # Full provider (video, image, transcription)
+├── elevenlabs.ts    # Speech & music provider
+├── openai.ts        # Extends @ai-sdk/openai with video
+├── google.ts        # Image & video provider
+├── higgsfield.ts    # Image-only provider
+├── replicate.ts     # Re-exports @ai-sdk/replicate
+└── CONTRIBUTING.md  # This file
+```
+
+## Model Types
+
+| Type | Interface | Use Case |
+|------|-----------|----------|
+| `VideoModelV3` | `../video-model.ts` | Video generation (t2v, i2v, lipsync) |
+| `ImageModelV3` | `@ai-sdk/provider` | Image generation |
+| `SpeechModelV3` | `@ai-sdk/provider` | Text-to-speech |
+| `MusicModelV3` | `../music-model.ts` | Music generation |
+| `TranscriptionModelV3` | `@ai-sdk/provider` | Speech-to-text |
+| `LanguageModelV3` | `@ai-sdk/provider` | LLM text generation |
+| `EmbeddingModelV3` | `@ai-sdk/provider` | Text embeddings |
+
+## Adding a New Model to an Existing Provider
+
+### Example: Adding a new video model to fal.ts
+
+1. **Add to the model mapping:**
+
+```typescript
+const VIDEO_MODELS: Record<string, { t2v: string; i2v: string }> = {
+  // existing models...
+  "new-model-v1": {
+    t2v: "fal-ai/new-model/text-to-video",
+    i2v: "fal-ai/new-model/image-to-video",
+  },
+};
+```
+
+2. **That's it!** The existing `FalVideoModel` class handles the rest.
+
+### Example: Adding a model with special handling
+
+If the new model needs custom logic, add conditional handling in `doGenerate()`:
+
+```typescript
+async doGenerate(options: VideoModelV3CallOptions) {
+  const isNewModel = this.modelId === "new-model-v1";
+  
+  if (isNewModel) {
+    // Custom input handling for this model
+    input.special_param = options.providerOptions?.fal?.specialParam;
+  }
+  
+  // ... rest of generation logic
+}
+```
+
+## Creating a New Provider
+
+### Step 1: Define the Provider Interface
+
+```typescript
+import {
+  type EmbeddingModelV3,
+  type ImageModelV3,
+  type LanguageModelV3,
+  NoSuchModelError,
+  type ProviderV3,
+} from "@ai-sdk/provider";
+import type { VideoModelV3 } from "../video-model";
+
+export interface MyProviderSettings {
+  apiKey?: string;
+  baseURL?: string;
+}
+
+export interface MyProvider extends ProviderV3 {
+  // Add methods for each model type you support
+  videoModel(modelId: string): VideoModelV3;
+  imageModel(modelId: string): ImageModelV3;
+}
+```
+
+### Step 2: Implement Model Classes
+
+Each model class must implement the corresponding interface:
+
+```typescript
+class MyVideoModel implements VideoModelV3 {
+  readonly specificationVersion = "v3" as const;
+  readonly provider = "myprovider";
+  readonly modelId: string;
+  readonly maxVideosPerCall = 1;
+
+  private apiKey: string;
+
+  constructor(modelId: string, options: { apiKey?: string } = {}) {
+    this.modelId = modelId;
+    this.apiKey = options.apiKey ?? process.env.MY_PROVIDER_API_KEY ?? "";
+  }
+
+  async doGenerate(options: VideoModelV3CallOptions) {
+    const {
+      prompt,
+      duration,
+      aspectRatio,
+      files,
+      providerOptions,
+      abortSignal,
+    } = options;
+    
+    const warnings: SharedV3Warning[] = [];
+
+    // 1. Build API request
+    const input: Record<string, unknown> = {
+      prompt,
+      duration: duration ?? 5,
+      ...(providerOptions?.myprovider ?? {}),
+    };
+
+    // 2. Handle file inputs (for image-to-video, etc.)
+    if (files && files.length > 0) {
+      const imageFile = files.find(f => 
+        f.type === "file" 
+          ? f.mediaType?.startsWith("image/")
+          : /\.(jpg|jpeg|png|webp)$/i.test(f.url)
+      );
+      if (imageFile) {
+        input.image_url = await this.uploadFile(imageFile);
+      }
+    }
+
+    // 3. Call the API
+    const response = await fetch("https://api.myprovider.com/v1/generate", {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${this.apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(input),
+      signal: abortSignal,
+    });
+
+    if (!response.ok) {
+      throw new Error(`API error: ${await response.text()}`);
+    }
+
+    const data = await response.json();
+
+    // 4. Download the result
+    const videoResponse = await fetch(data.video_url, { signal: abortSignal });
+    const videoBuffer = new Uint8Array(await videoResponse.arrayBuffer());
+
+    // 5. Return in standard format
+    return {
+      videos: [videoBuffer],
+      warnings,
+      response: {
+        timestamp: new Date(),
+        modelId: this.modelId,
+        headers: undefined,
+      },
+    };
+  }
+
+  private async uploadFile(file: ImageModelV3File): Promise<string> {
+    // Implementation depends on provider's upload mechanism
+  }
+}
+```
+
+### Step 3: Create the Provider Factory
+
+```typescript
+export function createMyProvider(
+  settings: MyProviderSettings = {},
+): MyProvider {
+  const apiKey = settings.apiKey ?? process.env.MY_PROVIDER_API_KEY;
+  
+  if (!apiKey) {
+    throw new Error("MY_PROVIDER_API_KEY not set");
+  }
+
+  return {
+    specificationVersion: "v3",
+    
+    videoModel(modelId: string): VideoModelV3 {
+      return new MyVideoModel(modelId, { apiKey });
+    },
+    
+    imageModel(modelId: string): ImageModelV3 {
+      return new MyImageModel(modelId, { apiKey });
+    },
+    
+    // Throw NoSuchModelError for unsupported model types
+    languageModel(modelId: string): LanguageModelV3 {
+      throw new NoSuchModelError({ modelId, modelType: "languageModel" });
+    },
+    
+    embeddingModel(modelId: string): EmbeddingModelV3 {
+      throw new NoSuchModelError({ modelId, modelType: "embeddingModel" });
+    },
+  };
+}
+```
+
+### Step 4: Export a Lazy Singleton
+
+```typescript
+// Lazy initialization - only creates client when first accessed
+let _myprovider: MyProvider | undefined;
+
+export const myprovider = new Proxy({} as MyProvider, {
+  get(_, prop) {
+    if (!_myprovider) {
+      _myprovider = createMyProvider();
+    }
+    return _myprovider[prop as keyof MyProvider];
+  },
+});
+```
+
+### Step 5: Re-export from index
+
+Add to `src/ai-sdk/index.ts`:
+
+```typescript
+export { createMyProvider, myprovider } from "./providers/myprovider";
+export type { MyProvider, MyProviderSettings } from "./providers/myprovider";
+```
+
+## Handling Warnings
+
+Use warnings to communicate unsupported features without failing:
+
+```typescript
+if (options.seed !== undefined) {
+  warnings.push({
+    type: "unsupported",
+    feature: "seed",
+    details: "Seed is not supported by this model",
+  });
+}
+
+if (options.fps !== undefined) {
+  warnings.push({
+    type: "unsupported",
+    feature: "fps",
+    details: "FPS is not configurable, using provider default",
+  });
+}
+```
+
+## Provider Options Passthrough
+
+Allow provider-specific options via `providerOptions`:
+
+```typescript
+// User code:
+await generateVideo({
+  model: myprovider.videoModel("model-v1"),
+  prompt: "a cat",
+  providerOptions: {
+    myprovider: {
+      customParam: "value",
+      negativePrompt: "blurry",
+    },
+  },
+});
+
+// In your model:
+const customOptions = providerOptions?.myprovider ?? {};
+input.custom_param = customOptions.customParam;
+input.negative_prompt = customOptions.negativePrompt;
+```
+
+## Async Job Polling
+
+Many video APIs are async. Here's the standard polling pattern:
+
+```typescript
+async doGenerate(options: VideoModelV3CallOptions) {
+  // 1. Create job
+  const createResponse = await fetch(`${this.baseURL}/jobs`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${this.apiKey}` },
+    body: JSON.stringify(input),
+    signal: options.abortSignal,
+  });
+  
+  const job = await createResponse.json();
+  
+  // 2. Poll for completion
+  let status = job.status;
+  while (status === "queued" || status === "processing") {
+    await new Promise(resolve => setTimeout(resolve, 2000));
+    
+    const statusResponse = await fetch(`${this.baseURL}/jobs/${job.id}`, {
+      headers: { Authorization: `Bearer ${this.apiKey}` },
+      signal: options.abortSignal,
+    });
+    
+    const statusData = await statusResponse.json();
+    status = statusData.status;
+  }
+  
+  if (status === "failed") {
+    throw new Error(`Generation failed: ${job.error}`);
+  }
+  
+  // 3. Download result
+  const videoResponse = await fetch(job.output_url);
+  return { videos: [new Uint8Array(await videoResponse.arrayBuffer())] };
+}
+```
+
+## File Upload Helpers
+
+Common pattern for handling file inputs:
+
+```typescript
+import type { ImageModelV3File } from "@ai-sdk/provider";
+
+async function fileToUrl(file: ImageModelV3File): Promise<string> {
+  if (file.type === "url") {
+    return file.url;
+  }
+  
+  // Convert base64/Uint8Array to upload
+  const bytes = typeof file.data === "string"
+    ? Uint8Array.from(atob(file.data), c => c.charCodeAt(0))
+    : file.data;
+    
+  const blob = new Blob([bytes], { type: file.mediaType ?? "image/png" });
+  
+  // Upload to provider's storage (or use data URL for small files)
+  return await uploadToStorage(blob);
+}
+
+function getMediaType(file: ImageModelV3File): string | undefined {
+  if (file.type === "file") return file.mediaType;
+  
+  const ext = file.url.split(".").pop()?.toLowerCase();
+  const mimeTypes: Record<string, string> = {
+    png: "image/png",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    mp4: "video/mp4",
+  };
+  return mimeTypes[ext ?? ""];
+}
+```
+
+## Extending Existing Providers
+
+To add video support to an existing AI SDK provider (like OpenAI):
+
+```typescript
+import {
+  createOpenAI as createOpenAIBase,
+  type OpenAIProvider as OpenAIProviderBase,
+} from "@ai-sdk/openai";
+
+// Extend the base provider interface
+export interface OpenAIProvider extends OpenAIProviderBase {
+  videoModel(modelId: string): VideoModelV3;
+}
+
+export function createOpenAI(settings = {}): OpenAIProvider {
+  const base = createOpenAIBase(settings);
+  
+  // Create callable function with all base methods
+  const provider = ((modelId: string) => base(modelId)) as OpenAIProvider;
+  Object.assign(provider, base);
+  
+  // Add video support
+  provider.videoModel = (modelId: string): VideoModelV3 =>
+    new OpenAIVideoModel(modelId, settings);
+  
+  return provider;
+}
+```
+
+## Re-exporting External Providers
+
+For providers that work as-is from `@ai-sdk/*`:
+
+```typescript
+// replicate.ts - simple re-export
+export {
+  createReplicate,
+  replicate,
+  type ReplicateProvider,
+  type ReplicateProviderSettings,
+} from "@ai-sdk/replicate";
+```
+
+## Testing Your Provider
+
+```typescript
+import { describe, test, expect } from "bun:test";
+import { createMyProvider } from "./myprovider";
+
+describe("MyProvider", () => {
+  test("creates video model", () => {
+    const provider = createMyProvider({ apiKey: "test-key" });
+    const model = provider.videoModel("model-v1");
+    
+    expect(model.provider).toBe("myprovider");
+    expect(model.modelId).toBe("model-v1");
+    expect(model.specificationVersion).toBe("v3");
+  });
+  
+  test("throws on missing api key", () => {
+    delete process.env.MY_PROVIDER_API_KEY;
+    expect(() => createMyProvider()).toThrow("MY_PROVIDER_API_KEY not set");
+  });
+});
+```
+
+## Checklist for New Providers
+
+- [ ] Implements `ProviderV3` interface
+- [ ] Model classes implement correct `*ModelV3` interfaces
+- [ ] `specificationVersion` is `"v3"`
+- [ ] Factory function `createProvider(settings)`
+- [ ] Lazy singleton export for convenience
+- [ ] API key from settings OR environment variable
+- [ ] `NoSuchModelError` for unsupported model types
+- [ ] Warnings for unsupported features (don't fail silently)
+- [ ] `providerOptions` passthrough for provider-specific params
+- [ ] `abortSignal` support for cancellation
+- [ ] Proper error handling with descriptive messages
+- [ ] Re-exported from `src/ai-sdk/index.ts`
+- [ ] Environment variable documented in README
+
+## Questions?
+
+Check existing providers for reference implementations:
+- **Full provider**: `fal.ts` (video, image, transcription)
+- **Audio provider**: `elevenlabs.ts` (speech, music)
+- **Extended provider**: `openai.ts` (adds video to base)
+- **Simple provider**: `higgsfield.ts` (image only)
+- **Re-export**: `replicate.ts`

package/src/ai-sdk/providers/editly/backends/index.ts

@@ -0,0 +1,8 @@
+export { LocalBackend, localBackend } from "./local";
+export type {
+  FFmpegBackend,
+  FFmpegInput,
+  FFmpegRunOptions,
+  FFmpegRunResult,
+  VideoInfo,
+} from "./types";

package/src/ai-sdk/providers/editly/backends/local.ts

@@ -0,0 +1,94 @@
+import { $ } from "bun";
+import type {
+  FFmpegBackend,
+  FFmpegInput,
+  FFmpegRunOptions,
+  FFmpegRunResult,
+  VideoInfo,
+} from "./types";
+
+export class LocalBackend implements FFmpegBackend {
+  readonly name = "local";
+
+  async ffprobe(input: string): Promise<VideoInfo> {
+    const result =
+      await $`ffprobe -v error -show_entries stream=width,height,r_frame_rate,codec_type -show_entries format=duration -of json ${input}`.json();
+
+    const videoStream = result.streams?.find(
+      (s: { codec_type: string }) => s.codec_type === "video",
+    );
+    const parsedDuration = parseFloat(result.format?.duration ?? "0");
+    const duration = Number.isFinite(parsedDuration) ? parsedDuration : 0;
+
+    let fps: number | undefined;
+    const framerateStr: string | undefined = videoStream?.r_frame_rate;
+    if (framerateStr) {
+      const parts = framerateStr.split("/").map(Number);
+      const num = parts[0];
+      const den = parts[1];
+      if (den && den > 0 && num) fps = num / den;
+    }
+
+    return {
+      duration,
+      width: videoStream?.width,
+      height: videoStream?.height,
+      fps,
+      framerateStr,
+    };
+  }
+
+  private buildInputArgs(inputs: FFmpegInput[]): string[] {
+    const args: string[] = [];
+    for (const input of inputs) {
+      if (typeof input === "string") {
+        args.push("-i", input);
+      } else if ("raw" in input) {
+        args.push(...input.raw);
+      } else {
+        if (input.options) args.push(...input.options);
+        args.push("-i", input.path);
+      }
+    }
+    return args;
+  }
+
+  async run(options: FFmpegRunOptions): Promise<FFmpegRunResult> {
+    const {
+      inputs,
+      filterComplex,
+      videoFilter,
+      outputArgs = [],
+      outputPath,
+      verbose,
+    } = options;
+
+    const inputArgs = this.buildInputArgs(inputs);
+
+    const ffmpegArgs = [
+      "-hide_banner",
+      "-loglevel",
+      verbose ? "info" : "error",
+      ...inputArgs,
+      ...(filterComplex ? ["-filter_complex", filterComplex] : []),
+      ...(videoFilter ? ["-vf", videoFilter] : []),
+      ...outputArgs,
+      "-y",
+      outputPath,
+    ];
+
+    if (verbose) {
+      console.log("ffmpeg", ffmpegArgs.join(" "));
+    }
+
+    const result = await $`ffmpeg ${ffmpegArgs}`.quiet();
+
+    if (result.exitCode !== 0) {
+      throw new Error(`ffmpeg failed with exit code ${result.exitCode}`);
+    }
+
+    return { output: { type: "file", path: outputPath } };
+  }
+}
+
+export const localBackend = new LocalBackend();

package/src/ai-sdk/providers/editly/backends/types.ts

@@ -0,0 +1,74 @@
+/**
+ * FFmpeg backend abstraction for dependency injection
+ * Allows switching between local ffmpeg and cloud services like Rendi
+ */
+
+import type { VideoInfo } from "../types";
+
+/**
+ * Represents the result of running ffprobe
+ */
+export type { VideoInfo };
+
+/**
+ * Represents an input to ffmpeg - can be a simple path/URL or structured with options
+ */
+export type FFmpegInput =
+  | string
+  | {
+      /** Path or URL to the input file */
+      path: string;
+      /** Options to apply BEFORE the -i flag (e.g. -ss 5 for seeking) */
+      options?: string[];
+    }
+  | {
+      /** Raw ffmpeg args that don't use -i (e.g. ["-f", "lavfi", "-i", "color=black"]) */
+      raw: string[];
+    };
+
+/**
+ * FFmpeg execution options - new interface where backend builds -i flags
+ */
+export interface FFmpegRunOptions {
+  /** Inputs - backend builds -i flags from these */
+  inputs: FFmpegInput[];
+  /** Filter complex string (uses input indices like [0:v], [1:a]) */
+  filterComplex?: string;
+  /** Video filter string for single-input operations */
+  videoFilter?: string;
+  /** Arguments after inputs but before output (codec, map, etc) */
+  outputArgs?: string[];
+  /** Output file path */
+  outputPath: string;
+  /** Enable verbose logging */
+  verbose?: boolean;
+}
+
+export type FFmpegOutput =
+  | { type: "file"; path: string }
+  | { type: "url"; url: string };
+
+export interface FFmpegRunResult {
+  output: FFmpegOutput;
+}
+
+/**
+ * Backend interface for ffmpeg/ffprobe execution
+ */
+export interface FFmpegBackend {
+  /** Backend name for identification */
+  readonly name: string;
+
+  /**
+   * Run ffprobe to get media file info
+   * @param input - File path (local) or URL
+   */
+  ffprobe(input: string): Promise<VideoInfo>;
+
+  /**
+   * Run ffmpeg command
+   * @param options - Execution options including args, inputs, and output path
+   * @returns Result with optional URL for cloud backends
+   */
+  run(options: FFmpegRunOptions): Promise<FFmpegRunResult>;
+}

package/src/ai-sdk/providers/editly/editly.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import { existsSync, unlinkSync } from "node:fs";
-import { ffprobe } from "./ffmpeg";
+import { localBackend } from "./backends/local";
 import { editly } from "./index";
 
 const VIDEO_1 = "output/sora-landscape.mp4";
@@ -9,6 +9,8 @@ const VIDEO_TALKING = "output/workflow-talking-synced.mp4";
 const IMAGE_SQUARE = "media/replicate-forest.png";
 const IMAGE_PORTRAIT = "media/madi-portrait.png";
 
+const ffprobe = localBackend.ffprobe;
+
 describe("editly", () => {
   test("requires outPath", async () => {
     await expect(
@@ -1105,4 +1107,50 @@ describe("editly", () => {
     expect(info.height).toBe(1920);
     expect(info.duration).toBeCloseTo(3, 0);
   });
+
+  test("video overlay with cropPosition", async () => {
+    const outPath = "output/editly-test-crop-position.mp4";
+    if (existsSync(outPath)) unlinkSync(outPath);
+
+    await editly({
+      outPath,
+      width: 1080,
+      height: 1920,
+      fps: 30,
+      clips: [
+        {
+          duration: 3,
+          layers: [
+            { type: "fill-color", color: "#000000" },
+            {
+              type: "video",
+              path: VIDEO_1,
+              width: 1080,
+              height: 960,
+              left: 0,
+              top: 0,
+              resizeMode: "cover",
+              cropPosition: "top",
+            },
+            {
+              type: "video",
+              path: VIDEO_2,
+              width: 1080,
+              height: 960,
+              left: 0,
+              top: 960,
+              resizeMode: "cover",
+              cropPosition: "bottom",
+            },
+          ],
+        },
+      ],
+    });
+
+    expect(existsSync(outPath)).toBe(true);
+    const info = await ffprobe(outPath);
+    expect(info.width).toBe(1080);
+    expect(info.height).toBe(1920);
+    expect(info.duration).toBeCloseTo(3, 0);
+  });
 });