varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/src/ai-sdk/providers/editly/plan.md

@@ -0,0 +1,144 @@
+# editly implementation plan
+
+## goal
+
+match the original editly interface/types so users can use the same config format, but implement everything with pure ffmpeg (no headless-gl, no Fabric.js, no canvas dependencies).
+
+## process
+
+1. implement feature
+2. create test that generates output video
+3. tell user what to look for in the output (specific visual behavior to verify)
+4. wait for user confirmation before moving to next feature
+
+## instructions
+
+implement features one by one. check with user and show examples before moving on to the next feature.
+
+---
+
+## layer types
+
+| layer | status | notes |
+|-------|--------|-------|
+| video | ✅ done | originX/originY, cutFrom/cutTo, contain-blur |
+| image | ✅ done | zoomDirection in/out/left/right, contain-blur |
+| image-overlay | ✅ done | position presets, PositionObject, Ken Burns zoom/pan |
+| title | ✅ done | fontPath, fontFamily, start/stop timing |
+| subtitle | ✅ done | centered bottom, background box, start/stop timing |
+| title-background | ✅ done | title with gradient/color background |
+| news-title | ✅ done | colored bar with text, top/bottom, start/stop timing |
+| slide-in-text | ✅ done | animated text sliding from left, start/stop timing |
+| fill-color | ✅ done | |
+| pause | ✅ done | aliased to fill-color |
+| radial-gradient | ✅ done | |
+| linear-gradient | ✅ done | |
+| rainbow-colors | ✅ done | animated hue rotation |
+| audio | ✅ done | works as clip layer with cutFrom/cutTo/mixVolume |
+| detached-audio | ✅ done | clip-relative timing with start offset |
+| canvas | 🚫 skip | requires dependencies |
+| fabric | 🚫 skip | requires dependencies |
+| gl | 🚫 skip | requires dependencies |
+
+## transitions
+
+| feature | status | notes |
+|---------|--------|-------|
+| ffmpeg xfade | ✅ working | ~40 transitions available |
+| gl-transitions | 🚫 skip | requires headless-gl |
+| audio crossfade | ✅ done | afade in/out during transitions |
+
+## audio
+
+| feature | status | notes |
+|---------|--------|-------|
+| audioFilePath | ✅ done | |
+| audioTracks | ✅ done | cutFrom/cutTo/start/mixVolume |
+| loopAudio | ✅ done | loops background audio to match video duration |
+| keepSourceAudio | ✅ done | extracts audio from video clips, syncs with cutFrom |
+| clipsAudioVolume | ✅ done | controls volume of source video audio |
+| outputVolume | ✅ done | |
+| audioNorm | ✅ done | dynaudnorm filter with gaussSize/maxGain |
+
+## config
+
+| option | status | notes |
+|--------|--------|-------|
+| defaults.layer | ✅ done | applies common props to all layers |
+| defaults.layerType | ✅ done | applies type-specific defaults |
+| layer start/stop timing | ✅ done | enable expression for text layers |
+| contain-blur resize | ✅ done | blurred background instead of black bars |
+| custom fonts | ✅ done | fontPath and fontFamily support |
+
+---
+
+## implementation order
+
+### phase 1: complete existing layers
+1. [x] video: add originX/originY ✅
+2. [x] image: add zoomDirection left/right ✅
+3. [x] title: add fontPath/fontFamily support ✅
+
+### phase 2: missing layer types
+4. [x] image-overlay layer ✅
+5. [x] subtitle layer ✅
+6. [x] title-background layer ✅
+7. [x] rainbow-colors layer ✅
+8. [x] news-title layer ✅
+9. [x] slide-in-text layer ✅
+
+### phase 3: audio features
+10. [x] audio layer (as clip layer, not just audioTracks) ✅
+11. [x] detached-audio layer ✅
+12. [x] loopAudio ✅
+13. [x] keepSourceAudio ✅
+14. [x] clipsAudioVolume ✅
+15. [x] audioNorm ✅
+16. [x] audioTracks cutFrom/cutTo/start ✅
+
+### phase 4: advanced features
+17. [x] layer start/stop timing ✅
+18. [x] contain-blur resize mode ✅
+19. [x] defaults.layer / defaults.layerType ✅
+20. [x] audio crossfade during transitions ✅
+
+---
+
+## ffmpeg xfade transitions (available)
+
+these work out of the box:
+- fade, fadeblack, fadewhite, fadeslow
+- wipeleft, wiperight, wipeup, wipedown
+- slideleft, slideright, slideup, slidedown
+- circlecrop, rectcrop
+- distance, fadegrayscale
+- hblur, pixelize, diagtl, diagtr, diagbl, diagbr
+- hlslice, hrslice, vuslice, vdslice
+- dissolve, radial, smoothleft, smoothright, smoothup, smoothdown
+- circleopen, circleclose, vertopen, vertclose, horzopen, horzclose
+- squeezev, squeezeh, zoomin, hlwind, hrwind, vuwind, vdwind
+- coverleft, coverright, coverup, coverdown
+- revealleft, revealright, revealup, revealdown
+
+---
+
+## key differences from original editly
+
+### continuous video overlays
+
+when the same video file is used as an overlay (with `left`/`top`/`width`/`height` positioning) across multiple clips, our implementation automatically makes it continuous:
+
+1. `collectVideoOverlays()` groups overlay videos by path
+2. calculates `totalDuration` across all clips
+3. uses a single ffmpeg input stream
+4. overlays it on the final composited video
+
+this means if you add `{ type: "video", path: "pip.mp4", left: 0.73, top: 0.73, width: 0.25, height: 0.25 }` to each clip, the video plays continuously across the timeline (not restarting per clip).
+
+**limitation**: the overlay video must be long enough to cover the total timeline. if your timeline is 12s but the overlay video is 5s, it stops at 5s.
+
+---
+
+## progress log
+
+<!-- append progress here as we implement -->

package/src/ai-sdk/providers/editly/types.ts

@@ -0,0 +1,328 @@
+// Types from original editly (https://github.com/mifi/editly)
+// Adapted for pure ffmpeg implementation (no fabric/canvas/gl dependencies)
+
+export type OriginX = "left" | "center" | "right";
+export type OriginY = "top" | "center" | "bottom";
+export type SizeValue = number | `${number}%` | `${number}px`;
+
+/**
+ * How to fit image to screen. Can be one of:
+ * - `'contain'` - All the video will be contained within the frame and letterboxed.
+ * - `'contain-blur'` - Like contain, but with a blurred copy as the letterbox.
+ * - `'cover'` - Video be cropped to cover the whole screen (aspect ratio preserved).
+ * - `'stretch'` - Video will be stretched to cover the whole screen (aspect ratio ignored).
+ *
+ * @default 'contain-blur'
+ */
+export type ResizeMode = "contain" | "contain-blur" | "cover" | "stretch";
+
+export interface PositionObject {
+  x: SizeValue;
+  y: SizeValue;
+  originX?: OriginX;
+  originY?: OriginY;
+}
+
+/**
+ * Certain layers support the position parameter.
+ */
+export type Position =
+  | "top"
+  | "top-left"
+  | "top-right"
+  | "center"
+  | "center-left"
+  | "center-right"
+  | "bottom"
+  | "bottom-left"
+  | "bottom-right"
+  | PositionObject;
+
+/**
+ * Arbitrary audio tracks.
+ */
+export interface AudioTrack {
+  path: string;
+  mixVolume?: number | string;
+  cutFrom?: number;
+  cutTo?: number;
+  start?: number;
+}
+
+/**
+ * Ken Burns parameters.
+ */
+export interface KenBurns {
+  zoomDirection?: "in" | "out" | "left" | "right" | null;
+  zoomAmount?: number;
+}
+
+export type LayerType =
+  | "video"
+  | "audio"
+  | "detached-audio"
+  | "image"
+  | "image-overlay"
+  | "title"
+  | "subtitle"
+  | "title-background"
+  | "news-title"
+  | "slide-in-text"
+  | "fill-color"
+  | "pause"
+  | "radial-gradient"
+  | "linear-gradient"
+  | "rainbow-colors";
+
+export interface BaseLayer {
+  type: LayerType;
+  start?: number;
+  stop?: number;
+}
+
+export interface TextLayer extends BaseLayer {
+  text: string;
+  textColor?: string;
+  fontPath?: string;
+  fontFamily?: string;
+}
+
+/**
+ * For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`.
+ * If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`.
+ */
+export interface VideoLayer extends BaseLayer {
+  type: "video";
+  path: string;
+  resizeMode?: ResizeMode;
+  cutFrom?: number;
+  cutTo?: number;
+  width?: SizeValue;
+  height?: SizeValue;
+  left?: SizeValue;
+  top?: SizeValue;
+  originX?: OriginX;
+  originY?: OriginY;
+  mixVolume?: number | string;
+}
+
+/**
+ * Audio layers will be mixed together.
+ */
+export interface AudioLayer extends BaseLayer {
+  type: "audio";
+  path: string;
+  cutFrom?: number;
+  cutTo?: number;
+  mixVolume?: number | string;
+}
+
+/**
+ * Detached audio - like audioTracks but start time is relative to clip's start.
+ */
+export interface DetachedAudioLayer extends BaseLayer, AudioTrack {
+  type: "detached-audio";
+}
+
+/**
+ * Full screen image.
+ */
+export interface ImageLayer extends BaseLayer, KenBurns {
+  type: "image";
+  path: string;
+  resizeMode?: ResizeMode;
+  duration?: number;
+}
+
+/**
+ * Image overlay with a custom position and size on the screen.
+ */
+export interface ImageOverlayLayer extends BaseLayer, KenBurns {
+  type: "image-overlay";
+  path: string;
+  position?: Position;
+  width?: SizeValue;
+  height?: SizeValue;
+}
+
+export interface TitleLayer extends TextLayer, KenBurns {
+  type: "title";
+  position?: Position;
+}
+
+export interface SubtitleLayer extends TextLayer {
+  type: "subtitle";
+  backgroundColor?: string;
+}
+
+/**
+ * Title with background.
+ */
+export interface TitleBackgroundLayer extends TextLayer {
+  type: "title-background";
+  background?: BackgroundLayer;
+}
+
+export interface NewsTitleLayer extends TextLayer {
+  type: "news-title";
+  backgroundColor?: string;
+  position?: Position;
+}
+
+export interface SlideInTextLayer extends TextLayer {
+  type: "slide-in-text";
+  fontSize?: number;
+  charSpacing?: number;
+  color?: string;
+  position?: Position;
+}
+
+export interface FillColorLayer extends BaseLayer {
+  type: "fill-color";
+  color?: string;
+}
+
+export interface PauseLayer extends BaseLayer {
+  type: "pause";
+  color?: string;
+}
+
+export interface RadialGradientLayer extends BaseLayer {
+  type: "radial-gradient";
+  colors?: [string, string];
+}
+
+export interface LinearGradientLayer extends BaseLayer {
+  type: "linear-gradient";
+  colors?: [string, string];
+}
+
+export interface RainbowColorsLayer extends BaseLayer {
+  type: "rainbow-colors";
+}
+
+export type Layer =
+  | VideoLayer
+  | AudioLayer
+  | DetachedAudioLayer
+  | ImageLayer
+  | ImageOverlayLayer
+  | TitleLayer
+  | SubtitleLayer
+  | TitleBackgroundLayer
+  | NewsTitleLayer
+  | SlideInTextLayer
+  | FillColorLayer
+  | PauseLayer
+  | RadialGradientLayer
+  | LinearGradientLayer
+  | RainbowColorsLayer;
+
+/**
+ * Special layers that can be used in the 'title-background' layer.
+ */
+export type BackgroundLayer =
+  | RadialGradientLayer
+  | LinearGradientLayer
+  | FillColorLayer;
+
+/**
+ * Curve types for audio fades.
+ * @see https://trac.ffmpeg.org/wiki/AfadeCurves
+ */
+export type CurveType =
+  | "tri"
+  | "qsin"
+  | "hsin"
+  | "esin"
+  | "log"
+  | "ipar"
+  | "qua"
+  | "cub"
+  | "squ"
+  | "cbr"
+  | "par"
+  | "exp"
+  | "iqsin"
+  | "ihsin"
+  | "dese"
+  | "desi"
+  | "losi"
+  | "nofade";
+
+export interface TransitionOptions {
+  duration?: number;
+  name?: string;
+  audioOutCurve?: CurveType;
+  audioInCurve?: CurveType;
+}
+
+export interface Clip {
+  layers: Layer[];
+  duration?: number;
+  transition?: TransitionOptions | null;
+}
+
+export interface DefaultLayerOptions {
+  fontPath?: string;
+  [key: string]: unknown;
+}
+
+export type DefaultLayerTypeOptions = {
+  [P in LayerType]?: Partial<Omit<Extract<Layer, { type: P }>, "type">>;
+};
+
+export interface DefaultOptions {
+  duration?: number;
+  layer?: DefaultLayerOptions;
+  layerType?: DefaultLayerTypeOptions;
+  transition?: TransitionOptions | null;
+}
+
+/**
+ * Audio normalization options.
+ * @see https://ffmpeg.org/ffmpeg-filters.html#dynaudnorm
+ */
+export interface AudioNormalizationOptions {
+  enable?: boolean;
+  gaussSize?: number;
+  maxGain?: number;
+}
+
+export interface EditlyConfig {
+  outPath: string;
+  clips: Clip[];
+  width?: number;
+  height?: number;
+  fps?: number;
+  customOutputArgs?: string[];
+  allowRemoteRequests?: boolean;
+  fast?: boolean;
+  defaults?: DefaultOptions;
+  audioTracks?: AudioTrack[];
+  audioFilePath?: string;
+  backgroundAudioVolume?: string | number;
+  loopAudio?: boolean;
+  keepSourceAudio?: boolean;
+  clipsAudioVolume?: number | string;
+  outputVolume?: number | string;
+  audioNorm?: AudioNormalizationOptions;
+  verbose?: boolean;
+  enableFfmpegLog?: boolean;
+}
+
+// Internal types used by our implementation
+export interface VideoInfo {
+  duration: number;
+  width?: number;
+  height?: number;
+  fps?: number;
+  framerateStr?: string;
+  rotation?: number;
+}
+
+export interface ProcessedClip {
+  layers: Layer[];
+  duration: number;
+  transition: Required<TransitionOptions>;
+}

package/src/ai-sdk/providers/elevenlabs-provider.ts

@@ -0,0 +1,255 @@
+import {
+  type EmbeddingModelV3,
+  type ImageModelV3,
+  type LanguageModelV3,
+  NoSuchModelError,
+  type ProviderV3,
+  type SharedV3Warning,
+  type SpeechModelV3,
+  type SpeechModelV3CallOptions,
+} from "@ai-sdk/provider";
+import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js";
+import type { MusicModelV3, MusicModelV3CallOptions } from "../music-model";
+
+const VOICES: Record<string, string> = {
+  rachel: "21m00Tcm4TlvDq8ikWAM",
+  domi: "AZnzlk1XvdvUeBnXmlld",
+  bella: "EXAVITQu4vr4xnSDxMaL",
+  antoni: "ErXwobaYiN019PkySvjV",
+  elli: "MF3mGyEYCl7XYWbV9V6O",
+  josh: "TxGEqnHWrfWFTfGW9XjX",
+  arnold: "VR6AewLTigWG4xSOukaG",
+  adam: "pNInz6obpgDQGcFmaJgB",
+  sam: "yoZ06aMxZJJ28mfd3POQ",
+};
+
+const TTS_MODELS: Record<string, string> = {
+  eleven_multilingual_v2: "eleven_multilingual_v2",
+  eleven_turbo_v2: "eleven_turbo_v2",
+  eleven_monolingual_v1: "eleven_monolingual_v1",
+  multilingual_v2: "eleven_multilingual_v2",
+  turbo_v2: "eleven_turbo_v2",
+  turbo: "eleven_turbo_v2",
+};
+
+function resolveVoiceId(voice: string): string {
+  const lower = voice.toLowerCase();
+  return VOICES[lower] ?? voice;
+}
+
+function resolveModelId(modelId: string): string {
+  return TTS_MODELS[modelId] ?? modelId;
+}
+
+class ElevenLabsMusicModel implements MusicModelV3 {
+  readonly specificationVersion = "v3" as const;
+  readonly provider = "elevenlabs";
+  readonly modelId: string;
+
+  private client: ElevenLabsClient;
+
+  constructor(modelId: string, client: ElevenLabsClient) {
+    this.modelId = modelId;
+    this.client = client;
+  }
+
+  async doGenerate(options: MusicModelV3CallOptions) {
+    const { prompt, duration, providerOptions } = options;
+    const warnings: SharedV3Warning[] = [];
+
+    const elevenLabsOptions = providerOptions?.elevenlabs ?? {};
+    const audio = await this.client.music.compose({
+      prompt,
+      musicLengthMs: duration ? duration * 1000 : undefined,
+      modelId: this.modelId,
+      ...elevenLabsOptions,
+    } as Parameters<typeof this.client.music.compose>[0]);
+
+    const reader = audio.getReader();
+    const chunks: Uint8Array[] = [];
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+    }
+
+    const totalLength = chunks.reduce((acc, c) => acc + c.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+      result.set(chunk, offset);
+      offset += chunk.length;
+    }
+
+    return {
+      audio: result,
+      warnings,
+      response: {
+        timestamp: new Date(),
+        modelId: this.modelId,
+        headers: undefined,
+      },
+    };
+  }
+}
+
+class ElevenLabsSpeechModel implements SpeechModelV3 {
+  readonly specificationVersion = "v3" as const;
+  readonly provider = "elevenlabs";
+  readonly modelId: string;
+
+  private client: ElevenLabsClient;
+
+  constructor(modelId: string, client: ElevenLabsClient) {
+    this.modelId = modelId;
+    this.client = client;
+  }
+
+  async doGenerate(options: SpeechModelV3CallOptions) {
+    const { text, voice, speed, providerOptions } = options;
+    const warnings: SharedV3Warning[] = [];
+
+    const voiceId = resolveVoiceId(voice ?? "rachel");
+    const model = resolveModelId(this.modelId);
+
+    if (speed !== undefined) {
+      warnings.push({
+        type: "unsupported",
+        feature: "speed",
+        details: "Speed control requires voice settings adjustment",
+      });
+    }
+
+    const elevenLabsOptions = providerOptions?.elevenlabs ?? {};
+    const audio = await this.client.textToSpeech.convert(voiceId, {
+      text,
+      modelId: model,
+      outputFormat: "mp3_44100_128",
+      ...elevenLabsOptions,
+    } as Parameters<typeof this.client.textToSpeech.convert>[1]);
+
+    const reader = audio.getReader();
+    const chunks: Uint8Array[] = [];
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+    }
+
+    const totalLength = chunks.reduce((acc, c) => acc + c.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+      result.set(chunk, offset);
+      offset += chunk.length;
+    }
+
+    return {
+      audio: result,
+      warnings,
+      response: {
+        timestamp: new Date(),
+        modelId: this.modelId,
+        headers: undefined,
+      },
+    };
+  }
+}
+
+export interface ElevenLabsProviderSettings {
+  apiKey?: string;
+}
+
+export interface ElevenLabsProvider extends ProviderV3 {
+  speechModel(modelId?: string): SpeechModelV3;
+  musicModel(modelId?: string): MusicModelV3;
+}
+
+export function createElevenLabs(
+  settings: ElevenLabsProviderSettings = {},
+): ElevenLabsProvider {
+  const apiKey = settings.apiKey ?? process.env.ELEVENLABS_API_KEY;
+  if (!apiKey) {
+    throw new Error("ELEVENLABS_API_KEY not set");
+  }
+  const client = new ElevenLabsClient({ apiKey });
+
+  return {
+    specificationVersion: "v3",
+    speechModel(modelId = "eleven_turbo_v2") {
+      return new ElevenLabsSpeechModel(modelId, client);
+    },
+    musicModel(modelId = "music_v1") {
+      return new ElevenLabsMusicModel(modelId, client);
+    },
+    languageModel(modelId: string): LanguageModelV3 {
+      throw new NoSuchModelError({ modelId, modelType: "languageModel" });
+    },
+    embeddingModel(modelId: string): EmbeddingModelV3 {
+      throw new NoSuchModelError({ modelId, modelType: "embeddingModel" });
+    },
+    imageModel(modelId: string): ImageModelV3 {
+      throw new NoSuchModelError({ modelId, modelType: "imageModel" });
+    },
+  };
+}
+
+export const elevenlabs_provider = createElevenLabs();
+export { elevenlabs_provider as elevenlabs, VOICES };
+
+export interface GenerateMusicOptions {
+  prompt: string;
+  durationSeconds?: number;
+  apiKey?: string;
+}
+
+export interface GenerateMusicResult {
+  audio: {
+    uint8Array: Uint8Array;
+    mimeType: string;
+  };
+}
+
+export async function generateMusic(
+  options: GenerateMusicOptions,
+): Promise<GenerateMusicResult> {
+  const { prompt, durationSeconds, apiKey } = options;
+  const key = apiKey ?? process.env.ELEVENLABS_API_KEY;
+  if (!key) {
+    throw new Error("ELEVENLABS_API_KEY not set");
+  }
+
+  const client = new ElevenLabsClient({ apiKey: key });
+
+  const audio = await client.music.compose({
+    prompt,
+    musicLengthMs: durationSeconds ? durationSeconds * 1000 : undefined,
+    modelId: "music_v1",
+  });
+
+  const reader = audio.getReader();
+  const chunks: Uint8Array[] = [];
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+
+  const totalLength = chunks.reduce((acc, c) => acc + c.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return {
+    audio: {
+      uint8Array: result,
+      mimeType: "audio/mpeg",
+    },
+  };
+}