vargai 0.4.0-alpha4 → 0.4.0-alpha40

package/src/ai-sdk/providers/editly/index.ts

@@ -1,5 +1,4 @@
-import { $ } from "bun";
-import { ffprobe, multipleOf2 } from "./ffmpeg";
+import { type FFmpegBackend, localBackend } from "./backends";
 import {
   getImageOverlayFilter,
   getImageOverlayPositionFilter,
@@ -18,6 +17,7 @@ import type {
   Clip,
   DetachedAudioLayer,
   EditlyConfig,
+  EditlyResult,
   ImageOverlayLayer,
   Layer,
   NewsTitleLayer,
@@ -28,6 +28,8 @@ import type {
   VideoLayer,
 } from "./types";
 
+export * from "./backends";
+export * from "./rendi";
 export * from "./types";
 
 const DEFAULT_DURATION = 4;
@@ -36,12 +38,22 @@ const DEFAULT_FPS = 30;
 const DEFAULT_WIDTH = 1280;
 const DEFAULT_HEIGHT = 720;
 
-async function getVideoDuration(path: string): Promise<number> {
-  const info = await ffprobe(path);
+function multipleOf2(n: number): number {
+  return Math.round(n / 2) * 2;
+}
+
+async function getVideoDuration(
+  path: string,
+  backend: FFmpegBackend,
+): Promise<number> {
+  const info = await backend.ffprobe(path);
   return info.duration;
 }
 
-async function getFirstVideoInfo(clips: Clip[]): Promise<{
+async function getFirstVideoInfo(
+  clips: Clip[],
+  backend: FFmpegBackend,
+): Promise<{
   width?: number;
   height?: number;
   fps?: number;
@@ -49,7 +61,7 @@ async function getFirstVideoInfo(clips: Clip[]): Promise<{
   for (const clip of clips) {
     for (const layer of clip.layers) {
       if (layer.type === "video") {
-        const info = await ffprobe((layer as VideoLayer).path);
+        const info = await backend.ffprobe((layer as VideoLayer).path);
         return { width: info.width, height: info.height, fps: info.fps };
       }
     }
@@ -72,6 +84,7 @@ function applyLayerDefaults(
 async function processClips(
   clips: Clip[],
   defaults: EditlyConfig["defaults"],
+  backend: FFmpegBackend,
 ): Promise<ProcessedClip[]> {
   const processed: ProcessedClip[] = [];
   const defaultDuration = defaults?.duration ?? DEFAULT_DURATION;
@@ -86,7 +99,7 @@ async function processClips(
     for (const layer of layers) {
       if (layer.type === "video" && !clip.duration) {
         const videoLayer = layer as VideoLayer;
-        const videoDuration = await getVideoDuration(videoLayer.path);
+        const videoDuration = await getVideoDuration(videoLayer.path, backend);
         const cutFrom = videoLayer.cutFrom ?? 0;
         const cutTo = videoLayer.cutTo ?? videoDuration;
         duration = cutTo - cutFrom;
@@ -137,6 +150,15 @@ function isOverlayLayer(layer: Layer): boolean {
   return isVideoOverlayLayer(layer) || isImageOverlayLayer(layer);
 }
 
+function isTextOverlayLayer(layer: Layer): boolean {
+  return (
+    layer.type === "title" ||
+    layer.type === "subtitle" ||
+    layer.type === "news-title" ||
+    layer.type === "slide-in-text"
+  );
+}
+
 function buildBaseClipFilter(
   clip: ProcessedClip,
   clipIndex: number,
@@ -164,7 +186,10 @@ function buildBaseClipFilter(
   let baseLabel = "";
   let inputIdx = inputOffset;
 
-  const baseLayers = clip.layers.filter((l) => l && !isOverlayLayer(l));
+  // Filter out overlay layers AND text overlay layers (text will be applied after image overlays)
+  const baseLayers = clip.layers.filter(
+    (l) => l && !isOverlayLayer(l) && !isTextOverlayLayer(l),
+  );
 
   for (let i = 0; i < baseLayers.length; i++) {
     const layer = baseLayers[i];
@@ -201,58 +226,6 @@ function buildBaseClipFilter(
         inputIdx++;
       }
     }
-
-    if (layer.type === "title") {
-      const titleFilter = getTitleFilter(
-        layer as TitleLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `title${clipIndex}_${i}`;
-      filters.push(`${titleFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
-
-    if (layer.type === "subtitle") {
-      const subtitleFilter = getSubtitleFilter(
-        layer as SubtitleLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `sub${clipIndex}_${i}`;
-      filters.push(`${subtitleFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
-
-    if (layer.type === "news-title") {
-      const newsFilter = getNewsTitleFilter(
-        layer as NewsTitleLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `news${clipIndex}_${i}`;
-      filters.push(`${newsFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
-
-    if (layer.type === "slide-in-text") {
-      const slideFilter = getSlideInTextFilter(
-        layer as SlideInTextLayer,
-        baseLabel,
-        width,
-        height,
-        clip.duration,
-      );
-      const newLabel = `slide${clipIndex}_${i}`;
-      filters.push(`${slideFilter}[${newLabel}]`);
-      baseLabel = newLabel;
-    }
   }
 
   return {
@@ -358,6 +331,41 @@ function collectAudioLayers(
   return audioLayers;
 }
 
+type TextLayer = TitleLayer | SubtitleLayer | NewsTitleLayer | SlideInTextLayer;
+
+interface TimedTextLayer {
+  layer: TextLayer;
+  startTime: number;
+  duration: number;
+}
+
+function collectTextLayers(clips: ProcessedClip[]): TimedTextLayer[] {
+  const textLayers: TimedTextLayer[] = [];
+  let currentTime = 0;
+
+  for (let i = 0; i < clips.length; i++) {
+    const clip = clips[i];
+    if (!clip) continue;
+
+    for (const layer of clip.layers) {
+      if (layer && isTextOverlayLayer(layer)) {
+        textLayers.push({
+          layer: layer as TextLayer,
+          startTime: currentTime,
+          duration: clip.duration,
+        });
+      }
+    }
+
+    currentTime += clip.duration;
+    if (i < clips.length - 1) {
+      currentTime -= clip.transition.duration;
+    }
+  }
+
+  return textLayers;
+}
+
 function buildTransitionFilter(
   fromLabel: string,
   toLabel: string,
@@ -527,7 +535,7 @@ function buildAudioFilter(
   };
 }
 
-export async function editly(config: EditlyConfig): Promise<void> {
+export async function editly(config: EditlyConfig): Promise<EditlyResult> {
   const {
     outPath,
     clips: clipsIn,
@@ -544,11 +552,17 @@ export async function editly(config: EditlyConfig): Promise<void> {
     fast,
   } = config;
 
+  const backend: FFmpegBackend = config.backend ?? localBackend;
+
+  if (verbose) {
+    console.log(`[editly] using backend: ${backend.name}`);
+  }
+
   if (!clipsIn || clipsIn.length === 0) {
     throw new Error("At least one clip is required");
   }
 
-  const firstVideoInfo = await getFirstVideoInfo(clipsIn);
+  const firstVideoInfo = await getFirstVideoInfo(clipsIn, backend);
   let width = config.width ?? firstVideoInfo.width ?? DEFAULT_WIDTH;
   let height = config.height ?? firstVideoInfo.height ?? DEFAULT_HEIGHT;
   const fps = config.fps ?? firstVideoInfo.fps ?? DEFAULT_FPS;
@@ -566,7 +580,7 @@ export async function editly(config: EditlyConfig): Promise<void> {
     console.log(`Output: ${width}x${height} @ ${fps}fps`);
   }
 
-  const clips = await processClips(clipsIn, defaults);
+  const clips = await processClips(clipsIn, defaults, backend);
 
   const continuousVideoOverlays = collectContinuousVideoOverlays(clips);
   const imageOverlays = collectImageOverlays(clips);
@@ -744,6 +758,67 @@ export async function editly(config: EditlyConfig): Promise<void> {
     finalVideoLabel = currentBase;
   }
 
+  const textLayers = collectTextLayers(clips);
+  if (textLayers.length > 0) {
+    let currentBase = finalVideoLabel;
+
+    for (let i = 0; i < textLayers.length; i++) {
+      const timedLayer = textLayers[i];
+      if (!timedLayer) continue;
+
+      const { layer, startTime, duration } = timedLayer;
+      const outputLabel = `vwithtext${i}`;
+
+      const timedLayerWithEnable = {
+        ...layer,
+        start: layer.start ?? startTime,
+        stop: layer.stop ?? startTime + duration,
+      };
+
+      if (layer.type === "title") {
+        const titleFilter = getTitleFilter(
+          timedLayerWithEnable as TitleLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${titleFilter}[${outputLabel}]`);
+      } else if (layer.type === "subtitle") {
+        const subtitleFilter = getSubtitleFilter(
+          timedLayerWithEnable as SubtitleLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${subtitleFilter}[${outputLabel}]`);
+      } else if (layer.type === "news-title") {
+        const newsFilter = getNewsTitleFilter(
+          timedLayerWithEnable as NewsTitleLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${newsFilter}[${outputLabel}]`);
+      } else if (layer.type === "slide-in-text") {
+        const slideFilter = getSlideInTextFilter(
+          timedLayerWithEnable as SlideInTextLayer,
+          currentBase,
+          width,
+          height,
+          totalDuration,
+        );
+        allFilters.push(`${slideFilter}[${outputLabel}]`);
+      }
+
+      currentBase = outputLabel;
+    }
+
+    finalVideoLabel = currentBase;
+  }
+
   const clipAudioLayers = collectAudioLayers(clips);
   const videoInputCount = allInputs.length;
   const audioFilter = buildAudioFilter(
@@ -765,10 +840,9 @@ export async function editly(config: EditlyConfig): Promise<void> {
     allFilters.push(audioFilter.filter);
   }
 
-  const inputArgs = allInputs.flatMap((input) => ["-i", input]);
   const filterComplex = allFilters.join(";");
 
-  const outputArgs = customOutputArgs ?? [
+  const codecArgs = customOutputArgs ?? [
     "-c:v",
     "libx264",
     "-preset",
@@ -785,33 +859,43 @@ export async function editly(config: EditlyConfig): Promise<void> {
     ? ["-map", `[${finalVideoLabel}]`, "-map", `[${audioFilter.outputLabel}]`]
     : ["-map", `[${finalVideoLabel}]`];
 
-  const ffmpegArgs = [
-    "-hide_banner",
-    "-loglevel",
-    verbose ? "info" : "error",
-    ...inputArgs,
-    "-filter_complex",
-    filterComplex,
+  const outputArgs = [
     ...mapArgs,
     "-r",
     String(fps),
-    ...outputArgs,
-    "-y",
-    outPath,
+    ...codecArgs,
+    ...(config.shortest ? ["-shortest"] : []),
   ];
 
   if (verbose) {
-    console.log("ffmpeg", ffmpegArgs.join(" "));
+    const inputArgs = allInputs.flatMap((input) => ["-i", input]);
+    console.log(
+      "ffmpeg",
+      [
+        ...inputArgs,
+        "-filter_complex",
+        filterComplex,
+        ...outputArgs,
+        "-y",
+        outPath,
+      ].join(" "),
+    );
     console.log("\nFilter complex:\n", filterComplex.split(";").join(";\n"));
   }
 
-  const result = await $`ffmpeg ${ffmpegArgs}`.quiet();
+  const result = await backend.run({
+    inputs: allInputs,
+    filterComplex,
+    outputArgs,
+    outputPath: outPath,
+    verbose,
+  });
 
-  if (result.exitCode !== 0) {
-    throw new Error(`ffmpeg failed with exit code ${result.exitCode}`);
+  if (result.output.type === "file" && verbose) {
+    console.log(`Output: ${result.output.path}`);
   }
 
-  console.log(`Output: ${outPath}`);
+  return { output: result.output };
 }
 
 export default editly;

package/src/ai-sdk/providers/editly/layers.ts

@@ -1,4 +1,5 @@
 import type {
+  CropPosition,
   FillColorLayer,
   ImageLayer,
   ImageOverlayLayer,
@@ -15,6 +16,33 @@ import type {
   VideoLayer,
 } from "./types";
 
+function getCropPositionExpr(position: CropPosition | undefined): {
+  x: string;
+  y: string;
+} {
+  switch (position) {
+    case "top-left":
+      return { x: "0", y: "0" };
+    case "top":
+      return { x: "(iw-ow)/2", y: "0" };
+    case "top-right":
+      return { x: "iw-ow", y: "0" };
+    case "left":
+      return { x: "0", y: "(ih-oh)/2" };
+    case "right":
+      return { x: "iw-ow", y: "(ih-oh)/2" };
+    case "bottom-left":
+      return { x: "0", y: "ih-oh" };
+    case "bottom":
+      return { x: "(iw-ow)/2", y: "ih-oh" };
+    case "bottom-right":
+      return { x: "iw-ow", y: "ih-oh" };
+    case "center":
+    default:
+      return { x: "(iw-ow)/2", y: "(ih-oh)/2" };
+  }
+}
+
 function escapeDrawText(text: string): string {
   return text
     .replace(/\\/g, "\\\\")
@@ -161,9 +189,14 @@ export function getVideoFilterWithTrim(
   const layerHeight = parseSize(layer.height, height);
 
   if (isOverlay) {
-    filters.push(
-      `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=decrease`,
-    );
+    let scaleFilter = `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=decrease`;
+    if (layer.resizeMode === "cover") {
+      const { x, y } = getCropPositionExpr(layer.cropPosition);
+      scaleFilter = `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=increase,crop=${layerWidth}:${layerHeight}:${x}:${y}`;
+    } else if (layer.resizeMode === "stretch") {
+      scaleFilter = `scale=${layerWidth}:${layerHeight}`;
+    }
+    filters.push(scaleFilter);
     filters.push("setsar=1");
     filters.push("fps=30");
     filters.push("settb=1/30");
@@ -555,7 +588,13 @@ export function getTitleFilter(
 ): string {
   const text = escapeDrawText(layer.text);
   const color = layer.textColor ?? "white";
-  const fontSize = Math.round(Math.min(width, height) * 0.08);
+
+  // Auto-size font to fit within 90% of frame width (same approach as subtitle)
+  const maxFontSize = Math.round(Math.min(width, height) * 0.08);
+  const maxTextWidth = width * 0.9;
+  // Average char width ≈ fontSize * 0.55 for sans-serif fonts
+  const fittedFontSize = Math.floor(maxTextWidth / (layer.text.length * 0.55));
+  const fontSize = Math.max(16, Math.min(maxFontSize, fittedFontSize));
 
   let x = "(w-text_w)/2";
   let y = "(h-text_h)/2";
@@ -587,7 +626,13 @@ export function getSubtitleFilter(
   const text = escapeDrawText(layer.text);
   const textColor = layer.textColor ?? "white";
   const bgColor = layer.backgroundColor ?? "black@0.7";
-  const fontSize = Math.round(Math.min(width, height) * 0.05);
+
+  // Auto-size font to fit within 90% of frame width
+  const maxFontSize = Math.round(Math.min(width, height) * 0.05);
+  const maxTextWidth = width * 0.9;
+  // Average char width ≈ fontSize * 0.55 for sans-serif fonts
+  const fittedFontSize = Math.floor(maxTextWidth / (layer.text.length * 0.55));
+  const fontSize = Math.max(16, Math.min(maxFontSize, fittedFontSize));
   const boxPadding = Math.round(fontSize * 0.4);
 
   const fontFile = layer.fontPath
@@ -626,7 +671,14 @@ export function getTitleBackgroundFilter(
 
   const text = escapeDrawText(layer.text);
   const textColor = layer.textColor ?? "white";
-  const fontSize = Math.round(Math.min(width, height) * 0.1);
+
+  // Auto-size font to fit within 90% of frame width
+  const maxFontSizeBg = Math.round(Math.min(width, height) * 0.1);
+  const maxTextWidthBg = width * 0.9;
+  const fittedFontSizeBg = Math.floor(
+    maxTextWidthBg / (layer.text.length * 0.55),
+  );
+  const fontSize = Math.max(16, Math.min(maxFontSizeBg, fittedFontSizeBg));
 
   const fontFile = layer.fontPath
     ? `:fontfile='${escapeDrawText(layer.fontPath)}'`