vargai 0.3.1 → 0.4.0-alpha.1

package/docs/plan.md

@@ -1,66 +0,0 @@
-# varg ts-sdk migration plan
-
-## core
-
-- [ ] File (load from path, url, buffer)
-- [ ] File.toTemp() - save to temp file for editly integration
-- [ ] scene`` tagged template for composing elements in prompts
-- [ ] types (AspectRatio, Element, etc)
-
-## providers
-
-- [ ] @varg/fal (kling, nano-banana, lipsync)
-- [ ] @varg/elevenlabs (tts, music)
-- [ ] @varg/higgsfield (soul, characters)
-- [ ] @varg/heygen (talking heads)
-- [ ] @varg/openai (sora, gpt-image, dall-e)
-- [ ] @varg/replicate (birefnet, generic models)
-
-## generation
-
-- [ ] generateImage (text to image)
-- [ ] generateImage (image to image / edit)
-- [ ] generateVideo (text to video)
-- [ ] generateVideo (image to video)
-- [ ] generateSpeech (voiceover via elevenlabs)
-- [ ] generateMusic (music generation via elevenlabs)
-- [ ] generateLipsync (sync video to audio via fal)
-- [ ] generateTalkingHead (photo to talking video via heygen)
-- [ ] generateElement (character/item/style reference for consistent generation)
-
-## editing (use editly directly)
-
-users use editly npm package directly for video composition.
-we provide `File.toTemp()` to bridge ai outputs → editly paths.
-
-editly covers these py-sdk features:
-- [ ] crop_to_aspect_ratio → editly clips with resizeMode
-- [ ] mix_audio_with_ducking → editly audioNorm + audioTracks
-- [ ] apply_zoom_effect → editly zoomDirection on layers
-- [ ] create_ken_burns_effect → editly zoomDirection: "in" | "out"
-- [ ] create_slideshow → editly clips[] with transitions
-- [ ] concatenate_videos → editly clips[] with transitions
-- [ ] picture_in_picture → editly layers with position/size
-- [ ] blur_background_resize → editly resizeMode: "contain-blur"
-- [ ] 67 gl-transitions (fade, crossfade, wipe, cube, etc.)
-
-## custom effects (not in editly)
-
-- [ ] addCaptions (tiktok-style word-by-word) - custom fabric.js layer for editly
-- [ ] createSlider (before/after reveal animation)
-- [ ] createPushTransition (card push effect)
-- [ ] createSwipeAnimation (tinder-style card swipes)
-- [ ] createSplitScreen (side-by-side comparison)
-- [ ] createPackshot (end card with cta button + blinking cta)
-
-## image processing
-
-- [ ] removeBackground (via replicate birefnet)
-
-## notes
-
-- editly used directly (not wrapped)
-- File.toTemp() bridges ai outputs to editly string paths
-- providers are separate packages (@varg/fal, @varg/elevenlabs, etc)
-- generateElement returns { images, text } for use in prompts
-- scene`` template composes elements in prompts

package/docs/todo.md

@@ -1,14 +0,0 @@
-
-# TODO
-
-• [ ] fal
-• [ ] higgsfield
-• [ ] captions (exactly as in TikTok)
-• [ ] editing (simple concat of video frames)
-• [ ] voiceover via elevenlabs
-• [ ] music generation via elevenlabs
-• [ ] adding music on top of video
-• [ ] lypsync (via sync models over fal)
-• [ ] resize with blur: i.e. 9:16 is resided to 4:5 by adding sides with a blur on all video
-• [ ] Picture-in-the-picture: one video on top of another
-• [ ] fade / slideshow transitions

package/src/ai-sdk/middleware/index.ts

@@ -1,25 +0,0 @@
-export {
-  generatePlaceholder,
-  type PlaceholderOptions,
-  type PlaceholderResult,
-} from "./placeholder";
-export {
-  type ImagePlaceholderFallbackOptions,
-  imagePlaceholderFallbackMiddleware,
-  withImagePlaceholderFallback,
-} from "./wrap-image-model";
-export {
-  type MusicModelMiddleware,
-  type MusicPlaceholderFallbackOptions,
-  musicPlaceholderFallbackMiddleware,
-  withMusicPlaceholderFallback,
-  wrapMusicModel,
-} from "./wrap-music-model";
-export {
-  type PlaceholderFallbackOptions,
-  placeholderFallbackMiddleware,
-  type RenderMode,
-  type VideoModelMiddleware,
-  withPlaceholderFallback,
-  wrapVideoModel,
-} from "./wrap-video-model";

package/src/ai-sdk/middleware/placeholder.ts

@@ -1,111 +0,0 @@
-import { unlink } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { $ } from "bun";
-
-export interface PlaceholderOptions {
-  type: "image" | "video" | "audio";
-  prompt: string;
-  duration?: number;
-  width?: number;
-  height?: number;
-}
-
-export interface PlaceholderResult {
-  data: Uint8Array;
-  placeholder: true;
-}
-
-function promptToColor(prompt: string): string {
-  let hash = 0;
-  for (let i = 0; i < prompt.length; i++) {
-    const char = prompt.charCodeAt(i);
-    hash = (hash << 5) - hash + char;
-    hash = hash & hash;
-  }
-
-  const h = Math.abs(hash) % 360;
-  const s = 40 + (Math.abs(hash >> 8) % 30);
-  const l = 35 + (Math.abs(hash >> 16) % 20);
-
-  return `hsl(${h},${s}%,${l}%)`;
-}
-
-function hslToHex(hsl: string): string {
-  const match = hsl.match(/hsl\((\d+),(\d+)%,(\d+)%\)/);
-  if (!match) return "333333";
-
-  const h = Number.parseInt(match[1]!) / 360;
-  const s = Number.parseInt(match[2]!) / 100;
-  const l = Number.parseInt(match[3]!) / 100;
-
-  const hue2rgb = (p: number, q: number, t: number) => {
-    if (t < 0) t += 1;
-    if (t > 1) t -= 1;
-    if (t < 1 / 6) return p + (q - p) * 6 * t;
-    if (t < 1 / 2) return q;
-    if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6;
-    return p;
-  };
-
-  const q = l < 0.5 ? l * (1 + s) : l + s - l * s;
-  const p = 2 * l - q;
-  const r = Math.round(hue2rgb(p, q, h + 1 / 3) * 255);
-  const g = Math.round(hue2rgb(p, q, h) * 255);
-  const b = Math.round(hue2rgb(p, q, h - 1 / 3) * 255);
-
-  return `${r.toString(16).padStart(2, "0")}${g.toString(16).padStart(2, "0")}${b.toString(16).padStart(2, "0")}`;
-}
-
-function truncatePrompt(text: string, maxLen: number): string {
-  const clean = text.replace(/[^a-zA-Z0-9 .,!?-]/g, "");
-  if (clean.length <= maxLen) return clean;
-  return `${clean.slice(0, maxLen - 3)}...`;
-}
-
-export async function generatePlaceholder(
-  options: PlaceholderOptions,
-): Promise<PlaceholderResult> {
-  const { type, prompt, duration = 3, width = 1080, height = 1920 } = options;
-
-  const color = promptToColor(prompt);
-  const hexColor = hslToHex(color);
-  const labelFontSize = Math.floor(Math.min(width, height) / 20);
-  const promptFontSize = Math.floor(Math.min(width, height) / 35);
-  const maxChars = Math.floor((width * 0.7) / (promptFontSize * 0.5));
-  const typeLabel = type.toUpperCase();
-  const promptText = truncatePrompt(prompt, maxChars);
-
-  const ext = type === "audio" ? "mp3" : type === "image" ? "png" : "mp4";
-  const outputPath = join(
-    tmpdir(),
-    `placeholder_${Date.now()}_${Math.random().toString(36).slice(2)}.${ext}`,
-  );
-
-  try {
-    if (type === "audio") {
-      await $`ffmpeg -y -f lavfi -i anullsrc=r=44100:cl=stereo -t ${duration} -c:a libmp3lame ${outputPath}`.quiet();
-    } else if (type === "image") {
-      const colorInput = `color=c=0x${hexColor}:s=${width}x${height}:d=1`;
-      const labelY = `(h/2)-${labelFontSize}`;
-      const promptY = `(h/2)+${Math.floor(labelFontSize * 0.5)}`;
-      const drawLabel = `drawtext=text='${typeLabel}':fontcolor=white:fontsize=${labelFontSize}:x=(w-text_w)/2:y=${labelY}`;
-      const drawPrompt = `drawtext=text='${promptText}':fontcolor=white@0.7:fontsize=${promptFontSize}:x=(w-text_w)/2:y=${promptY}`;
-      await $`ffmpeg -y -f lavfi -i ${colorInput} -vf ${drawLabel},${drawPrompt} -frames:v 1 -update 1 ${outputPath}`.quiet();
-    } else {
-      const colorInput = `color=c=0x${hexColor}:s=${width}x${height}:d=${duration}:r=30`;
-      const labelY = `(h/2)-${labelFontSize}`;
-      const promptY = `(h/2)+${Math.floor(labelFontSize * 0.5)}`;
-      const drawLabel = `drawtext=text='${typeLabel}':fontcolor=white:fontsize=${labelFontSize}:x=(w-text_w)/2:y=${labelY}`;
-      const drawPrompt = `drawtext=text='${promptText}':fontcolor=white@0.7:fontsize=${promptFontSize}:x=(w-text_w)/2:y=${promptY}`;
-      await $`ffmpeg -y -f lavfi -i ${colorInput} -vf ${drawLabel},${drawPrompt} -c:v libx264 -preset ultrafast -pix_fmt yuv420p ${outputPath}`.quiet();
-    }
-
-    const data = await Bun.file(outputPath).bytes();
-    await unlink(outputPath).catch(() => {});
-    return { data: new Uint8Array(data), placeholder: true };
-  } catch (e) {
-    await unlink(outputPath).catch(() => {});
-    throw e;
-  }
-}

package/src/ai-sdk/middleware/wrap-image-model.ts

@@ -1,86 +0,0 @@
-import type {
-  ImageModelV3,
-  ImageModelV3CallOptions,
-  ImageModelV3Middleware,
-} from "@ai-sdk/provider";
-import { wrapImageModel } from "ai";
-import { generatePlaceholder } from "./placeholder";
-import type { RenderMode } from "./wrap-video-model";
-
-export interface ImagePlaceholderFallbackOptions {
-  mode: RenderMode;
-  onFallback?: (error: Error, prompt: string) => void;
-}
-
-export function imagePlaceholderFallbackMiddleware(
-  options: ImagePlaceholderFallbackOptions,
-): ImageModelV3Middleware {
-  const { mode, onFallback } = options;
-
-  return {
-    specificationVersion: "v3",
-    wrapGenerate: async ({ doGenerate, params, model }) => {
-      const createPlaceholderResult = async () => {
-        const [width, height] = (params.size?.split("x").map(Number) ?? [
-          1024, 1024,
-        ]) as [number, number];
-        const prompt =
-          typeof params.prompt === "string"
-            ? params.prompt
-            : ((params.prompt as { text?: string } | undefined)?.text ??
-              "placeholder");
-
-        const placeholder = await generatePlaceholder({
-          type: "image",
-          prompt,
-          width,
-          height,
-        });
-
-        return {
-          images: [placeholder.data],
-          warnings: [
-            {
-              type: "other" as const,
-              message: "placeholder: provider skipped or failed",
-            },
-          ],
-          response: {
-            timestamp: new Date(),
-            modelId: model.modelId,
-            headers: undefined,
-          },
-        };
-      };
-
-      if (mode === "preview") {
-        return createPlaceholderResult();
-      }
-
-      try {
-        return await doGenerate();
-      } catch (e) {
-        if (mode === "strict") throw e;
-
-        const error = e instanceof Error ? e : new Error(String(e));
-        const promptText =
-          typeof params.prompt === "string"
-            ? params.prompt
-            : ((params.prompt as { text?: string } | undefined)?.text ??
-              "placeholder");
-        onFallback?.(error, promptText);
-        return createPlaceholderResult();
-      }
-    },
-  };
-}
-
-export function withImagePlaceholderFallback(
-  model: ImageModelV3,
-  options: ImagePlaceholderFallbackOptions,
-): ImageModelV3 {
-  return wrapImageModel({
-    model,
-    middleware: imagePlaceholderFallbackMiddleware(options),
-  });
-}

package/src/ai-sdk/middleware/wrap-music-model.ts

@@ -1,108 +0,0 @@
-import type { MusicModelV3, MusicModelV3CallOptions } from "../music-model";
-import { generatePlaceholder } from "./placeholder";
-import type { RenderMode } from "./wrap-video-model";
-
-export interface MusicModelMiddleware {
-  transformParams?: (options: {
-    params: MusicModelV3CallOptions;
-    model: MusicModelV3;
-  }) => PromiseLike<MusicModelV3CallOptions> | MusicModelV3CallOptions;
-
-  wrapGenerate?: (options: {
-    doGenerate: () => PromiseLike<
-      Awaited<ReturnType<MusicModelV3["doGenerate"]>>
-    >;
-    params: MusicModelV3CallOptions;
-    model: MusicModelV3;
-  }) => PromiseLike<Awaited<ReturnType<MusicModelV3["doGenerate"]>>>;
-}
-
-export function wrapMusicModel({
-  model,
-  middleware,
-}: {
-  model: MusicModelV3;
-  middleware: MusicModelMiddleware;
-}): MusicModelV3 {
-  const { transformParams, wrapGenerate } = middleware;
-
-  return {
-    specificationVersion: "v3",
-    provider: model.provider,
-    modelId: model.modelId,
-
-    async doGenerate(params: MusicModelV3CallOptions) {
-      const transformedParams = transformParams
-        ? await transformParams({ params, model })
-        : params;
-
-      const doGenerate = () => model.doGenerate(transformedParams);
-
-      return wrapGenerate
-        ? wrapGenerate({ doGenerate, params: transformedParams, model })
-        : doGenerate();
-    },
-  };
-}
-
-export interface MusicPlaceholderFallbackOptions {
-  mode: RenderMode;
-  onFallback?: (error: Error, prompt: string) => void;
-}
-
-export function musicPlaceholderFallbackMiddleware(
-  options: MusicPlaceholderFallbackOptions,
-): MusicModelMiddleware {
-  const { mode, onFallback } = options;
-
-  return {
-    wrapGenerate: async ({ doGenerate, params, model }) => {
-      const createPlaceholderResult = async () => {
-        const placeholder = await generatePlaceholder({
-          type: "audio",
-          prompt: params.prompt,
-          duration: params.duration ?? 10,
-        });
-
-        return {
-          audio: placeholder.data,
-          warnings: [
-            {
-              type: "other" as const,
-              message: "placeholder: provider skipped or failed",
-            },
-          ],
-          response: {
-            timestamp: new Date(),
-            modelId: model.modelId,
-            headers: undefined,
-          },
-        };
-      };
-
-      if (mode === "preview") {
-        return createPlaceholderResult();
-      }
-
-      try {
-        return await doGenerate();
-      } catch (e) {
-        if (mode === "strict") throw e;
-
-        const error = e instanceof Error ? e : new Error(String(e));
-        onFallback?.(error, params.prompt);
-        return createPlaceholderResult();
-      }
-    },
-  };
-}
-
-export function withMusicPlaceholderFallback(
-  model: MusicModelV3,
-  options: MusicPlaceholderFallbackOptions,
-): MusicModelV3 {
-  return wrapMusicModel({
-    model,
-    middleware: musicPlaceholderFallbackMiddleware(options),
-  });
-}

package/src/ai-sdk/middleware/wrap-video-model.ts

@@ -1,115 +0,0 @@
-import type { VideoModelV3, VideoModelV3CallOptions } from "../video-model";
-import { generatePlaceholder } from "./placeholder";
-
-export type RenderMode = "strict" | "default" | "preview";
-
-export interface VideoModelMiddleware {
-  transformParams?: (options: {
-    params: VideoModelV3CallOptions;
-    model: VideoModelV3;
-  }) => PromiseLike<VideoModelV3CallOptions> | VideoModelV3CallOptions;
-
-  wrapGenerate?: (options: {
-    doGenerate: () => PromiseLike<
-      Awaited<ReturnType<VideoModelV3["doGenerate"]>>
-    >;
-    params: VideoModelV3CallOptions;
-    model: VideoModelV3;
-  }) => PromiseLike<Awaited<ReturnType<VideoModelV3["doGenerate"]>>>;
-}
-
-export function wrapVideoModel({
-  model,
-  middleware,
-}: {
-  model: VideoModelV3;
-  middleware: VideoModelMiddleware;
-}): VideoModelV3 {
-  const { transformParams, wrapGenerate } = middleware;
-
-  return {
-    specificationVersion: "v3",
-    provider: model.provider,
-    modelId: model.modelId,
-    maxVideosPerCall: model.maxVideosPerCall,
-
-    async doGenerate(params: VideoModelV3CallOptions) {
-      const transformedParams = transformParams
-        ? await transformParams({ params, model })
-        : params;
-
-      const doGenerate = () => model.doGenerate(transformedParams);
-
-      return wrapGenerate
-        ? wrapGenerate({ doGenerate, params: transformedParams, model })
-        : doGenerate();
-    },
-  };
-}
-
-export interface PlaceholderFallbackOptions {
-  mode: RenderMode;
-  onFallback?: (error: Error, prompt: string) => void;
-}
-
-export function placeholderFallbackMiddleware(
-  options: PlaceholderFallbackOptions,
-): VideoModelMiddleware {
-  const { mode, onFallback } = options;
-
-  return {
-    wrapGenerate: async ({ doGenerate, params, model }) => {
-      const createPlaceholderResult = async () => {
-        const [width, height] = (params.resolution?.split("x").map(Number) ?? [
-          1080, 1920,
-        ]) as [number, number];
-        const placeholder = await generatePlaceholder({
-          type: "video",
-          prompt: params.prompt,
-          duration: params.duration ?? 3,
-          width,
-          height,
-        });
-
-        return {
-          videos: [placeholder.data],
-          warnings: [
-            {
-              type: "other" as const,
-              message: "placeholder: provider skipped or failed",
-            },
-          ],
-          response: {
-            timestamp: new Date(),
-            modelId: model.modelId,
-            headers: undefined,
-          },
-        };
-      };
-
-      if (mode === "preview") {
-        return createPlaceholderResult();
-      }
-
-      try {
-        return await doGenerate();
-      } catch (e) {
-        if (mode === "strict") throw e;
-
-        const error = e instanceof Error ? e : new Error(String(e));
-        onFallback?.(error, params.prompt);
-        return createPlaceholderResult();
-      }
-    },
-  };
-}
-
-export function withPlaceholderFallback(
-  model: VideoModelV3,
-  options: PlaceholderFallbackOptions,
-): VideoModelV3 {
-  return wrapVideoModel({
-    model,
-    middleware: placeholderFallbackMiddleware(options),
-  });
-}