varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/ffmpeg/CLAUDE.md

@@ -0,0 +1,68 @@
+# ffmpeg video mixing lessons
+
+## problem: audio/video desync when mixing clips
+
+### what went wrong
+
+using `-ss X -t Y` to pre-trim input, then applying relative trim filters caused timing drift:
+
+```bash
+# BAD: relative timestamps after pre-trim
+ffmpeg -ss 64 -t 36 -i original.mp4 ...
+  -filter_complex "
+    [0:v]split=5[orig1][orig2]...;
+    [orig1]trim=0:4,setpts=PTS-STARTPTS[o1];
+    [orig2]trim=4:11,setpts=PTS-STARTPTS[o2];
+    ..."
+```
+
+this produced wrong duration (38s instead of 36s) with audio desync.
+
+### solution: use absolute timestamps from full input
+
+trim directly from full original using absolute timestamps:
+
+```bash
+# GOOD: absolute timestamps from full file
+ffmpeg -i original.mp4 -i scene1.mp4 -i scene2.mp4 ...
+  -filter_complex "
+    [0:v]trim=64:68,setpts=PTS-STARTPTS[o1];
+    [1:v]scale=1280:720,trim=4:6,setpts=PTS-STARTPTS[s1];
+    [0:v]trim=70:75,setpts=PTS-STARTPTS[o2];
+    ...
+    [o1][s1][o2]...concat=n=N:v=1:a=0[outv];
+    [0:a]atrim=64:100,asetpts=PTS-STARTPTS[outa]
+  "
+  -map "[outv]" -map "[outa]"
+```
+
+### key points
+
+1. **absolute timestamps**: trim from full input file, not pre-trimmed
+2. **separate audio handling**: use `atrim` on audio stream independently
+3. **setpts reset**: always use `setpts=PTS-STARTPTS` after trim to reset timestamps
+4. **scale before trim**: when mixing different resolutions, scale first then trim
+5. **video duration = audio duration**: ensure total video segments match audio segment length
+
+### example: inserting clips into original
+
+to insert generated clips at specific timestamps while keeping continuous audio:
+
+```bash
+ffmpeg -y \
+  -i original.mp4 \
+  -i generated-scene.mp4 \
+  -filter_complex "
+    [0:v]trim=START1:END1,setpts=PTS-STARTPTS[o1];
+    [1:v]scale=1280:720,trim=0:DURATION,setpts=PTS-STARTPTS[s1];
+    [0:v]trim=START2:END2,setpts=PTS-STARTPTS[o2];
+    [o1][s1][o2]concat=n=3:v=1:a=0[outv];
+    [0:a]atrim=AUDIO_START:AUDIO_END,asetpts=PTS-STARTPTS[outa]
+  " \
+  -map "[outv]" -map "[outa]" \
+  -c:v libx264 -preset fast -crf 18 \
+  -c:a aac -b:a 192k \
+  output.mp4
+```
+
+timestamps must add up: `(END1-START1) + DURATION + (END2-START2) = AUDIO_END - AUDIO_START`

package/package.json

@@ -1,42 +1,75 @@
 {
   "name": "varg.ai-sdk",
-  "module": "index.ts",
+  "module": "src/index.ts",
   "type": "module",
   "bin": {
-    "varg": "./cli/index.ts"
+    "varg": "./src/cli/index.ts"
   },
   "scripts": {
+    "check": "biome check . && tsc --noEmit",
     "lint": "biome check .",
-    "format": "biome format --write ."
+    "format": "biome format --write .",
+    "type-check": "tsc --noEmit",
+    "prepare": "husky || true",
+    "size": "size-limit",
+    "studio": "bun run src/studio/index.ts"
+  },
+  "lint-staged": {
+    "*.{js,ts,tsx}": [
+      "biome check --write --no-errors-on-unmatched"
+    ],
+    "*.json": [
+      "biome format --write"
+    ]
   },
   "devDependencies": {
     "@biomejs/biome": "^2.3.7",
-    "@types/bun": "latest"
+    "@commitlint/cli": "^20.1.0",
+    "@commitlint/config-conventional": "^20.0.0",
+    "@size-limit/preset-small-lib": "^11.2.0",
+    "@types/bun": "latest",
+    "@types/react": "^19.2.7",
+    "husky": "^9.1.7",
+    "lint-staged": "^16.2.7"
   },
   "peerDependencies": {
     "typescript": "^5"
   },
   "dependencies": {
     "@ai-sdk/fal": "^1.0.23",
-    "@ai-sdk/replicate": "^1.0.18",
+    "@ai-sdk/fireworks": "^2.0.16",
+    "@ai-sdk/groq": "^3.0.12",
+    "@ai-sdk/openai": "^3.0.9",
+    "@ai-sdk/provider": "^3.0.2",
+    "@ai-sdk/replicate": "^2.0.5",
     "@aws-sdk/client-s3": "^3.937.0",
     "@aws-sdk/s3-request-presigner": "^3.937.0",
-    "@elevenlabs/elevenlabs-js": "^2.25.0",
+    "@elevenlabs/elevenlabs-js": "^2.28.0",
     "@fal-ai/client": "^1.7.2",
     "@higgsfield/client": "^0.1.2",
+    "@inkjs/ui": "^2.0.0",
     "@remotion/cli": "^4.0.377",
     "@types/fluent-ffmpeg": "^2.1.28",
-    "ai": "^5.0.98",
+    "ai": "^6.0.26",
+    "apify-client": "^2.20.0",
     "citty": "^0.1.6",
     "fluent-ffmpeg": "^2.1.3",
     "groq-sdk": "^0.36.0",
+    "ink": "^6.5.1",
     "react": "^19.2.0",
     "react-dom": "^19.2.0",
     "remotion": "^4.0.377",
-    "replicate": "^1.4.0"
+    "replicate": "^1.4.0",
+    "zod": "^4.2.1"
   },
-  "version": "0.1.1",
+  "version": "0.4.0-alpha.1",
   "exports": {
-    ".": "./index.ts"
+    ".": "./src/index.ts",
+    "./ai": "./src/ai-sdk/index.ts",
+    "./core": "./src/core/index.ts",
+    "./providers": "./src/providers/index.ts",
+    "./definitions": "./src/definitions/index.ts",
+    "./react": "./src/react/index.ts",
+    "./studio": "./src/studio/index.ts"
   }
 }

package/pipeline/cookbooks/scripts/animate-frames-parallel.ts

@@ -0,0 +1,84 @@
+/**
+ * Animate multiple frames in parallel using kling
+ * Usage: bun run pipeline/cookbooks/scripts/animate-frames-parallel.ts
+ */
+
+import { fal } from "@fal-ai/client";
+
+interface VideoConfig {
+  name: string;
+  framePath: string;
+  prompt: string;
+  duration?: "5" | "10";
+}
+
+async function animateFrames(configs: VideoConfig[], outputDir: string) {
+  console.log(`Animating ${configs.length} frames in parallel...\n`);
+
+  // Upload all frames first
+  const frameUrls: string[] = [];
+  for (const config of configs) {
+    const url = await fal.storage.upload(Bun.file(config.framePath));
+    frameUrls.push(url);
+  }
+
+  const promises = configs.map((config, i) => {
+    return fal.subscribe("fal-ai/kling-video/v2.5-turbo/pro/image-to-video", {
+      input: {
+        prompt: `${config.prompt}, NO talking NO lip movement`,
+        image_url: frameUrls[i],
+        duration: config.duration || "5",
+        // note: aspect_ratio is determined by input image dimensions
+      },
+    });
+  });
+
+  const results = await Promise.all(promises);
+
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i] as { data?: { video?: { url?: string } } };
+    const url = result.data?.video?.url;
+    const config = configs[i];
+    if (url && config) {
+      const response = await fetch(url);
+      const buffer = await response.arrayBuffer();
+      await Bun.write(`${outputDir}/${config.name}_video.mp4`, buffer);
+      console.log(`${config.name}_video.mp4 saved`);
+    } else {
+      console.error(`No URL for ${config?.name ?? "unknown"}`);
+    }
+  }
+
+  console.log("\nAll videos saved!");
+}
+
+// Example usage:
+async function main() {
+  const outputDir = "media/girl-ruined-you";
+
+  const configs: VideoConfig[] = [
+    {
+      name: "scene6",
+      framePath: `${outputDir}/scene6_frame.jpg`,
+      prompt:
+        "3D pixar animation, two cats meet eyes in coffee shop, warm romantic moment",
+      duration: "5",
+    },
+    {
+      name: "scene7",
+      framePath: `${outputDir}/scene7_frame.jpg`,
+      prompt: "3D pixar animation, two cats walking together, sunset, romantic",
+      duration: "5",
+    },
+    {
+      name: "scene14",
+      framePath: `${outputDir}/scene14_frame.jpg`,
+      prompt: "3D pixar animation, cat looks at sunrise, hopeful realization",
+      duration: "5",
+    },
+  ];
+
+  await animateFrames(configs, outputDir);
+}
+
+main().catch(console.error);

package/pipeline/cookbooks/scripts/combine-scenes.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Combine multiple scene videos with audio clips
+# Usage: ./combine-scenes.sh <project_dir>
+
+PROJECT_DIR=${1:-"media/girl-ruined-you"}
+
+# Scene timing configuration (adjust as needed)
+# Format: scene_num:start_time:duration
+SCENES=(
+  "1:0:3.5"
+  "2:3.5:6.5"
+  "3:10:10"
+  "4:20:15"
+  "5:35:7"
+)
+
+echo "Extracting audio clips..."
+for scene_config in "${SCENES[@]}"; do
+  IFS=':' read -r num start dur <<< "$scene_config"
+  ffmpeg -y -i "$PROJECT_DIR/voiceover.mp3" -ss "$start" -t "$dur" "$PROJECT_DIR/audio_scene${num}.mp3" 2>/dev/null
+  echo "  audio_scene${num}.mp3 ($dur sec)"
+done
+
+echo ""
+echo "Combining videos with audio..."
+for scene_config in "${SCENES[@]}"; do
+  IFS=':' read -r num start dur <<< "$scene_config"
+  
+  # Calculate loop count needed (5s videos)
+  loops=$(echo "($dur / 5) - 1" | bc)
+  if [ "$loops" -lt 0 ]; then loops=0; fi
+  
+  ffmpeg -y -stream_loop "$loops" -i "$PROJECT_DIR/scene${num}_video.mp4" \
+    -i "$PROJECT_DIR/audio_scene${num}.mp3" \
+    -t "$dur" -c:v libx264 -preset fast -crf 20 -c:a aac -b:a 128k -shortest \
+    "$PROJECT_DIR/scene${num}_final.mp4" 2>/dev/null
+  echo "  scene${num}_final.mp4"
+done
+
+echo ""
+echo "Creating concat file..."
+rm -f "$PROJECT_DIR/scenes.txt"
+for scene_config in "${SCENES[@]}"; do
+  IFS=':' read -r num start dur <<< "$scene_config"
+  echo "file 'scene${num}_final.mp4'" >> "$PROJECT_DIR/scenes.txt"
+done
+
+echo "Concatenating all scenes..."
+cd "$PROJECT_DIR" && ffmpeg -y -f concat -safe 0 -i scenes.txt -c copy combined_scenes.mp4 2>/dev/null
+
+duration=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 combined_scenes.mp4)
+echo ""
+echo "Done! combined_scenes.mp4 ($duration sec)"

package/pipeline/cookbooks/scripts/generate-frames-parallel.ts

@@ -0,0 +1,99 @@
+/**
+ * Generate multiple scene frames in parallel using flux kontext
+ * Usage: bun run pipeline/cookbooks/scripts/generate-frames-parallel.ts
+ */
+
+import { fal } from "@fal-ai/client";
+
+interface FrameConfig {
+  name: string;
+  prompt: string;
+  imageUrls: string[]; // character reference URLs
+  multi?: boolean; // use kontext/multi for multiple characters
+}
+
+async function generateFrames(configs: FrameConfig[], outputDir: string) {
+  console.log(`Generating ${configs.length} frames in parallel...\n`);
+
+  const promises = configs.map((config) => {
+    if (config.multi) {
+      return fal.subscribe("fal-ai/flux-pro/kontext/multi", {
+        input: {
+          prompt: config.prompt,
+          image_urls: config.imageUrls,
+          aspect_ratio: "9:16" as const,
+        },
+      });
+    } else {
+      return fal.subscribe("fal-ai/flux-pro/kontext", {
+        input: {
+          prompt: config.prompt,
+          image_url: config.imageUrls[0] ?? "",
+          aspect_ratio: "9:16" as const,
+        },
+      });
+    }
+  });
+
+  const results = await Promise.all(promises);
+
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i] as {
+      data?: { images?: Array<{ url?: string }> };
+    };
+    const url = result.data?.images?.[0]?.url;
+    const config = configs[i];
+    if (url && config) {
+      const response = await fetch(url);
+      const buffer = await response.arrayBuffer();
+      await Bun.write(`${outputDir}/${config.name}_frame.jpg`, buffer);
+      console.log(`${config.name}_frame.jpg saved`);
+    } else {
+      console.error(`No URL for ${config?.name ?? "unknown"}`);
+    }
+  }
+
+  console.log("\nAll frames saved!");
+}
+
+// Example usage:
+async function main() {
+  const outputDir = "media/girl-ruined-you";
+
+  // Upload character references first
+  const protagonist = await fal.storage.upload(
+    Bun.file(`${outputDir}/cat_protagonist.png`),
+  );
+  const secondGirl = await fal.storage.upload(
+    Bun.file(`${outputDir}/cat_second_girl.png`),
+  );
+
+  const configs: FrameConfig[] = [
+    {
+      name: "scene6",
+      prompt:
+        "3D pixar style: male cat in hoodie (first) and elegant female cat (second) meeting eyes in coffee shop, warm golden lighting, vertical portrait 9:16",
+      imageUrls: [protagonist, secondGirl],
+      multi: true,
+    },
+    {
+      name: "scene7",
+      prompt:
+        "3D pixar style: male cat and female cat walking together, sunset, romantic, vertical portrait 9:16",
+      imageUrls: [protagonist, secondGirl],
+      multi: true,
+    },
+    // Single character scene
+    {
+      name: "scene14",
+      prompt:
+        "Place this cat looking at sunrise through window, hopeful, vertical portrait 9:16",
+      imageUrls: [protagonist],
+      multi: false,
+    },
+  ];
+
+  await generateFrames(configs, outputDir);
+}
+
+main().catch(console.error);

package/pipeline/cookbooks/scripts/still-to-video.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Convert still frame to video with ken burns effect (slow zoom)
+# Usage: ./still-to-video.sh <input.jpg> <output.mp4> <duration> [zoom_direction]
+# zoom_direction: in (default), out
+
+INPUT=$1
+OUTPUT=$2
+DURATION=$3
+ZOOM=${4:-"in"}
+
+if [ -z "$INPUT" ] || [ -z "$OUTPUT" ] || [ -z "$DURATION" ]; then
+  echo "Usage: ./still-to-video.sh <input.jpg> <output.mp4> <duration> [in|out]"
+  exit 1
+fi
+
+# Get input dimensions
+WIDTH=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of csv=p=0 "$INPUT")
+HEIGHT=$(ffprobe -v error -select_streams v:0 -show_entries stream=height -of csv=p=0 "$INPUT")
+
+echo "Creating $DURATION sec video from $INPUT ($WIDTH x $HEIGHT)..."
+
+if [ "$ZOOM" = "out" ]; then
+  # Zoom out: start zoomed in, end at normal
+  FILTER="zoompan=z='1.2-0.2*on/(${DURATION}*25)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=${DURATION}*25:s=${WIDTH}x${HEIGHT}:fps=25"
+else
+  # Zoom in: start normal, end zoomed
+  FILTER="zoompan=z='1+0.2*on/(${DURATION}*25)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=${DURATION}*25:s=${WIDTH}x${HEIGHT}:fps=25"
+fi
+
+ffmpeg -y -loop 1 -i "$INPUT" \
+  -vf "$FILTER" \
+  -t "$DURATION" \
+  -c:v libx264 -preset fast -crf 20 \
+  -pix_fmt yuv420p \
+  "$OUTPUT"
+
+echo "Done: $OUTPUT"