varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/action/sync/SKILL.md

@@ -1,136 +0,0 @@
----
-name: video-lipsync
-description: sync video with audio using wav2lip ai model or simple audio overlay. use when creating talking videos, matching lip movements to audio, or combining video with voiceovers.
-allowed-tools: Read, Bash
----
-
-# video lipsync
-
-sync video with audio using ai-powered lipsync or simple overlay.
-
-## methods
-
-### wav2lip (ai-powered)
-- uses replicate wav2lip model
-- matches lip movements to audio
-- works with url inputs
-- processing time: 30-60 seconds
-- best for: talking character videos
-
-### overlay (simple)
-- adds audio track to video using ffmpeg
-- no lip movement matching
-- works with local files
-- processing time: instant
-- best for: background music, voiceovers
-
-## usage
-
-### sync with method selection
-```bash
-bun run service/sync.ts sync <videoUrl> <audioUrl> [method] [output]
-```
-
-**parameters:**
-- `videoUrl` (required): video file path or url
-- `audioUrl` (required): audio file path or url
-- `method` (optional): "wav2lip" or "overlay" (default: overlay)
-- `output` (optional): output path (default: output-synced.mp4)
-
-**example:**
-```bash
-bun run service/sync.ts sync video.mp4 audio.mp3 overlay output.mp4
-```
-
-### wav2lip direct
-```bash
-bun run service/sync.ts wav2lip <videoUrl> <audioUrl>
-```
-
-**example:**
-```bash
-bun run service/sync.ts wav2lip https://example.com/character.mp4 https://example.com/voice.mp3
-```
-
-### overlay direct
-```bash
-bun run service/sync.ts overlay <videoPath> <audioPath> [output]
-```
-
-**example:**
-```bash
-bun run service/sync.ts overlay character.mp4 narration.mp3 final.mp4
-```
-
-## as library
-
-```typescript
-import { lipsync, lipsyncWav2Lip, lipsyncOverlay } from "./service/sync"
-
-// flexible sync
-const result = await lipsync({
-  videoUrl: "video.mp4",
-  audioUrl: "audio.mp3",
-  method: "wav2lip",
-  output: "synced.mp4"
-})
-
-// wav2lip specific
-const lipsynced = await lipsyncWav2Lip({
-  videoUrl: "https://example.com/video.mp4",
-  audioUrl: "https://example.com/audio.mp3"
-})
-
-// overlay specific
-const overlayed = await lipsyncOverlay(
-  "video.mp4",
-  "audio.mp3",
-  "output.mp4"
-)
-```
-
-## when to use each method
-
-### use wav2lip when:
-- creating talking character videos
-- lip movements must match speech
-- have urls for video and audio
-- quality is more important than speed
-
-### use overlay when:
-- adding background music
-- audio doesn't require lip sync
-- working with local files
-- need instant processing
-
-## typical workflow
-
-1. generate character image (image service)
-2. animate character (video service)
-3. generate voiceover (voice service)
-4. sync with wav2lip (this service)
-5. add captions (captions service)
-
-## tips
-
-**for wav2lip:**
-- use close-up character shots for best results
-- ensure audio is clear and well-paced
-- video should show face clearly
-- works best with 5-10 second clips
-
-**for overlay:**
-- match audio length to video length
-- ffmpeg will loop short audio or trim long audio
-- preserves original video quality
-
-## environment variables
-
-required (for wav2lip):
-- `REPLICATE_API_TOKEN` - for wav2lip model
-
-no special requirements for overlay method (ffmpeg must be installed)
-
-## error handling
-
-if wav2lip fails, the service automatically falls back to overlay method with a warning message.

package/action/sync/index.ts

@@ -1,145 +0,0 @@
-#!/usr/bin/env bun
-
-/**
- * lipsync service - combines video with audio using various methods
- * supports wav2lip, synclabs, and simple audio overlay
- */
-
-import type { ActionMeta } from "../../cli/types";
-import { addAudio } from "../../lib/ffmpeg";
-import { runModel } from "../../lib/replicate";
-
-export const meta: ActionMeta = {
-  name: "sync",
-  type: "action",
-  description: "sync audio to video (lipsync)",
-  inputType: "video+audio",
-  outputType: "video",
-  schema: {
-    input: {
-      type: "object",
-      required: ["video", "audio"],
-      properties: {
-        video: {
-          type: "string",
-          format: "file-path",
-          description: "input video file or url",
-        },
-        audio: {
-          type: "string",
-          format: "file-path",
-          description: "audio file or url to sync",
-        },
-        method: {
-          type: "string",
-          enum: ["wav2lip", "overlay"],
-          default: "overlay",
-          description: "sync method (wav2lip requires urls)",
-        },
-        output: {
-          type: "string",
-          format: "file-path",
-          description: "output video path",
-        },
-      },
-    },
-    output: { type: "string", format: "file-path", description: "video path" },
-  },
-  async run(options) {
-    const { video, audio, method, output } = options as {
-      video: string;
-      audio: string;
-      method?: "wav2lip" | "overlay";
-      output?: string;
-    };
-    return lipsync({ videoUrl: video, audioUrl: audio, method, output });
-  },
-};
-
-// types
-export interface LipsyncOptions {
-  videoUrl: string;
-  audioUrl: string;
-  method?: "wav2lip" | "synclabs" | "overlay";
-  output?: string;
-}
-
-export interface Wav2LipOptions {
-  videoUrl: string;
-  audioUrl: string;
-}
-
-// core functions
-export async function lipsync(options: LipsyncOptions) {
-  const { videoUrl, audioUrl, method = "overlay", output } = options;
-
-  if (!videoUrl || !audioUrl) {
-    throw new Error("videoUrl and audioUrl are required");
-  }
-
-  console.log(`[sync] syncing video with audio using ${method}...`);
-
-  switch (method) {
-    case "wav2lip":
-      return await lipsyncWav2Lip({ videoUrl, audioUrl });
-
-    case "synclabs":
-      console.log(
-        `[sync] synclabs not yet implemented, falling back to overlay`,
-      );
-      return await lipsyncOverlay(videoUrl, audioUrl, output);
-
-    case "overlay":
-      return await lipsyncOverlay(videoUrl, audioUrl, output);
-
-    default:
-      throw new Error(`unknown lipsync method: ${method}`);
-  }
-}
-
-export async function lipsyncWav2Lip(options: Wav2LipOptions) {
-  const { videoUrl, audioUrl } = options;
-
-  console.log(`[sync] using wav2lip model...`);
-
-  try {
-    const output = await runModel("devxpy/cog-wav2lip", {
-      face: videoUrl,
-      audio: audioUrl,
-    });
-
-    console.log(`[sync] wav2lip completed`);
-    return output;
-  } catch (error) {
-    console.error(`[sync] wav2lip error:`, error);
-    throw error;
-  }
-}
-
-export async function lipsyncOverlay(
-  videoPath: string,
-  audioPath: string,
-  output: string = "output-synced.mp4",
-) {
-  console.log(`[sync] overlaying audio on video...`);
-
-  try {
-    const result = await addAudio({
-      videoPath,
-      audioPath,
-      output,
-    });
-
-    console.log(`[sync] overlay completed`);
-    return result;
-  } catch (error) {
-    console.error(`[sync] overlay error:`, error);
-    throw error;
-  }
-}
-
-// cli
-if (import.meta.main) {
-  const { runCli } = await import("../../cli/runner");
-  runCli(meta);
-}

package/action/transcribe/SKILL.md

@@ -1,179 +0,0 @@
----
-name: audio-transcription
-description: transcribe audio to text or subtitles using groq whisper or fireworks with srt/vtt support. use when converting speech to text, generating subtitles, or need word-level timestamps for captions.
-allowed-tools: Read, Bash
----
-
-# audio transcription
-
-convert audio to text or subtitle files using ai transcription.
-
-## providers
-
-### groq (ultra-fast)
-- uses whisper-large-v3
-- fastest transcription (~5-10 seconds)
-- plain text output
-- sentence-level timing
-- best for: quick transcripts, text extraction
-
-### fireworks (word-level)
-- uses whisper-v3
-- word-level timestamps
-- outputs srt or vtt format
-- precise subtitle timing
-- best for: captions, subtitles, timed transcripts
-
-## usage
-
-### basic transcription
-```bash
-bun run service/transcribe.ts <audioUrl> <provider> [outputPath]
-```
-
-**example:**
-```bash
-bun run service/transcribe.ts media/audio.mp3 groq
-bun run service/transcribe.ts media/audio.mp3 fireworks output.srt
-```
-
-### with output format
-```bash
-bun run lib/fireworks.ts <audioPath> <outputPath>
-```
-
-**example:**
-```bash
-bun run lib/fireworks.ts media/audio.mp3 output.srt
-```
-
-## as library
-
-```typescript
-import { transcribe } from "./service/transcribe"
-
-// groq transcription
-const groqResult = await transcribe({
-  audioUrl: "media/audio.mp3",
-  provider: "groq",
-  outputFormat: "text"
-})
-console.log(groqResult.text)
-
-// fireworks with srt
-const fireworksResult = await transcribe({
-  audioUrl: "media/audio.mp3",
-  provider: "fireworks",
-  outputFormat: "srt",
-  outputPath: "subtitles.srt"
-})
-console.log(fireworksResult.text)
-console.log(fireworksResult.outputPath) // subtitles.srt
-```
-
-## output formats
-
-### text (groq default)
-```
-This is the transcribed text from the audio file.
-All words in plain text format.
-```
-
-### srt (subtitle format)
-```
-1
-00:00:00,000 --> 00:00:02,500
-This is the first subtitle
-
-2
-00:00:02,500 --> 00:00:05,000
-This is the second subtitle
-```
-
-### vtt (web video text tracks)
-```
-WEBVTT
-
-00:00:00.000 --> 00:00:02.500
-This is the first subtitle
-
-00:00:02.500 --> 00:00:05.000
-This is the second subtitle
-```
-
-## when to use
-
-use this skill when:
-- converting speech to text
-- generating subtitles for videos
-- creating accessible content
-- need word-level timing for captions
-- extracting dialogue from media
-- preparing transcripts for analysis
-
-## provider comparison
-
-| feature | groq | fireworks |
-|---------|------|-----------|
-| speed | ultra-fast (5-10s) | moderate (15-30s) |
-| output | plain text | srt/vtt with timestamps |
-| timing | sentence-level | word-level |
-| use case | quick transcripts | precise subtitles |
-
-## typical workflows
-
-### for captions
-1. record or generate audio (voice service)
-2. transcribe with fireworks (this service)
-3. add captions to video (captions service)
-
-### for transcripts
-1. extract audio from video
-2. transcribe with groq (this service)
-3. use text for analysis or documentation
-
-## tips
-
-**provider selection:**
-- use **groq** when you just need the text fast
-- use **fireworks** when you need subtitle files
-- use **fireworks** for captions on social media videos
-
-**audio quality:**
-- clear audio transcribes more accurately
-- reduce background noise when possible
-- supports mp3, wav, m4a, and most audio formats
-
-**timing accuracy:**
-- fireworks provides word-level timestamps
-- perfect for lip-sync verification
-- great for precise subtitle placement
-
-## integration with other services
-
-perfect companion for:
-- **captions service** - auto-generate video subtitles
-- **voice service** - transcribe generated speech
-- **sync service** - verify audio timing
-
-## environment variables
-
-required:
-- `GROQ_API_KEY` - for groq provider
-- `FIREWORKS_API_KEY` - for fireworks provider
-
-## processing time
-
-- **groq**: 5-10 seconds (any audio length)
-- **fireworks**: 15-30 seconds (depending on audio length)
-
-## supported formats
-
-input audio:
-- mp3, wav, m4a, ogg, flac
-- video files (extracts audio automatically)
-
-output formats:
-- text (plain text)
-- srt (subtitles)
-- vtt (web video text tracks)

package/action/video/SKILL.md

@@ -1,116 +0,0 @@
----
-name: video-generation
-description: generate videos from images or text prompts using fal.ai. use when user wants to animate images, create videos from text, or needs ai video generation with 5-10 second clips.
-allowed-tools: Read, Bash
----
-
-# video generation
-
-generate ai videos from images or text using fal.ai with automatic s3 upload support.
-
-## capabilities
-
-- **image-to-video**: animate static images with motion prompts
-- **text-to-video**: generate videos directly from text descriptions
-- supports 5 or 10 second duration
-- automatic s3 upload
-
-## usage
-
-### generate from image
-```bash
-bun run service/video.ts from_image <prompt> <imageUrl> [duration] [upload]
-```
-
-**parameters:**
-- `prompt` (required): motion description (e.g., "camera pan left")
-- `imageUrl` (required): url of the source image
-- `duration` (optional): 5 or 10 seconds (default: 5)
-- `upload` (optional): "true" to upload to s3
-
-**example:**
-```bash
-bun run service/video.ts from_image "person talking naturally" https://example.com/headshot.jpg 5 true
-```
-
-### generate from text
-```bash
-bun run service/video.ts from_text <prompt> [duration] [upload]
-```
-
-**parameters:**
-- `prompt` (required): video scene description
-- `duration` (optional): 5 or 10 seconds (default: 5)
-- `upload` (optional): "true" to upload to s3
-
-**example:**
-```bash
-bun run service/video.ts from_text "waves crashing on beach at sunset" 10 true
-```
-
-## as library
-
-```typescript
-import { generateVideoFromImage, generateVideoFromText } from "./service/video"
-
-// animate an image
-const videoResult = await generateVideoFromImage(
-  "camera zoom in slowly",
-  "https://example.com/portrait.jpg",
-  { duration: 5, upload: true }
-)
-console.log(videoResult.videoUrl)
-console.log(videoResult.uploaded) // s3 url if upload=true
-
-// generate from text
-const textVideo = await generateVideoFromText(
-  "forest path with sunlight filtering through trees",
-  { duration: 10, upload: true }
-)
-```
-
-## output
-
-returns `VideoGenerationResult`:
-```typescript
-{
-  videoUrl: string,      // direct video url
-  duration?: number,     // actual video duration
-  uploaded?: string      // s3 url if upload requested
-}
-```
-
-## when to use
-
-use this skill when:
-- animating character headshots or portraits
-- creating motion from static images
-- generating video clips from text descriptions
-- preparing videos for lipsync or editing pipeline
-- need short form video content (5-10s)
-
-## tips
-
-**for character animation:**
-- use subtle prompts like "person talking naturally" or "slight head movement"
-- keep duration at 5 seconds for character shots
-- combine with lipsync for talking videos
-
-**for scene generation:**
-- be descriptive about camera movement and scene dynamics
-- 10 seconds works better for landscape/scene videos
-
-## environment variables
-
-required:
-- `FAL_API_KEY` - for fal video generation
-
-optional (for s3 upload):
-- `CLOUDFLARE_R2_API_URL`
-- `CLOUDFLARE_ACCESS_KEY_ID`
-- `CLOUDFLARE_ACCESS_SECRET`
-- `CLOUDFLARE_R2_BUCKET`
-
-## generation time
-
-expect 2-3 minutes per video clip

package/action/video/index.ts

@@ -1,125 +0,0 @@
-#!/usr/bin/env bun
-/**
- * video generation service combining fal and higgsfield
- * usage: bun run service/video.ts <command> <args>
- */
-
-import type { ActionMeta } from "../../cli/types";
-import { imageToVideo, textToVideo } from "../../lib/fal";
-import { uploadFromUrl } from "../../utilities/s3";
-
-export const meta: ActionMeta = {
-  name: "video",
-  type: "action",
-  description: "generate video from text or image",
-  inputType: "text/image",
-  outputType: "video",
-  schema: {
-    input: {
-      type: "object",
-      required: ["prompt"],
-      properties: {
-        prompt: { type: "string", description: "what to generate" },
-        image: {
-          type: "string",
-          format: "file-path",
-          description: "input image (enables image-to-video)",
-        },
-        duration: {
-          type: "integer",
-          enum: [5, 10],
-          default: 5,
-          description: "video duration in seconds",
-        },
-      },
-    },
-    output: { type: "string", format: "file-path", description: "video path" },
-  },
-  async run(options) {
-    const { prompt, image, duration } = options as {
-      prompt: string;
-      image?: string;
-      duration?: 5 | 10;
-    };
-    if (image) {
-      return generateVideoFromImage(prompt, image, { duration });
-    }
-    return generateVideoFromText(prompt, { duration });
-  },
-};
-
-export interface VideoGenerationResult {
-  videoUrl: string;
-  duration?: number;
-  uploaded?: string;
-}
-
-export async function generateVideoFromImage(
-  prompt: string,
-  imageUrl: string,
-  options: { duration?: 5 | 10; upload?: boolean } = {},
-): Promise<VideoGenerationResult> {
-  console.log("[service/video] generating video from image");
-
-  const result = await imageToVideo({
-    prompt,
-    imageUrl,
-    duration: options.duration,
-  });
-
-  const videoUrl = result.data?.video?.url;
-  if (!videoUrl) {
-    throw new Error("no video url in result");
-  }
-
-  let uploaded: string | undefined;
-  if (options.upload) {
-    const timestamp = Date.now();
-    const objectKey = `videos/generated/${timestamp}.mp4`;
-    uploaded = await uploadFromUrl(videoUrl, objectKey);
-    console.log(`[service/video] uploaded to ${uploaded}`);
-  }
-
-  return {
-    videoUrl,
-    duration: result.data?.duration,
-    uploaded,
-  };
-}
-
-export async function generateVideoFromText(
-  prompt: string,
-  options: { duration?: 5 | 10; upload?: boolean } = {},
-): Promise<VideoGenerationResult> {
-  console.log("[service/video] generating video from text");
-
-  const result = await textToVideo({
-    prompt,
-    duration: options.duration,
-  });
-
-  const videoUrl = result.data?.video?.url;
-  if (!videoUrl) {
-    throw new Error("no video url in result");
-  }
-
-  let uploaded: string | undefined;
-  if (options.upload) {
-    const timestamp = Date.now();
-    const objectKey = `videos/generated/${timestamp}.mp4`;
-    uploaded = await uploadFromUrl(videoUrl, objectKey);
-    console.log(`[service/video] uploaded to ${uploaded}`);
-  }
-
-  return {
-    videoUrl,
-    duration: result.data?.duration,
-    uploaded,
-  };
-}
-
-// cli
-if (import.meta.main) {
-  const { runCli } = await import("../../cli/runner");
-  runCli(meta);
-}