varg.ai-sdk 0.1.0

package/action/image/SKILL.md

@@ -0,0 +1,140 @@
+---
+name: image-generation
+description: generate ai images using fal (flux models) or higgsfield soul characters. use when user wants to create images, headshots, character portraits, or needs image generation with specific models.
+allowed-tools: Read, Bash
+---
+
+# image generation
+
+generate ai images using multiple providers with automatic s3 upload support.
+
+## providers
+
+### fal (flux models)
+- high quality image generation
+- supports flux-pro, flux-dev, and other flux models
+- configurable model selection
+- automatic image opening on generation
+
+### higgsfield soul
+- character headshot generation
+- consistent character style
+- professional portrait quality
+- custom style references
+
+## usage
+
+### generate with fal
+```bash
+bun run service/image.ts fal "a beautiful sunset over mountains" [model] [upload]
+```
+
+**parameters:**
+- `prompt` (required): text description of the image
+- `model` (optional): fal model to use (default: flux-pro)
+- `upload` (optional): "true" to upload to s3
+
+**example:**
+```bash
+bun run service/image.ts fal "professional headshot, studio lighting" true
+```
+
+### generate with soul
+```bash
+bun run service/image.ts soul "friendly person smiling" [styleId] [upload]
+```
+
+**parameters:**
+- `prompt` (required): character description
+- `styleId` (optional): custom higgsfield style reference
+- `upload` (optional): "true" to upload to s3
+
+**example:**
+```bash
+bun run service/image.ts soul "professional business woman" true
+```
+
+## as library
+
+```typescript
+import { generateWithFal, generateWithSoul } from "./service/image"
+
+// fal generation
+const falResult = await generateWithFal("sunset over ocean", {
+  model: "fal-ai/flux-pro/v1.1",
+  upload: true
+})
+console.log(falResult.imageUrl)
+console.log(falResult.uploaded) // s3 url if upload=true
+
+// soul generation
+const soulResult = await generateWithSoul("friendly character", {
+  upload: true
+})
+console.log(soulResult.imageUrl)
+```
+
+## output
+
+returns `ImageGenerationResult`:
+```typescript
+{
+  imageUrl: string,      // direct image url
+  uploaded?: string      // s3 url if upload requested
+}
+```
+
+## when to use
+
+use this skill when:
+- generating images from text descriptions
+- creating character headshots or portraits
+- need consistent character style (use soul)
+- need high quality photorealistic images (use fal)
+- preparing images for video generation pipeline
+
+## nsfw filtering and content moderation
+
+fal.ai has content safety filters that may flag images as nsfw:
+
+**common triggers:**
+- prompts mentioning "athletic wear", "fitted sportswear", "gym clothes"
+- certain body descriptions even when clothed
+- prompts that could be interpreted as revealing clothing
+
+**symptoms:**
+- image generation returns but file is empty (often 7.6KB)
+- no error message, just an unusable file
+- happens inconsistently across similar prompts
+
+**solutions:**
+- specify modest, full-coverage clothing explicitly:
+  - ✅ "long sleeve athletic top and full length leggings"
+  - ✅ "fully covered in modest workout attire"
+  - ❌ "athletic wear" (too vague, may trigger filter)
+  - ❌ "fitted sportswear" (may trigger filter)
+- add "professional", "modest", "appropriate" to descriptions
+- if multiple images in batch get flagged, adjust prompts to be more explicit about coverage
+- always check output file sizes - empty files (< 10KB) indicate nsfw filtering
+
+**example:**
+```bash
+# ❌ may get flagged as nsfw
+bun run service/image.ts fal "woman in athletic wear"
+
+# ✅ less likely to trigger filter
+bun run service/image.ts fal "woman wearing long sleeve athletic top and full length leggings"
+```
+
+## environment variables
+
+required:
+- `FAL_API_KEY` - for fal image generation
+- `HIGGSFIELD_API_KEY` - for soul character generation
+- `HIGGSFIELD_SECRET` - for higgsfield authentication
+
+optional (for s3 upload):
+- `CLOUDFLARE_R2_API_URL`
+- `CLOUDFLARE_ACCESS_KEY_ID`
+- `CLOUDFLARE_ACCESS_SECRET`
+- `CLOUDFLARE_R2_BUCKET`

package/action/image/index.ts

@@ -0,0 +1,112 @@
+#!/usr/bin/env bun
+/**
+ * image generation service combining fal and higgsfield
+ * usage: bun run service/image.ts <command> <args>
+ */
+
+import { generateImage } from "../../lib/fal";
+import { generateSoul } from "../../lib/higgsfield";
+import { uploadFromUrl } from "../../utilities/s3";
+
+export interface ImageGenerationResult {
+  imageUrl: string;
+  uploaded?: string;
+}
+
+export async function generateWithFal(
+  prompt: string,
+  options: { model?: string; upload?: boolean } = {},
+): Promise<ImageGenerationResult> {
+  console.log("[service/image] generating with fal");
+
+  const result = await generateImage({ prompt, model: options.model });
+
+  const imageUrl = result.data?.images?.[0]?.url;
+  if (!imageUrl) {
+    throw new Error("no image url in result");
+  }
+
+  let uploaded: string | undefined;
+  if (options.upload) {
+    const timestamp = Date.now();
+    const objectKey = `images/fal/${timestamp}.png`;
+    uploaded = await uploadFromUrl(imageUrl, objectKey);
+    console.log(`[service/image] uploaded to ${uploaded}`);
+  }
+
+  return { imageUrl, uploaded };
+}
+
+export async function generateWithSoul(
+  prompt: string,
+  options: { styleId?: string; upload?: boolean } = {},
+): Promise<ImageGenerationResult> {
+  console.log("[service/image] generating with higgsfield soul");
+
+  const result = await generateSoul({
+    prompt,
+    styleId: options.styleId,
+  });
+
+  const imageUrl = result.jobs?.[0]?.results?.raw?.url;
+  if (!imageUrl) {
+    throw new Error("no image url in result");
+  }
+
+  let uploaded: string | undefined;
+  if (options.upload) {
+    const timestamp = Date.now();
+    const objectKey = `images/soul/${timestamp}.png`;
+    uploaded = await uploadFromUrl(imageUrl, objectKey);
+    console.log(`[service/image] uploaded to ${uploaded}`);
+  }
+
+  return { imageUrl, uploaded };
+}
+
+// cli runner
+if (import.meta.main) {
+  const [command, ...args] = process.argv.slice(2);
+
+  switch (command) {
+    case "fal": {
+      if (!args[0]) {
+        console.log(`
+usage:
+  bun run service/image.ts fal <prompt> [model] [upload]
+        `);
+        process.exit(1);
+      }
+      const falResult = await generateWithFal(args[0], {
+        model: args[1],
+        upload: args[2] === "true",
+      });
+      console.log(JSON.stringify(falResult, null, 2));
+      break;
+    }
+
+    case "soul": {
+      if (!args[0]) {
+        console.log(`
+usage:
+  bun run service/image.ts soul <prompt> [styleId] [upload]
+        `);
+        process.exit(1);
+      }
+      const soulResult = await generateWithSoul(args[0], {
+        styleId: args[1],
+        upload: args[2] === "true",
+      });
+      console.log(JSON.stringify(soulResult, null, 2));
+      break;
+    }
+
+    default:
+      console.log(`
+usage:
+  bun run service/image.ts fal <prompt> [model] [upload]
+  bun run service/image.ts soul <prompt> [styleId] [upload]
+      `);
+      process.exit(1);
+  }
+}

package/action/sync/SKILL.md

@@ -0,0 +1,136 @@
+---
+name: video-lipsync
+description: sync video with audio using wav2lip ai model or simple audio overlay. use when creating talking videos, matching lip movements to audio, or combining video with voiceovers.
+allowed-tools: Read, Bash
+---
+
+# video lipsync
+
+sync video with audio using ai-powered lipsync or simple overlay.
+
+## methods
+
+### wav2lip (ai-powered)
+- uses replicate wav2lip model
+- matches lip movements to audio
+- works with url inputs
+- processing time: 30-60 seconds
+- best for: talking character videos
+
+### overlay (simple)
+- adds audio track to video using ffmpeg
+- no lip movement matching
+- works with local files
+- processing time: instant
+- best for: background music, voiceovers
+
+## usage
+
+### sync with method selection
+```bash
+bun run service/sync.ts sync <videoUrl> <audioUrl> [method] [output]
+```
+
+**parameters:**
+- `videoUrl` (required): video file path or url
+- `audioUrl` (required): audio file path or url
+- `method` (optional): "wav2lip" or "overlay" (default: overlay)
+- `output` (optional): output path (default: output-synced.mp4)
+
+**example:**
+```bash
+bun run service/sync.ts sync video.mp4 audio.mp3 overlay output.mp4
+```
+
+### wav2lip direct
+```bash
+bun run service/sync.ts wav2lip <videoUrl> <audioUrl>
+```
+
+**example:**
+```bash
+bun run service/sync.ts wav2lip https://example.com/character.mp4 https://example.com/voice.mp3
+```
+
+### overlay direct
+```bash
+bun run service/sync.ts overlay <videoPath> <audioPath> [output]
+```
+
+**example:**
+```bash
+bun run service/sync.ts overlay character.mp4 narration.mp3 final.mp4
+```
+
+## as library
+
+```typescript
+import { lipsync, lipsyncWav2Lip, lipsyncOverlay } from "./service/sync"
+
+// flexible sync
+const result = await lipsync({
+  videoUrl: "video.mp4",
+  audioUrl: "audio.mp3",
+  method: "wav2lip",
+  output: "synced.mp4"
+})
+
+// wav2lip specific
+const lipsynced = await lipsyncWav2Lip({
+  videoUrl: "https://example.com/video.mp4",
+  audioUrl: "https://example.com/audio.mp3"
+})
+
+// overlay specific
+const overlayed = await lipsyncOverlay(
+  "video.mp4",
+  "audio.mp3",
+  "output.mp4"
+)
+```
+
+## when to use each method
+
+### use wav2lip when:
+- creating talking character videos
+- lip movements must match speech
+- have urls for video and audio
+- quality is more important than speed
+
+### use overlay when:
+- adding background music
+- audio doesn't require lip sync
+- working with local files
+- need instant processing
+
+## typical workflow
+
+1. generate character image (image service)
+2. animate character (video service)
+3. generate voiceover (voice service)
+4. sync with wav2lip (this service)
+5. add captions (captions service)
+
+## tips
+
+**for wav2lip:**
+- use close-up character shots for best results
+- ensure audio is clear and well-paced
+- video should show face clearly
+- works best with 5-10 second clips
+
+**for overlay:**
+- match audio length to video length
+- ffmpeg will loop short audio or trim long audio
+- preserves original video quality
+
+## environment variables
+
+required (for wav2lip):
+- `REPLICATE_API_TOKEN` - for wav2lip model
+
+no special requirements for overlay method (ffmpeg must be installed)
+
+## error handling
+
+if wav2lip fails, the service automatically falls back to overlay method with a warning message.

package/action/sync/index.ts

@@ -0,0 +1,187 @@
+#!/usr/bin/env bun
+
+/**
+ * lipsync service - combines video with audio using various methods
+ * supports wav2lip, synclabs, and simple audio overlay
+ */
+
+import { addAudio } from "../../lib/ffmpeg";
+import { runModel } from "../../lib/replicate";
+
+// types
+export interface LipsyncOptions {
+  videoUrl: string;
+  audioUrl: string;
+  method?: "wav2lip" | "synclabs" | "overlay";
+  output?: string;
+}
+
+export interface Wav2LipOptions {
+  videoUrl: string;
+  audioUrl: string;
+}
+
+// core functions
+export async function lipsync(options: LipsyncOptions) {
+  const { videoUrl, audioUrl, method = "overlay", output } = options;
+
+  if (!videoUrl || !audioUrl) {
+    throw new Error("videoUrl and audioUrl are required");
+  }
+
+  console.log(`[sync] syncing video with audio using ${method}...`);
+
+  switch (method) {
+    case "wav2lip":
+      return await lipsyncWav2Lip({ videoUrl, audioUrl });
+
+    case "synclabs":
+      console.log(
+        `[sync] synclabs not yet implemented, falling back to overlay`,
+      );
+      return await lipsyncOverlay(videoUrl, audioUrl, output);
+
+    case "overlay":
+      return await lipsyncOverlay(videoUrl, audioUrl, output);
+
+    default:
+      throw new Error(`unknown lipsync method: ${method}`);
+  }
+}
+
+export async function lipsyncWav2Lip(options: Wav2LipOptions) {
+  const { videoUrl, audioUrl } = options;
+
+  console.log(`[sync] using wav2lip model...`);
+
+  try {
+    const output = await runModel("devxpy/cog-wav2lip", {
+      face: videoUrl,
+      audio: audioUrl,
+    });
+
+    console.log(`[sync] wav2lip completed`);
+    return output;
+  } catch (error) {
+    console.error(`[sync] wav2lip error:`, error);
+    throw error;
+  }
+}
+
+export async function lipsyncOverlay(
+  videoPath: string,
+  audioPath: string,
+  output: string = "output-synced.mp4",
+) {
+  console.log(`[sync] overlaying audio on video...`);
+
+  try {
+    const result = await addAudio({
+      videoPath,
+      audioPath,
+      output,
+    });
+
+    console.log(`[sync] overlay completed`);
+    return result;
+  } catch (error) {
+    console.error(`[sync] overlay error:`, error);
+    throw error;
+  }
+}
+
+// cli
+async function cli() {
+  const args = process.argv.slice(2);
+  const command = args[0];
+
+  if (!command || command === "help") {
+    console.log(`
+usage:
+  bun run service/sync.ts <command> [args]
+
+commands:
+  sync <videoUrl> <audioUrl> [method] [output]     sync video with audio
+  wav2lip <videoUrl> <audioUrl>                    use wav2lip model
+  overlay <videoPath> <audioPath> [output]         simple audio overlay
+  help                                             show this help
+
+methods:
+  wav2lip    - ai-powered lipsync using replicate (url inputs)
+  overlay    - simple audio overlay using ffmpeg (local files)
+
+examples:
+  bun run service/sync.ts sync video.mp4 audio.mp3 overlay output.mp4
+  bun run service/sync.ts wav2lip https://example.com/video.mp4 https://example.com/audio.mp3
+  bun run service/sync.ts overlay video.mp4 audio.mp3 synced.mp4
+
+environment:
+  REPLICATE_API_TOKEN - required for wav2lip method
+    `);
+    process.exit(0);
+  }
+
+  try {
+    switch (command) {
+      case "sync": {
+        const videoUrl = args[1];
+        const audioUrl = args[2];
+        const method = (args[3] || "overlay") as "wav2lip" | "overlay";
+        const output = args[4];
+
+        if (!videoUrl || !audioUrl) {
+          throw new Error("videoUrl and audioUrl are required");
+        }
+
+        const result = await lipsync({
+          videoUrl,
+          audioUrl,
+          method,
+          output,
+        });
+
+        console.log(`[sync] result:`, result);
+        break;
+      }
+
+      case "wav2lip": {
+        const videoUrl = args[1];
+        const audioUrl = args[2];
+
+        if (!videoUrl || !audioUrl) {
+          throw new Error("videoUrl and audioUrl are required");
+        }
+
+        const result = await lipsyncWav2Lip({ videoUrl, audioUrl });
+        console.log(`[sync] result:`, result);
+        break;
+      }
+
+      case "overlay": {
+        const videoPath = args[1];
+        const audioPath = args[2];
+        const output = args[3];
+
+        if (!videoPath || !audioPath) {
+          throw new Error("videoPath and audioPath are required");
+        }
+
+        const result = await lipsyncOverlay(videoPath, audioPath, output);
+        console.log(`[sync] result:`, result);
+        break;
+      }
+
+      default:
+        console.error(`unknown command: ${command}`);
+        console.log(`run 'bun run service/sync.ts help' for usage`);
+        process.exit(1);
+    }
+  } catch (error) {
+    console.error(`[sync] error:`, error);
+    process.exit(1);
+  }
+}
+
+if (import.meta.main) {
+  cli();
+}