varg.ai-sdk 0.1.0

package/action/voice/SKILL.md

@@ -0,0 +1,125 @@
+---
+name: voice-synthesis
+description: generate realistic text-to-speech audio using elevenlabs with multiple voice options. use when user needs voiceovers, narration, character voices, or audio for lipsync videos.
+allowed-tools: Read, Bash
+---
+
+# voice synthesis
+
+generate high-quality text-to-speech audio with elevenlabs.
+
+## available voices
+
+- **rachel** - clear, professional female voice
+- **domi** - warm, friendly female voice
+- **bella** - energetic female voice
+- **antoni** - friendly male voice
+- **elli** - young, clear female voice
+- **josh** - deep, clear male voice
+- **arnold** - strong, authoritative male voice
+- **adam** - natural, conversational male voice
+- **sam** - raspy, character male voice
+
+## usage
+
+### generate voice
+```bash
+bun run service/voice.ts generate <text> [voice] [provider] [upload]
+```
+
+**parameters:**
+- `text` (required): text to convert to speech
+- `voice` (optional): voice name (default: rachel)
+- `provider` (optional): elevenlabs (default)
+- `upload` (optional): "true" to upload to s3
+
+**example:**
+```bash
+bun run service/voice.ts generate "hello world, this is my voice" rachel elevenlabs true
+```
+
+### shorthand for elevenlabs
+```bash
+bun run service/voice.ts elevenlabs <text> [voice] [upload]
+```
+
+**example:**
+```bash
+bun run service/voice.ts elevenlabs "welcome to our video" josh true
+```
+
+## as library
+
+```typescript
+import { generateVoice } from "./service/voice"
+
+const result = await generateVoice({
+  text: "hello world",
+  voice: "rachel",
+  provider: "elevenlabs",
+  upload: true,
+  outputPath: "media/voiceover.mp3"
+})
+
+console.log(result.provider)
+console.log(result.voiceId)
+console.log(result.uploadUrl)
+```
+
+## output
+
+returns `VoiceResult`:
+```typescript
+{
+  audio: Buffer,         // raw audio buffer
+  provider: string,      // "elevenlabs"
+  voiceId: string,       // actual voice id used
+  uploadUrl?: string     // s3 url if upload requested
+}
+```
+
+saves audio file to `media/voice-{timestamp}.mp3`
+
+## when to use
+
+use this skill when:
+- creating voiceovers for videos
+- generating narration or character dialogue
+- preparing audio for lipsync videos
+- need text-to-speech for talking character pipeline
+- testing different voice options
+
+## tips
+
+**voice selection:**
+- use **rachel** or **josh** for professional narration
+- use **bella** or **antoni** for friendly, casual content
+- use **arnold** for authoritative or dramatic content
+- use **sam** for character or stylized voices
+
+**text formatting:**
+- add punctuation for natural pauses
+- use shorter sentences for clearer speech
+- spell out numbers and abbreviations
+
+## integration with other services
+
+perfect companion for:
+- **lipsync service** - sync generated voice with video
+- **video generation** - create talking character videos
+- **captions service** - auto-generate subtitles from voiceover
+
+## environment variables
+
+required:
+- `ELEVENLABS_API_KEY` - for voice generation
+
+optional (for s3 upload):
+- `CLOUDFLARE_R2_API_URL`
+- `CLOUDFLARE_ACCESS_KEY_ID`
+- `CLOUDFLARE_ACCESS_SECRET`
+- `CLOUDFLARE_R2_BUCKET`
+
+## generation time
+
+expect 5-15 seconds depending on text length

package/action/voice/index.ts

@@ -0,0 +1,201 @@
+#!/usr/bin/env bun
+
+/**
+ * voice service - high-level voice generation combining multiple providers
+ * supports elevenlabs and future providers
+ */
+
+import { textToSpeech, VOICES } from "../../lib/elevenlabs";
+import { uploadFile } from "../../utilities/s3";
+
+// types
+export interface GenerateVoiceOptions {
+  text: string;
+  voice?: string;
+  provider?: "elevenlabs";
+  upload?: boolean;
+  outputPath?: string;
+}
+
+export interface VoiceResult {
+  audio: Buffer;
+  provider: string;
+  voiceId: string;
+  uploadUrl?: string;
+}
+
+// core functions
+export async function generateVoice(
+  options: GenerateVoiceOptions,
+): Promise<VoiceResult> {
+  const {
+    text,
+    voice = "rachel",
+    provider = "elevenlabs",
+    upload = false,
+    outputPath,
+  } = options;
+
+  if (!text) {
+    throw new Error("text is required");
+  }
+
+  console.log(`[voice] generating with ${provider} (${voice})...`);
+
+  let audio: Buffer;
+  let voiceId: string;
+
+  switch (provider) {
+    case "elevenlabs": {
+      // map friendly names to voice ids
+      const voiceMap: Record<string, string> = {
+        rachel: VOICES.RACHEL,
+        domi: VOICES.DOMI,
+        bella: VOICES.BELLA,
+        antoni: VOICES.ANTONI,
+        elli: VOICES.ELLI,
+        josh: VOICES.JOSH,
+        arnold: VOICES.ARNOLD,
+        adam: VOICES.ADAM,
+        sam: VOICES.SAM,
+      };
+
+      voiceId = voiceMap[voice.toLowerCase()] || voice;
+
+      audio = await textToSpeech({
+        text,
+        voiceId,
+        outputPath,
+      });
+      break;
+    }
+
+    default:
+      throw new Error(`unsupported provider: ${provider}`);
+  }
+
+  const result: VoiceResult = {
+    audio,
+    provider,
+    voiceId,
+  };
+
+  // upload to s3 if requested
+  if (upload && outputPath) {
+    const objectKey = `voice/${Date.now()}-${voice}.mp3`;
+    const uploadUrl = await uploadFile(outputPath, objectKey);
+    result.uploadUrl = uploadUrl;
+    console.log(`[voice] uploaded to ${uploadUrl}`);
+  }
+
+  return result;
+}
+
+// cli
+async function cli() {
+  const args = process.argv.slice(2);
+  const command = args[0];
+
+  if (!command || command === "help") {
+    console.log(`
+usage:
+  bun run service/voice.ts <command> [args]
+
+commands:
+  generate <text> [voice] [provider] [upload]    generate voice from text
+  elevenlabs <text> [voice] [upload]             generate with elevenlabs
+  help                                           show this help
+
+examples:
+  bun run service/voice.ts generate "hello world" rachel elevenlabs false
+  bun run service/voice.ts elevenlabs "hello world" josh true
+  bun run service/voice.ts generate "welcome to ai" bella
+
+available voices:
+  rachel, domi, bella, antoni, elli, josh, arnold, adam, sam
+
+providers:
+  elevenlabs (default)
+
+environment:
+  ELEVENLABS_API_KEY - required for elevenlabs
+  CLOUDFLARE_* - required for upload
+    `);
+    process.exit(0);
+  }
+
+  try {
+    switch (command) {
+      case "generate": {
+        const text = args[1];
+        const voice = args[2];
+        const provider = (args[3] || "elevenlabs") as "elevenlabs";
+        const upload = args[4] === "true";
+
+        if (!text) {
+          throw new Error("text is required");
+        }
+
+        const outputPath = `media/voice-${Date.now()}.mp3`;
+
+        const result = await generateVoice({
+          text,
+          voice,
+          provider,
+          upload,
+          outputPath,
+        });
+
+        console.log(`[voice] result:`, {
+          provider: result.provider,
+          voiceId: result.voiceId,
+          audioSize: result.audio.length,
+          outputPath,
+          uploadUrl: result.uploadUrl,
+        });
+        break;
+      }
+
+      case "elevenlabs": {
+        const text = args[1];
+        const voice = args[2];
+        const upload = args[3] === "true";
+
+        if (!text) {
+          throw new Error("text is required");
+        }
+
+        const outputPath = `media/voice-${Date.now()}.mp3`;
+
+        const result = await generateVoice({
+          text,
+          voice,
+          provider: "elevenlabs",
+          upload,
+          outputPath,
+        });
+
+        console.log(`[voice] result:`, {
+          provider: result.provider,
+          voiceId: result.voiceId,
+          audioSize: result.audio.length,
+          outputPath,
+          uploadUrl: result.uploadUrl,
+        });
+        break;
+      }
+
+      default:
+        console.error(`unknown command: ${command}`);
+        console.log(`run 'bun run service/voice.ts help' for usage`);
+        process.exit(1);
+    }
+  } catch (error) {
+    console.error(`[voice] error:`, error);
+    process.exit(1);
+  }
+}
+
+if (import.meta.main) {
+  cli();
+}

package/biome.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "https://biomejs.dev/schemas/2.3.7/schema.json",
+  "vcs": {
+    "enabled": true,
+    "clientKind": "git",
+    "useIgnoreFile": true
+  },
+  "files": {
+    "ignoreUnknown": false
+  },
+  "formatter": {
+    "enabled": true,
+    "indentStyle": "space"
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true
+    }
+  },
+  "assist": {
+    "actions": {
+      "source": {
+        "organizeImports": "on"
+      }
+    }
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "double"
+    }
+  }
+}

package/index.ts

@@ -0,0 +1,38 @@
+/**
+ * varg.ai sdk
+ * video generation and editing tools
+ */
+
+// re-export external clients
+export { fal } from "@ai-sdk/fal";
+export { replicate } from "@ai-sdk/replicate";
+export { fal as falClient } from "@fal-ai/client";
+export { HiggsfieldClient } from "@higgsfield/client";
+// lib exports - ai-sdk/fal (provider)
+export * as aiSdkFal from "./lib/ai-sdk/fal";
+// lib exports - ai-sdk/replicate (provider)
+export * as aiSdkReplicate from "./lib/ai-sdk/replicate";
+// lib exports - elevenlabs
+export * from "./lib/elevenlabs";
+// lib exports - fal (client)
+export * from "./lib/fal";
+// lib exports - ffmpeg
+export * from "./lib/ffmpeg";
+// lib exports - fireworks
+export * from "./lib/fireworks";
+// lib exports - groq
+export * from "./lib/groq";
+// lib exports - higgsfield
+export * from "./lib/higgsfield";
+// lib exports - replicate
+export * from "./lib/replicate";
+// service exports
+export * from "./service/captions";
+export * from "./service/edit";
+export * from "./service/image";
+export * from "./service/sync";
+export * from "./service/transcribe";
+export * from "./service/video";
+export * from "./service/voice";
+// utilities exports
+export * from "./utilities/s3";

package/lib/README.md

@@ -0,0 +1,144 @@
+# lib/ modules
+
+## two fal implementations
+
+### lib/ai-sdk/fal.ts - ai-sdk provider (recommended for images)
+
+uses `@ai-sdk/fal` with the vercel ai sdk's `experimental_generateImage`
+
+**benefits:**
+- clean, typed api via vercel ai sdk
+- automatic image format handling (uint8array)
+- consistent interface with other ai providers
+- built-in aspect ratio support
+- better for standard image generation
+
+**example:**
+```bash
+bun run lib/ai-sdk/fal.ts generate_image "cyberpunk city" "fal-ai/flux/dev" "16:9"
+```
+
+**code:**
+```typescript
+import { fal } from "@ai-sdk/fal"
+import { experimental_generateImage as generateImage } from "ai"
+
+const { image, providerMetadata } = await generateImage({
+  model: fal.image("fal-ai/flux/dev"),
+  prompt: "beautiful sunset",
+  aspectRatio: "16:9",
+})
+```
+
+### lib/fal.ts - fal client direct (for video & advanced features)
+
+uses `@fal-ai/client` directly with the raw fal api
+
+**benefits:**
+- access to all fal features (video, advanced params)
+- streaming/queue updates
+- full control over api parameters
+- required for video generation (no ai-sdk support yet)
+- **supports local images** - automatically uploads local files to fal storage
+
+**examples:**
+```bash
+# image generation
+bun run lib/fal.ts generate_image "aurora borealis" "fal-ai/flux-pro/v1.1"
+
+# video from url
+bun run lib/fal.ts image_to_video "person talking" "https://image.url" 5
+
+# video from local file (auto-uploads)
+bun run lib/fal.ts image_to_video "ocean waves" "./media/beach.jpg" 10
+```
+
+**code:**
+```typescript
+import { imageToVideo } from "./lib/fal"
+
+// works with both urls and local files
+const result = await imageToVideo({
+  prompt: "person talking",
+  imageUrl: "./local/image.jpg", // or "https://..."
+  duration: 5,
+})
+
+// local files are automatically uploaded to fal storage
+```
+
+## when to use which?
+
+| use case | approach |
+|----------|----------|
+| standard image generation | ai-sdk provider ✓ |
+| video generation | fal client direct ✓ |
+| advanced fal features | fal client direct ✓ |
+| multi-provider app | ai-sdk provider ✓ |
+| custom queue handling | fal client direct ✓ |
+
+## higgsfield.ts
+
+uses `@higgsfield/client` for soul character generation
+
+**features:**
+- generate soul images with custom styles
+- create and manage character references
+- list available soul styles
+- poll for job completion
+
+**example:**
+```bash
+HF_API_KEY=xxx HF_API_SECRET=xxx bun run lib/higgsfield.ts generate_soul "professional headshot"
+```
+
+## elevenlabs.ts
+
+uses `@elevenlabs/elevenlabs-js` for voice, music, and sound effects generation
+
+**features:**
+- text-to-speech with multiple voices
+- music generation from text prompts
+- sound effects generation
+- voice management
+
+**examples:**
+```bash
+# text-to-speech
+bun run lib/elevenlabs.ts tts "hello world" rachel output.mp3
+
+# music generation
+bun run lib/elevenlabs.ts music "upbeat electronic dance music" 30000 music.mp3
+
+# sound effects
+bun run lib/elevenlabs.ts sfx "ocean waves crashing" 5 waves.mp3
+
+# list voices
+bun run lib/elevenlabs.ts voices
+```
+
+**code:**
+```typescript
+import { textToSpeech, generateMusic, generateSoundEffect } from "./lib/elevenlabs"
+
+// voice
+const audio = await textToSpeech({
+  text: "hello world",
+  voiceId: "rachel",
+  outputPath: "output.mp3"
+})
+
+// music
+const music = await generateMusic({
+  prompt: "epic orchestral music",
+  musicLengthMs: 60000,
+  outputPath: "music.mp3"
+})
+
+// sound effects
+const sfx = await generateSoundEffect({
+  text: "thunder and rain",
+  durationSeconds: 10,
+  outputPath: "sfx.mp3"
+})
+```

package/lib/ai-sdk/fal.ts

@@ -0,0 +1,106 @@
+#!/usr/bin/env bun
+/**
+ * fal.ai wrapper using @ai-sdk/fal provider
+ * recommended for standard image generation with vercel ai sdk
+ *
+ * usage: bun run lib/ai-sdk/fal.ts <command> <args>
+ */
+
+import { fal } from "@ai-sdk/fal";
+import { experimental_generateImage as generateImageAI } from "ai";
+
+export async function generateImage(args: {
+  prompt: string;
+  model?: string;
+  aspectRatio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4";
+}) {
+  const modelId = args.model || "fal-ai/flux/dev";
+
+  console.log(`[ai-sdk/fal] generating image with ${modelId}`);
+  console.log(`[ai-sdk/fal] prompt: ${args.prompt}`);
+  if (args.aspectRatio) {
+    console.log(`[ai-sdk/fal] aspect ratio: ${args.aspectRatio}`);
+  }
+
+  try {
+    const { image, providerMetadata } = await generateImageAI({
+      model: fal.image(modelId),
+      prompt: args.prompt,
+      aspectRatio: args.aspectRatio,
+    });
+
+    console.log("[ai-sdk/fal] completed!");
+
+    // return in consistent format
+    return {
+      image: {
+        url: image.base64 ? `data:image/png;base64,${image.base64}` : undefined,
+        uint8Array: image.uint8Array,
+      },
+      metadata: providerMetadata?.fal,
+    };
+  } catch (error) {
+    console.error("[ai-sdk/fal] error:", error);
+    throw error;
+  }
+}
+
+// cli runner
+if (import.meta.main) {
+  const [command, ...args] = process.argv.slice(2);
+
+  switch (command) {
+    case "generate_image": {
+      if (!args[0]) {
+        console.log(`
+usage:
+  bun run lib/ai-sdk/fal.ts generate_image <prompt> [model] [aspectRatio]
+
+examples:
+  bun run lib/ai-sdk/fal.ts generate_image "sunset over ocean" "fal-ai/flux/dev" "16:9"
+  bun run lib/ai-sdk/fal.ts generate_image "portrait photo" "fal-ai/flux-pro/v1.1" "9:16"
+
+available models:
+  - fal-ai/flux/dev (default, fast)
+  - fal-ai/flux-pro/v1.1 (high quality)
+  - fal-ai/flux/schnell (very fast)
+  - fal-ai/ideogram/character (character consistency)
+        `);
+        process.exit(1);
+      }
+
+      const result = await generateImage({
+        prompt: args[0],
+        model: args[1],
+        aspectRatio: args[2] as
+          | "1:1"
+          | "16:9"
+          | "9:16"
+          | "4:3"
+          | "3:4"
+          | undefined,
+      });
+
+      // save image to file
+      if (result.image.uint8Array) {
+        const filename = `/tmp/fal-ai-sdk-${Date.now()}.png`;
+        await Bun.write(filename, result.image.uint8Array);
+        console.log(`\nimage saved to: ${filename}`);
+
+        // open image
+        await Bun.spawn(["open", filename]);
+      }
+
+      console.log("\nmetadata:");
+      console.log(JSON.stringify(result.metadata, null, 2));
+      break;
+    }
+
+    default:
+      console.log(`
+usage:
+  bun run lib/ai-sdk/fal.ts generate_image <prompt> [model] [aspectRatio]
+      `);
+      process.exit(1);
+  }
+}