varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/src/definitions/actions/index.ts

@@ -0,0 +1,114 @@
+/**
+ * Action definitions index
+ */
+
+export type { AddCaptionsOptions, SubtitleStyle } from "./captions";
+// Captions
+export { addCaptions, definition as captions } from "./captions";
+export type {
+  CutOptions,
+  CutResult,
+  FadeOptions,
+  FadeResult,
+  MergeOptions,
+  MergeResult,
+  RemoveOptions,
+  RemoveResult,
+  SplitOptions,
+  SplitResult,
+  TransitionOptions,
+  TransitionResult,
+  TrimOptions,
+  TrimResult,
+} from "./edit";
+// Video editing (FFmpeg)
+export {
+  cut,
+  cutDefinition,
+  fade,
+  fadeDefinition,
+  merge,
+  mergeDefinition,
+  remove,
+  removeDefinition,
+  split,
+  splitDefinition,
+  transition,
+  transitionDefinition,
+  trim,
+  trimDefinition,
+} from "./edit";
+export type { ImageGenerationResult } from "./image";
+// Image generation
+export {
+  definition as image,
+  generateWithFal,
+  generateWithSoul,
+} from "./image";
+export type { GenerateMusicOptions, MusicResult } from "./music";
+// Music generation
+export { definition as music, generateMusic } from "./music";
+export type { LipsyncOptions, LipsyncResult, Wav2LipOptions } from "./sync";
+// Lip sync
+export {
+  definition as sync,
+  lipsync,
+  lipsyncOverlay,
+  lipsyncWav2Lip,
+} from "./sync";
+export type { TranscribeOptions, TranscribeResult } from "./transcribe";
+// Transcription
+export {
+  definition as transcribe,
+  transcribe as transcribeAudio,
+} from "./transcribe";
+export type { UploadOptions, UploadResult } from "./upload";
+// Upload
+export { definition as uploadDef, upload } from "./upload";
+export type { VideoGenerationResult } from "./video";
+// Video generation
+export {
+  definition as video,
+  generateVideoFromImage,
+  generateVideoFromText,
+} from "./video";
+export type { GenerateVoiceOptions, VoiceResult } from "./voice";
+// Voice generation
+export { definition as voice, generateVoice } from "./voice";
+
+// All action definitions for auto-loading
+import { definition as captionsDefinition } from "./captions";
+import {
+  cutDefinition,
+  fadeDefinition,
+  mergeDefinition,
+  removeDefinition,
+  splitDefinition,
+  transitionDefinition,
+  trimDefinition,
+} from "./edit";
+import { definition as imageDefinition } from "./image";
+import { definition as musicDefinition } from "./music";
+import { definition as syncDefinition } from "./sync";
+import { definition as transcribeDefinition } from "./transcribe";
+import { definition as uploadDefinition } from "./upload";
+import { definition as videoDefinition } from "./video";
+import { definition as voiceDefinition } from "./voice";
+
+export const allActions = [
+  videoDefinition,
+  imageDefinition,
+  voiceDefinition,
+  transcribeDefinition,
+  musicDefinition,
+  syncDefinition,
+  captionsDefinition,
+  trimDefinition,
+  cutDefinition,
+  mergeDefinition,
+  splitDefinition,
+  fadeDefinition,
+  transitionDefinition,
+  removeDefinition,
+  uploadDefinition,
+];

package/src/definitions/actions/music.ts

@@ -0,0 +1,205 @@
+/**
+ * Music generation action
+ * Text-to-music via Fal/Sonauto
+ */
+
+import { writeFile } from "node:fs/promises";
+import { z } from "zod";
+import { audioFormatSchema, filePathSchema } from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { falProvider } from "../../providers/fal";
+import { storageProvider } from "../../providers/storage";
+
+// Input schema with Zod
+const musicInputSchema = z.object({
+  prompt: z.string().optional().describe("Description of music to generate"),
+  tags: z
+    .array(z.string())
+    .optional()
+    .describe("Style tags like 'rock', 'energetic'"),
+  lyrics: z.string().optional().describe("Optional lyrics prompt"),
+  format: audioFormatSchema.default("mp3").describe("Output format"),
+  numSongs: z
+    .union([z.literal(1), z.literal(2)])
+    .default(1)
+    .describe("Number of songs to generate"),
+  output: filePathSchema.optional().describe("Output file path"),
+});
+
+// Output schema with Zod
+const musicOutputSchema = z.object({
+  seed: z.number(),
+  tags: z.array(z.string()).optional(),
+  lyrics: z.string().optional(),
+  audio: z.array(
+    z.object({
+      url: z.string(),
+      fileName: z.string(),
+      contentType: z.string(),
+      fileSize: z.number(),
+    }),
+  ),
+  uploadUrls: z.array(z.string()).optional(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof musicInputSchema, typeof musicOutputSchema> = {
+  input: musicInputSchema,
+  output: musicOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "music",
+  description: "Generate music from text prompt or tags",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    return generateMusic({
+      prompt: inputs.prompt,
+      tags: inputs.tags,
+      lyrics: inputs.lyrics,
+      format: inputs.format,
+      numSongs: inputs.numSongs,
+      outputPath: inputs.output,
+    });
+  },
+};
+
+// Types
+export interface GenerateMusicOptions {
+  prompt?: string;
+  tags?: string[];
+  lyrics?: string;
+  seed?: number;
+  promptStrength?: number;
+  balanceStrength?: number;
+  numSongs?: 1 | 2;
+  format?: "flac" | "mp3" | "wav" | "ogg" | "m4a";
+  bitRate?: 128 | 192 | 256 | 320;
+  bpm?: number | "auto";
+  upload?: boolean;
+  outputPath?: string;
+}
+
+export interface MusicResult {
+  seed: number;
+  tags?: string[];
+  lyrics?: string;
+  audio: Array<{
+    url: string;
+    fileName: string;
+    contentType: string;
+    fileSize: number;
+  }>;
+  uploadUrls?: string[];
+}
+
+export async function generateMusic(
+  options: GenerateMusicOptions,
+): Promise<MusicResult> {
+  const {
+    prompt,
+    tags,
+    lyrics,
+    seed,
+    promptStrength = 2,
+    balanceStrength = 0.7,
+    numSongs = 1,
+    format = "mp3",
+    bitRate,
+    bpm = "auto",
+    upload = false,
+    outputPath,
+  } = options;
+
+  if (!prompt && !tags) {
+    throw new Error("Either prompt or tags is required");
+  }
+
+  console.log(`[music] generating ${numSongs} song(s)...`);
+  if (prompt) console.log(`[music] prompt: ${prompt}`);
+  if (tags) console.log(`[music] tags: ${tags.join(", ")}`);
+
+  const result = await falProvider.textToMusic({
+    prompt,
+    tags,
+    lyricsPrompt: lyrics,
+    seed,
+    promptStrength,
+    balanceStrength,
+    numSongs,
+    outputFormat: format,
+    outputBitRate: bitRate,
+    bpm,
+  });
+
+  const musicResult: MusicResult = {
+    seed: result.data.seed,
+    tags: result.data.tags,
+    lyrics: result.data.lyrics,
+    audio: Array.isArray(result.data.audio)
+      ? result.data.audio.map(
+          (a: {
+            url: string;
+            file_name: string;
+            content_type: string;
+            file_size: number;
+          }) => ({
+            url: a.url,
+            fileName: a.file_name,
+            contentType: a.content_type,
+            fileSize: a.file_size,
+          }),
+        )
+      : [
+          {
+            url: result.data.audio.url,
+            fileName: result.data.audio.file_name,
+            contentType: result.data.audio.content_type,
+            fileSize: result.data.audio.file_size,
+          },
+        ],
+  };
+
+  // Save files locally if requested
+  if (outputPath) {
+    for (let i = 0; i < musicResult.audio.length; i++) {
+      const audio = musicResult.audio[i];
+      if (!audio) continue;
+
+      const ext = format || "wav";
+      const filePath =
+        musicResult.audio.length === 1
+          ? outputPath
+          : outputPath.replace(/\.[^.]+$/, `-${i + 1}.${ext}`);
+
+      const response = await fetch(audio.url);
+      const buffer = await response.arrayBuffer();
+      await writeFile(filePath, Buffer.from(buffer));
+      console.log(`[music] saved to ${filePath}`);
+    }
+  }
+
+  // Upload to storage if requested
+  if (upload) {
+    const uploadUrls: string[] = [];
+    for (let i = 0; i < musicResult.audio.length; i++) {
+      const audio = musicResult.audio[i];
+      if (!audio) continue;
+
+      const objectKey = `music/${Date.now()}-${i + 1}.${format || "wav"}`;
+      const uploadUrl = await storageProvider.uploadFromUrl(
+        audio.url,
+        objectKey,
+      );
+      uploadUrls.push(uploadUrl);
+      console.log(`[music] uploaded to ${uploadUrl}`);
+    }
+    musicResult.uploadUrls = uploadUrls;
+  }
+
+  return musicResult;
+}
+
+export default definition;

package/src/definitions/actions/sync.ts

@@ -0,0 +1,128 @@
+/**
+ * Lip sync action
+ * Audio-to-video synchronization
+ */
+
+import { z } from "zod";
+import {
+  filePathSchema,
+  resolutionSchema,
+  videoDurationStringSchema,
+} from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { falProvider } from "../../providers/fal";
+import { ffmpegProvider } from "../../providers/ffmpeg";
+
+// Input schema with Zod
+const syncInputSchema = z.object({
+  image: filePathSchema.describe("Input image"),
+  audio: filePathSchema.describe("Audio file"),
+  prompt: z.string().describe("Description of the scene"),
+  duration: videoDurationStringSchema.default("5").describe("Output duration"),
+  resolution: resolutionSchema.default("480p").describe("Output resolution"),
+});
+
+// Output schema with Zod
+const syncOutputSchema = z.object({
+  videoUrl: z.string(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof syncInputSchema, typeof syncOutputSchema> = {
+  input: syncInputSchema,
+  output: syncOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "sync",
+  description: "Lip sync audio to video/image",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    const { image, audio, prompt, duration, resolution } = inputs;
+    return lipsync({ image, audio, prompt, duration, resolution });
+  },
+};
+
+// Types
+export interface LipsyncOptions {
+  image: string;
+  audio: string;
+  prompt: string;
+  duration?: "5" | "10";
+  resolution?: "480p" | "720p" | "1080p";
+}
+
+export interface LipsyncResult {
+  videoUrl: string;
+}
+
+export interface Wav2LipOptions {
+  videoPath: string;
+  audioPath: string;
+  outputPath: string;
+}
+
+/**
+ * Generate lip-synced video using Wan-25
+ */
+export async function lipsync(options: LipsyncOptions): Promise<LipsyncResult> {
+  const { image, audio, prompt, duration = "5", resolution = "480p" } = options;
+
+  console.log("[sync] generating lip-synced video with wan-25...");
+
+  const result = await falProvider.wan25({
+    imageUrl: image,
+    audioUrl: audio,
+    prompt,
+    duration,
+    resolution,
+  });
+
+  const videoUrl = result.data?.video?.url;
+  if (!videoUrl) {
+    throw new Error("No video URL in result");
+  }
+
+  return { videoUrl };
+}
+
+/**
+ * Overlay lip-synced face onto original video
+ */
+export async function lipsyncOverlay(options: {
+  originalVideo: string;
+  lipsyncedVideo: string;
+  outputPath: string;
+}): Promise<string> {
+  const { lipsyncedVideo, outputPath } = options;
+
+  console.log("[sync] overlaying lip-synced video...");
+
+  // This would require more complex ffmpeg operations
+  // For now, just return the lip-synced video as-is
+  await ffmpegProvider.convertFormat({
+    input: lipsyncedVideo,
+    output: outputPath,
+  });
+
+  return outputPath;
+}
+
+/**
+ * Wav2Lip-style lip sync (placeholder for future implementation)
+ */
+export async function lipsyncWav2Lip(options: Wav2LipOptions): Promise<string> {
+  console.warn("[sync] wav2lip not yet implemented, using wan-25 fallback");
+
+  // For now, just copy the video
+  await ffmpegProvider.convertFormat({
+    input: options.videoPath,
+    output: options.outputPath,
+  });
+
+  return options.outputPath;
+}
+
+export default definition;

package/{action/transcribe/index.ts → src/definitions/actions/transcribe.ts}

@@ -1,63 +1,63 @@
-#!/usr/bin/env bun
-
 /**
- * audio transcription service
- * supports groq whisper, fireworks api, and future providers
+ * Transcription action
+ * Speech-to-text via Groq or Fireworks
  */
 
 import { writeFileSync } from "node:fs";
 import { toFile } from "groq-sdk/uploads";
-import type { ActionMeta } from "../../cli/types";
+import { z } from "zod";
+import {
+  filePathSchema,
+  transcriptionProviderSchema,
+} from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
 import {
   convertFireworksToSRT,
-  transcribeWithFireworks as fireworksTranscribe,
-} from "../../lib/fireworks";
-import { GROQ_MODELS, transcribeAudio as groqTranscribe } from "../../lib/groq";
+  fireworksProvider,
+} from "../../providers/fireworks";
+import { GROQ_MODELS, groqProvider } from "../../providers/groq";
+
+// Input schema with Zod
+const transcribeInputSchema = z.object({
+  audio: filePathSchema.describe("Audio/video file to transcribe"),
+  provider: transcriptionProviderSchema
+    .default("groq")
+    .describe("Transcription provider"),
+  output: filePathSchema.optional().describe("Output file path"),
+});
+
+// Output schema with Zod
+const transcribeOutputSchema = z.object({
+  success: z.boolean(),
+  text: z.string().optional(),
+  srt: z.string().optional(),
+  error: z.string().optional(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<
+  typeof transcribeInputSchema,
+  typeof transcribeOutputSchema
+> = {
+  input: transcribeInputSchema,
+  output: transcribeOutputSchema,
+};
 
-export const meta: ActionMeta = {
-  name: "transcribe",
+export const definition: ActionDefinition<typeof schema> = {
   type: "action",
-  description: "speech to text transcription",
-  inputType: "audio",
-  outputType: "text",
-  schema: {
-    input: {
-      type: "object",
-      required: ["audio"],
-      properties: {
-        audio: {
-          type: "string",
-          format: "file-path",
-          description: "audio/video file to transcribe",
-        },
-        provider: {
-          type: "string",
-          enum: ["groq", "fireworks"],
-          default: "groq",
-          description: "transcription provider",
-        },
-        output: {
-          type: "string",
-          format: "file-path",
-          description: "output file path",
-        },
-      },
-    },
-    output: { type: "string", description: "transcribed text" },
-  },
-  async run(options) {
-    const { audio, provider, output } = options as {
-      audio: string;
-      provider?: "groq" | "fireworks";
-      output?: string;
-    };
+  name: "transcribe",
+  description: "Speech to text transcription",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    const { audio, provider, output } = inputs;
     return transcribe({ audioUrl: audio, provider, outputPath: output });
   },
 };
 
-// types
+// Types
 export interface TranscribeOptions {
-  audioUrl: string; // url or local file path
+  audioUrl: string;
   provider?: "groq" | "fireworks";
   model?: string;
   language?: string;
@@ -72,7 +72,7 @@ export interface TranscribeResult {
   error?: string;
 }
 
-// groq transcription
+// Groq transcription
 async function transcribeWithGroq(
   audioUrl: string,
   options: {
@@ -84,16 +84,14 @@ async function transcribeWithGroq(
   try {
     console.log("[transcribe] using groq whisper...");
 
-    // load audio file (local or remote)
+    // Load audio file
     let audioBuffer: ArrayBuffer;
     let fileName = "audio.mp3";
 
     if (audioUrl.startsWith("http://") || audioUrl.startsWith("https://")) {
-      // fetch remote file
       const audioResponse = await fetch(audioUrl);
       audioBuffer = await audioResponse.arrayBuffer();
     } else {
-      // read local file with bun
       const file = Bun.file(audioUrl);
       audioBuffer = await file.arrayBuffer();
       fileName = audioUrl.split("/").pop() || "audio.mp3";
@@ -101,8 +99,7 @@ async function transcribeWithGroq(
 
     const audioFile = await toFile(audioBuffer, fileName);
 
-    // transcribe with groq
-    const text = await groqTranscribe({
+    const text = await groqProvider.transcribeAudio({
       file: audioFile,
       model: options.model || GROQ_MODELS.WHISPER_LARGE,
       language: options.language,
@@ -111,10 +108,8 @@ async function transcribeWithGroq(
     console.log("[transcribe] groq transcription complete");
 
     if (options.outputFormat === "srt") {
-      // groq returns plain text, so we need to convert to srt
-      // for now just return text with warning
       console.warn(
-        "[transcribe] groq returns plain text, use fireworks for srt format",
+        "[transcribe] groq returns plain text, use fireworks for SRT format",
       );
       return { success: true, text, srt: text };
     }
@@ -125,19 +120,19 @@ async function transcribeWithGroq(
     return {
       success: false,
       error:
-        error instanceof Error ? error.message : "groq transcription failed",
+        error instanceof Error ? error.message : "Groq transcription failed",
     };
   }
 }
 
-// fireworks transcription (with srt support)
+// Fireworks transcription (with SRT support)
 async function transcribeWithFireworks(
   audioUrl: string,
 ): Promise<TranscribeResult> {
   try {
     console.log("[transcribe] using fireworks api...");
 
-    const data = await fireworksTranscribe({
+    const data = await fireworksProvider.transcribe({
       audioPath: audioUrl,
     });
 
@@ -152,12 +147,12 @@ async function transcribeWithFireworks(
       error:
         error instanceof Error
           ? error.message
-          : "fireworks transcription failed",
+          : "Fireworks transcription failed",
     };
   }
 }
 
-// main transcription function
+// Main transcription function
 export async function transcribe(
   options: TranscribeOptions,
 ): Promise<TranscribeResult> {
@@ -178,7 +173,6 @@ export async function transcribe(
 
   let result: TranscribeResult;
 
-  // choose provider
   if (provider === "groq") {
     result = await transcribeWithGroq(audioUrl, {
       model,
@@ -188,10 +182,10 @@ export async function transcribe(
   } else if (provider === "fireworks") {
     result = await transcribeWithFireworks(audioUrl);
   } else {
-    throw new Error(`unknown provider: ${provider}`);
+    throw new Error(`Unknown provider: ${provider}`);
   }
 
-  // save to file if requested
+  // Save to file if requested
   if (result.success && outputPath) {
     const content = outputFormat === "srt" ? result.srt : result.text;
     if (content) {
@@ -203,8 +197,4 @@ export async function transcribe(
   return result;
 }
 
-// cli
-if (import.meta.main) {
-  const { runCli } = await import("../../cli/runner");
-  runCli(meta);
-}
+export default definition;