vargai 0.3.0

package/src/definitions/actions/index.ts

@@ -0,0 +1,114 @@
+/**
+ * Action definitions index
+ */
+
+export type { AddCaptionsOptions, SubtitleStyle } from "./captions";
+// Captions
+export { addCaptions, definition as captions } from "./captions";
+export type {
+  CutOptions,
+  CutResult,
+  FadeOptions,
+  FadeResult,
+  MergeOptions,
+  MergeResult,
+  RemoveOptions,
+  RemoveResult,
+  SplitOptions,
+  SplitResult,
+  TransitionOptions,
+  TransitionResult,
+  TrimOptions,
+  TrimResult,
+} from "./edit";
+// Video editing (FFmpeg)
+export {
+  cut,
+  cutDefinition,
+  fade,
+  fadeDefinition,
+  merge,
+  mergeDefinition,
+  remove,
+  removeDefinition,
+  split,
+  splitDefinition,
+  transition,
+  transitionDefinition,
+  trim,
+  trimDefinition,
+} from "./edit";
+export type { ImageGenerationResult } from "./image";
+// Image generation
+export {
+  definition as image,
+  generateWithFal,
+  generateWithSoul,
+} from "./image";
+export type { GenerateMusicOptions, MusicResult } from "./music";
+// Music generation
+export { definition as music, generateMusic } from "./music";
+export type { LipsyncOptions, LipsyncResult, Wav2LipOptions } from "./sync";
+// Lip sync
+export {
+  definition as sync,
+  lipsync,
+  lipsyncOverlay,
+  lipsyncWav2Lip,
+} from "./sync";
+export type { TranscribeOptions, TranscribeResult } from "./transcribe";
+// Transcription
+export {
+  definition as transcribe,
+  transcribe as transcribeAudio,
+} from "./transcribe";
+export type { UploadOptions, UploadResult } from "./upload";
+// Upload
+export { definition as uploadDef, upload } from "./upload";
+export type { VideoGenerationResult } from "./video";
+// Video generation
+export {
+  definition as video,
+  generateVideoFromImage,
+  generateVideoFromText,
+} from "./video";
+export type { GenerateVoiceOptions, VoiceResult } from "./voice";
+// Voice generation
+export { definition as voice, generateVoice } from "./voice";
+
+// All action definitions for auto-loading
+import { definition as captionsDefinition } from "./captions";
+import {
+  cutDefinition,
+  fadeDefinition,
+  mergeDefinition,
+  removeDefinition,
+  splitDefinition,
+  transitionDefinition,
+  trimDefinition,
+} from "./edit";
+import { definition as imageDefinition } from "./image";
+import { definition as musicDefinition } from "./music";
+import { definition as syncDefinition } from "./sync";
+import { definition as transcribeDefinition } from "./transcribe";
+import { definition as uploadDefinition } from "./upload";
+import { definition as videoDefinition } from "./video";
+import { definition as voiceDefinition } from "./voice";
+
+export const allActions = [
+  videoDefinition,
+  imageDefinition,
+  voiceDefinition,
+  transcribeDefinition,
+  musicDefinition,
+  syncDefinition,
+  captionsDefinition,
+  trimDefinition,
+  cutDefinition,
+  mergeDefinition,
+  splitDefinition,
+  fadeDefinition,
+  transitionDefinition,
+  removeDefinition,
+  uploadDefinition,
+];

package/src/definitions/actions/music.ts

@@ -0,0 +1,205 @@
+/**
+ * Music generation action
+ * Text-to-music via Fal/Sonauto
+ */
+
+import { writeFile } from "node:fs/promises";
+import { z } from "zod";
+import { audioFormatSchema, filePathSchema } from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { falProvider } from "../../providers/fal";
+import { storageProvider } from "../../providers/storage";
+
+// Input schema with Zod
+const musicInputSchema = z.object({
+  prompt: z.string().optional().describe("Description of music to generate"),
+  tags: z
+    .array(z.string())
+    .optional()
+    .describe("Style tags like 'rock', 'energetic'"),
+  lyrics: z.string().optional().describe("Optional lyrics prompt"),
+  format: audioFormatSchema.default("mp3").describe("Output format"),
+  numSongs: z
+    .union([z.literal(1), z.literal(2)])
+    .default(1)
+    .describe("Number of songs to generate"),
+  output: filePathSchema.optional().describe("Output file path"),
+});
+
+// Output schema with Zod
+const musicOutputSchema = z.object({
+  seed: z.number(),
+  tags: z.array(z.string()).optional(),
+  lyrics: z.string().optional(),
+  audio: z.array(
+    z.object({
+      url: z.string(),
+      fileName: z.string(),
+      contentType: z.string(),
+      fileSize: z.number(),
+    }),
+  ),
+  uploadUrls: z.array(z.string()).optional(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof musicInputSchema, typeof musicOutputSchema> = {
+  input: musicInputSchema,
+  output: musicOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "music",
+  description: "Generate music from text prompt or tags",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    return generateMusic({
+      prompt: inputs.prompt,
+      tags: inputs.tags,
+      lyrics: inputs.lyrics,
+      format: inputs.format,
+      numSongs: inputs.numSongs,
+      outputPath: inputs.output,
+    });
+  },
+};
+
+// Types
+export interface GenerateMusicOptions {
+  prompt?: string;
+  tags?: string[];
+  lyrics?: string;
+  seed?: number;
+  promptStrength?: number;
+  balanceStrength?: number;
+  numSongs?: 1 | 2;
+  format?: "flac" | "mp3" | "wav" | "ogg" | "m4a";
+  bitRate?: 128 | 192 | 256 | 320;
+  bpm?: number | "auto";
+  upload?: boolean;
+  outputPath?: string;
+}
+
+export interface MusicResult {
+  seed: number;
+  tags?: string[];
+  lyrics?: string;
+  audio: Array<{
+    url: string;
+    fileName: string;
+    contentType: string;
+    fileSize: number;
+  }>;
+  uploadUrls?: string[];
+}
+
+export async function generateMusic(
+  options: GenerateMusicOptions,
+): Promise<MusicResult> {
+  const {
+    prompt,
+    tags,
+    lyrics,
+    seed,
+    promptStrength = 2,
+    balanceStrength = 0.7,
+    numSongs = 1,
+    format = "mp3",
+    bitRate,
+    bpm = "auto",
+    upload = false,
+    outputPath,
+  } = options;
+
+  if (!prompt && !tags) {
+    throw new Error("Either prompt or tags is required");
+  }
+
+  console.log(`[music] generating ${numSongs} song(s)...`);
+  if (prompt) console.log(`[music] prompt: ${prompt}`);
+  if (tags) console.log(`[music] tags: ${tags.join(", ")}`);
+
+  const result = await falProvider.textToMusic({
+    prompt,
+    tags,
+    lyricsPrompt: lyrics,
+    seed,
+    promptStrength,
+    balanceStrength,
+    numSongs,
+    outputFormat: format,
+    outputBitRate: bitRate,
+    bpm,
+  });
+
+  const musicResult: MusicResult = {
+    seed: result.data.seed,
+    tags: result.data.tags,
+    lyrics: result.data.lyrics,
+    audio: Array.isArray(result.data.audio)
+      ? result.data.audio.map(
+          (a: {
+            url: string;
+            file_name: string;
+            content_type: string;
+            file_size: number;
+          }) => ({
+            url: a.url,
+            fileName: a.file_name,
+            contentType: a.content_type,
+            fileSize: a.file_size,
+          }),
+        )
+      : [
+          {
+            url: result.data.audio.url,
+            fileName: result.data.audio.file_name,
+            contentType: result.data.audio.content_type,
+            fileSize: result.data.audio.file_size,
+          },
+        ],
+  };
+
+  // Save files locally if requested
+  if (outputPath) {
+    for (let i = 0; i < musicResult.audio.length; i++) {
+      const audio = musicResult.audio[i];
+      if (!audio) continue;
+
+      const ext = format || "wav";
+      const filePath =
+        musicResult.audio.length === 1
+          ? outputPath
+          : outputPath.replace(/\.[^.]+$/, `-${i + 1}.${ext}`);
+
+      const response = await fetch(audio.url);
+      const buffer = await response.arrayBuffer();
+      await writeFile(filePath, Buffer.from(buffer));
+      console.log(`[music] saved to ${filePath}`);
+    }
+  }
+
+  // Upload to storage if requested
+  if (upload) {
+    const uploadUrls: string[] = [];
+    for (let i = 0; i < musicResult.audio.length; i++) {
+      const audio = musicResult.audio[i];
+      if (!audio) continue;
+
+      const objectKey = `music/${Date.now()}-${i + 1}.${format || "wav"}`;
+      const uploadUrl = await storageProvider.uploadFromUrl(
+        audio.url,
+        objectKey,
+      );
+      uploadUrls.push(uploadUrl);
+      console.log(`[music] uploaded to ${uploadUrl}`);
+    }
+    musicResult.uploadUrls = uploadUrls;
+  }
+
+  return musicResult;
+}
+
+export default definition;

package/src/definitions/actions/sync.ts

@@ -0,0 +1,128 @@
+/**
+ * Lip sync action
+ * Audio-to-video synchronization
+ */
+
+import { z } from "zod";
+import {
+  filePathSchema,
+  resolutionSchema,
+  videoDurationStringSchema,
+} from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { falProvider } from "../../providers/fal";
+import { ffmpegProvider } from "../../providers/ffmpeg";
+
+// Input schema with Zod
+const syncInputSchema = z.object({
+  image: filePathSchema.describe("Input image"),
+  audio: filePathSchema.describe("Audio file"),
+  prompt: z.string().describe("Description of the scene"),
+  duration: videoDurationStringSchema.default("5").describe("Output duration"),
+  resolution: resolutionSchema.default("480p").describe("Output resolution"),
+});
+
+// Output schema with Zod
+const syncOutputSchema = z.object({
+  videoUrl: z.string(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof syncInputSchema, typeof syncOutputSchema> = {
+  input: syncInputSchema,
+  output: syncOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "sync",
+  description: "Lip sync audio to video/image",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    const { image, audio, prompt, duration, resolution } = inputs;
+    return lipsync({ image, audio, prompt, duration, resolution });
+  },
+};
+
+// Types
+export interface LipsyncOptions {
+  image: string;
+  audio: string;
+  prompt: string;
+  duration?: "5" | "10";
+  resolution?: "480p" | "720p" | "1080p";
+}
+
+export interface LipsyncResult {
+  videoUrl: string;
+}
+
+export interface Wav2LipOptions {
+  videoPath: string;
+  audioPath: string;
+  outputPath: string;
+}
+
+/**
+ * Generate lip-synced video using Wan-25
+ */
+export async function lipsync(options: LipsyncOptions): Promise<LipsyncResult> {
+  const { image, audio, prompt, duration = "5", resolution = "480p" } = options;
+
+  console.log("[sync] generating lip-synced video with wan-25...");
+
+  const result = await falProvider.wan25({
+    imageUrl: image,
+    audioUrl: audio,
+    prompt,
+    duration,
+    resolution,
+  });
+
+  const videoUrl = result.data?.video?.url;
+  if (!videoUrl) {
+    throw new Error("No video URL in result");
+  }
+
+  return { videoUrl };
+}
+
+/**
+ * Overlay lip-synced face onto original video
+ */
+export async function lipsyncOverlay(options: {
+  originalVideo: string;
+  lipsyncedVideo: string;
+  outputPath: string;
+}): Promise<string> {
+  const { lipsyncedVideo, outputPath } = options;
+
+  console.log("[sync] overlaying lip-synced video...");
+
+  // This would require more complex ffmpeg operations
+  // For now, just return the lip-synced video as-is
+  await ffmpegProvider.convertFormat({
+    input: lipsyncedVideo,
+    output: outputPath,
+  });
+
+  return outputPath;
+}
+
+/**
+ * Wav2Lip-style lip sync (placeholder for future implementation)
+ */
+export async function lipsyncWav2Lip(options: Wav2LipOptions): Promise<string> {
+  console.warn("[sync] wav2lip not yet implemented, using wan-25 fallback");
+
+  // For now, just copy the video
+  await ffmpegProvider.convertFormat({
+    input: options.videoPath,
+    output: options.outputPath,
+  });
+
+  return options.outputPath;
+}
+
+export default definition;

package/src/definitions/actions/transcribe.ts

@@ -0,0 +1,200 @@
+/**
+ * Transcription action
+ * Speech-to-text via Groq or Fireworks
+ */
+
+import { writeFileSync } from "node:fs";
+import { toFile } from "groq-sdk/uploads";
+import { z } from "zod";
+import {
+  filePathSchema,
+  transcriptionProviderSchema,
+} from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import {
+  convertFireworksToSRT,
+  fireworksProvider,
+} from "../../providers/fireworks";
+import { GROQ_MODELS, groqProvider } from "../../providers/groq";
+
+// Input schema with Zod
+const transcribeInputSchema = z.object({
+  audio: filePathSchema.describe("Audio/video file to transcribe"),
+  provider: transcriptionProviderSchema
+    .default("groq")
+    .describe("Transcription provider"),
+  output: filePathSchema.optional().describe("Output file path"),
+});
+
+// Output schema with Zod
+const transcribeOutputSchema = z.object({
+  success: z.boolean(),
+  text: z.string().optional(),
+  srt: z.string().optional(),
+  error: z.string().optional(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<
+  typeof transcribeInputSchema,
+  typeof transcribeOutputSchema
+> = {
+  input: transcribeInputSchema,
+  output: transcribeOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "transcribe",
+  description: "Speech to text transcription",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    const { audio, provider, output } = inputs;
+    return transcribe({ audioUrl: audio, provider, outputPath: output });
+  },
+};
+
+// Types
+export interface TranscribeOptions {
+  audioUrl: string;
+  provider?: "groq" | "fireworks";
+  model?: string;
+  language?: string;
+  outputFormat?: "text" | "srt";
+  outputPath?: string;
+}
+
+export interface TranscribeResult {
+  success: boolean;
+  text?: string;
+  srt?: string;
+  error?: string;
+}
+
+// Groq transcription
+async function transcribeWithGroq(
+  audioUrl: string,
+  options: {
+    model?: string;
+    language?: string;
+    outputFormat?: "text" | "srt";
+  },
+): Promise<TranscribeResult> {
+  try {
+    console.log("[transcribe] using groq whisper...");
+
+    // Load audio file
+    let audioBuffer: ArrayBuffer;
+    let fileName = "audio.mp3";
+
+    if (audioUrl.startsWith("http://") || audioUrl.startsWith("https://")) {
+      const audioResponse = await fetch(audioUrl);
+      audioBuffer = await audioResponse.arrayBuffer();
+    } else {
+      const file = Bun.file(audioUrl);
+      audioBuffer = await file.arrayBuffer();
+      fileName = audioUrl.split("/").pop() || "audio.mp3";
+    }
+
+    const audioFile = await toFile(audioBuffer, fileName);
+
+    const text = await groqProvider.transcribeAudio({
+      file: audioFile,
+      model: options.model || GROQ_MODELS.WHISPER_LARGE,
+      language: options.language,
+    });
+
+    console.log("[transcribe] groq transcription complete");
+
+    if (options.outputFormat === "srt") {
+      console.warn(
+        "[transcribe] groq returns plain text, use fireworks for SRT format",
+      );
+      return { success: true, text, srt: text };
+    }
+
+    return { success: true, text };
+  } catch (error) {
+    console.error("[transcribe] groq error:", error);
+    return {
+      success: false,
+      error:
+        error instanceof Error ? error.message : "Groq transcription failed",
+    };
+  }
+}
+
+// Fireworks transcription (with SRT support)
+async function transcribeWithFireworks(
+  audioUrl: string,
+): Promise<TranscribeResult> {
+  try {
+    console.log("[transcribe] using fireworks api...");
+
+    const data = await fireworksProvider.transcribe({
+      audioPath: audioUrl,
+    });
+
+    const srtText = convertFireworksToSRT(data.words || []);
+    console.log("[transcribe] fireworks transcription complete");
+
+    return { success: true, srt: srtText, text: data.text };
+  } catch (error) {
+    console.error("[transcribe] fireworks error:", error);
+    return {
+      success: false,
+      error:
+        error instanceof Error
+          ? error.message
+          : "Fireworks transcription failed",
+    };
+  }
+}
+
+// Main transcription function
+export async function transcribe(
+  options: TranscribeOptions,
+): Promise<TranscribeResult> {
+  const {
+    audioUrl,
+    provider = "groq",
+    model,
+    language,
+    outputFormat = "text",
+    outputPath,
+  } = options;
+
+  if (!audioUrl) {
+    throw new Error("audioUrl is required");
+  }
+
+  console.log(`[transcribe] transcribing ${audioUrl} with ${provider}...`);
+
+  let result: TranscribeResult;
+
+  if (provider === "groq") {
+    result = await transcribeWithGroq(audioUrl, {
+      model,
+      language,
+      outputFormat,
+    });
+  } else if (provider === "fireworks") {
+    result = await transcribeWithFireworks(audioUrl);
+  } else {
+    throw new Error(`Unknown provider: ${provider}`);
+  }
+
+  // Save to file if requested
+  if (result.success && outputPath) {
+    const content = outputFormat === "srt" ? result.srt : result.text;
+    if (content) {
+      writeFileSync(outputPath, content);
+      console.log(`[transcribe] saved to ${outputPath}`);
+    }
+  }
+
+  return result;
+}
+
+export default definition;