varg.ai-sdk 0.1.0 → 0.4.0-alpha.1

package/src/definitions/actions/upload.ts

@@ -0,0 +1,111 @@
+/**
+ * Upload action
+ * Upload files to S3/R2 storage
+ */
+
+import { existsSync } from "node:fs";
+import { basename, extname } from "node:path";
+import { z } from "zod";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { storageProvider } from "../../providers/storage";
+
+// Input schema
+const uploadInputSchema = z.object({
+  file: z.string().describe("Local file path or URL to upload"),
+  key: z
+    .string()
+    .optional()
+    .describe("Object key/path in storage (auto-generated if not provided)"),
+});
+
+// Output schema
+const uploadOutputSchema = z.object({
+  url: z.string().describe("Public URL of the uploaded file"),
+  key: z.string().describe("Object key in storage"),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof uploadInputSchema, typeof uploadOutputSchema> = {
+  input: uploadInputSchema,
+  output: uploadOutputSchema,
+};
+
+export interface UploadOptions {
+  key?: string;
+}
+
+export interface UploadResult {
+  url: string;
+  key: string;
+}
+
+/**
+ * Generate a unique object key based on file info
+ */
+function generateObjectKey(source: string): string {
+  const timestamp = Date.now();
+  const random = Math.random().toString(36).slice(2, 8);
+
+  // Extract extension from source
+  let ext = extname(source);
+  if (!ext) {
+    // Try to guess from URL or default to .bin
+    if (source.includes(".")) {
+      const parts = source.split(".");
+      ext = `.${parts[parts.length - 1]?.split("?")[0] || "bin"}`;
+    } else {
+      ext = ".bin";
+    }
+  }
+
+  const name = basename(source, ext).slice(0, 20) || "file";
+  return `uploads/${timestamp}-${random}-${name}${ext}`;
+}
+
+/**
+ * Check if a string is a URL
+ */
+function isUrl(str: string): boolean {
+  return str.startsWith("http://") || str.startsWith("https://");
+}
+
+/**
+ * Upload a file to storage
+ */
+export async function upload(
+  file: string,
+  options: UploadOptions = {},
+): Promise<UploadResult> {
+  const key = options.key || generateObjectKey(file);
+
+  if (isUrl(file)) {
+    console.log(`[upload] uploading from URL: ${file}`);
+    const url = await storageProvider.uploadFromUrl(file, key);
+    console.log(`[upload] uploaded to ${url}`);
+    return { url, key };
+  }
+
+  // Local file
+  if (!existsSync(file)) {
+    throw new Error(`File not found: ${file}`);
+  }
+
+  console.log(`[upload] uploading local file: ${file}`);
+  const url = await storageProvider.uploadLocalFile(file, key);
+  console.log(`[upload] uploaded to ${url}`);
+  return { url, key };
+}
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "upload",
+  description: "Upload file to S3/R2 storage",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    const { file, key } = inputs;
+    return upload(file, { key });
+  },
+};
+
+export default definition;

package/src/definitions/actions/video.ts

@@ -0,0 +1,163 @@
+/**
+ * Video generation action
+ * Routes to appropriate video generation models based on input
+ */
+
+import { z } from "zod";
+import {
+  aspectRatioSchema,
+  filePathSchema,
+  videoDurationSchema,
+} from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { falProvider } from "../../providers/fal";
+import { storageProvider } from "../../providers/storage";
+
+// Input schema with Zod
+const videoInputSchema = z.object({
+  prompt: z.string().describe("What to generate"),
+  image: filePathSchema
+    .optional()
+    .describe("Input image (enables image-to-video)"),
+  duration: videoDurationSchema
+    .default(5)
+    .describe("Video duration in seconds"),
+  aspectRatio: aspectRatioSchema
+    .default("16:9")
+    .describe("Aspect ratio for text-to-video"),
+});
+
+// Output schema with Zod
+const videoOutputSchema = z.object({
+  videoUrl: z.string(),
+  duration: z.number().optional(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof videoInputSchema, typeof videoOutputSchema> = {
+  input: videoInputSchema,
+  output: videoOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "video",
+  description: "Generate video from text or image",
+  schema,
+  routes: [
+    {
+      target: "kling",
+      priority: 10,
+    },
+  ],
+  execute: async (inputs) => {
+    // inputs is now fully typed as VideoInput - no more `as` cast!
+    const { prompt, image, duration, aspectRatio } = inputs;
+
+    let result: { data?: { video?: { url?: string }; duration?: number } };
+
+    if (image) {
+      console.log("[action/video] generating video from image");
+      result = await falProvider.imageToVideo({
+        prompt,
+        imageUrl: image,
+        duration,
+      });
+    } else {
+      console.log("[action/video] generating video from text");
+      result = await falProvider.textToVideo({
+        prompt,
+        duration,
+        aspectRatio,
+      });
+    }
+
+    const videoUrl = result.data?.video?.url;
+    if (!videoUrl) {
+      throw new Error("No video URL in result");
+    }
+
+    return {
+      videoUrl,
+      duration: result.data?.duration,
+    };
+  },
+};
+
+// Re-export types and functions for backward compatibility
+export interface VideoGenerationResult {
+  videoUrl: string;
+  duration?: number;
+  uploaded?: string;
+}
+
+export async function generateVideoFromImage(
+  prompt: string,
+  imageUrl: string,
+  options: { duration?: 5 | 10; upload?: boolean } = {},
+): Promise<VideoGenerationResult> {
+  console.log("[video] generating video from image");
+
+  const result = await falProvider.imageToVideo({
+    prompt,
+    imageUrl,
+    duration: options.duration,
+  });
+
+  const videoUrl = result.data?.video?.url;
+  if (!videoUrl) {
+    throw new Error("No video URL in result");
+  }
+
+  let uploaded: string | undefined;
+  if (options.upload) {
+    const timestamp = Date.now();
+    const objectKey = `videos/generated/${timestamp}.mp4`;
+    uploaded = await storageProvider.uploadFromUrl(videoUrl, objectKey);
+    console.log(`[video] uploaded to ${uploaded}`);
+  }
+
+  return {
+    videoUrl,
+    duration: result.data?.duration,
+    uploaded,
+  };
+}
+
+export async function generateVideoFromText(
+  prompt: string,
+  options: {
+    duration?: 5 | 10;
+    upload?: boolean;
+    aspectRatio?: "16:9" | "9:16" | "1:1";
+  } = {},
+): Promise<VideoGenerationResult> {
+  console.log("[video] generating video from text");
+
+  const result = await falProvider.textToVideo({
+    prompt,
+    duration: options.duration,
+    aspectRatio: options.aspectRatio,
+  });
+
+  const videoUrl = result.data?.video?.url;
+  if (!videoUrl) {
+    throw new Error("No video URL in result");
+  }
+
+  let uploaded: string | undefined;
+  if (options.upload) {
+    const timestamp = Date.now();
+    const objectKey = `videos/generated/${timestamp}.mp4`;
+    uploaded = await storageProvider.uploadFromUrl(videoUrl, objectKey);
+    console.log(`[video] uploaded to ${uploaded}`);
+  }
+
+  return {
+    videoUrl,
+    duration: result.data?.duration,
+    uploaded,
+  };
+}
+
+export default definition;

package/src/definitions/actions/voice.ts

@@ -0,0 +1,119 @@
+/**
+ * Voice generation action
+ * Text-to-speech via ElevenLabs
+ */
+
+import { z } from "zod";
+import { filePathSchema, voiceNameSchema } from "../../core/schema/shared";
+import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
+import { elevenlabsProvider, VOICES } from "../../providers/elevenlabs";
+import { storageProvider } from "../../providers/storage";
+
+// Input schema with Zod
+const voiceInputSchema = z.object({
+  text: z.string().describe("Text to convert to speech"),
+  voice: voiceNameSchema.default("rachel").describe("Voice to use"),
+  output: filePathSchema.optional().describe("Output file path"),
+});
+
+// Output schema with Zod
+const voiceOutputSchema = z.object({
+  audio: z.instanceof(Buffer),
+  provider: z.string(),
+  voiceId: z.string(),
+  uploadUrl: z.string().optional(),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof voiceInputSchema, typeof voiceOutputSchema> = {
+  input: voiceInputSchema,
+  output: voiceOutputSchema,
+};
+
+export const definition: ActionDefinition<typeof schema> = {
+  type: "action",
+  name: "voice",
+  description: "Text to speech generation",
+  schema,
+  routes: [],
+  execute: async (inputs) => {
+    const { text, voice, output } = inputs;
+    return generateVoice({ text, voice, outputPath: output });
+  },
+};
+
+// Types
+export interface GenerateVoiceOptions {
+  text: string;
+  voice?: string;
+  provider?: "elevenlabs";
+  upload?: boolean;
+  outputPath?: string;
+}
+
+export interface VoiceResult {
+  audio: Buffer;
+  provider: string;
+  voiceId: string;
+  uploadUrl?: string;
+}
+
+// Voice name to ID mapping
+const VOICE_MAP: Record<string, string> = {
+  rachel: VOICES.RACHEL,
+  domi: VOICES.DOMI,
+  bella: VOICES.BELLA,
+  antoni: VOICES.ANTONI,
+  elli: VOICES.ELLI,
+  josh: VOICES.JOSH,
+  arnold: VOICES.ARNOLD,
+  adam: VOICES.ADAM,
+  sam: VOICES.SAM,
+};
+
+export async function generateVoice(
+  options: GenerateVoiceOptions,
+): Promise<VoiceResult> {
+  const {
+    text,
+    voice = "rachel",
+    provider = "elevenlabs",
+    upload = false,
+    outputPath,
+  } = options;
+
+  if (!text) {
+    throw new Error("text is required");
+  }
+
+  console.log(`[voice] generating with ${provider} (${voice})...`);
+
+  const voiceId = VOICE_MAP[voice.toLowerCase()] || voice;
+
+  const audio = await elevenlabsProvider.textToSpeech({
+    text,
+    voiceId,
+    outputPath,
+  });
+
+  const result: VoiceResult = {
+    audio,
+    provider,
+    voiceId,
+  };
+
+  // Upload to storage if requested
+  if (upload && outputPath) {
+    const objectKey = `voice/${Date.now()}-${voice}.mp3`;
+    const uploadUrl = await storageProvider.uploadLocalFile(
+      outputPath,
+      objectKey,
+    );
+    result.uploadUrl = uploadUrl;
+    console.log(`[voice] uploaded to ${uploadUrl}`);
+  }
+
+  return result;
+}
+
+export default definition;

package/src/definitions/index.ts

@@ -0,0 +1,23 @@
+/**
+ * Definitions index
+ * Re-exports all actions, models, and skills
+ */
+
+// Actions
+export * from "./actions";
+export { allActions } from "./actions";
+
+// Models
+export * from "./models";
+export { allModels } from "./models";
+
+// Skills
+export * from "./skills";
+export { allSkills } from "./skills";
+
+// All definitions combined
+import { allActions } from "./actions";
+import { allModels } from "./models";
+import { allSkills } from "./skills";
+
+export const allDefinitions = [...allModels, ...allActions, ...allSkills];

package/src/definitions/models/elevenlabs.ts

@@ -0,0 +1,50 @@
+/**
+ * ElevenLabs voice models
+ * Text-to-speech generation
+ */
+
+import { z } from "zod";
+import { elevenLabsModelSchema, percentSchema } from "../../core/schema/shared";
+import type { ModelDefinition, ZodSchema } from "../../core/schema/types";
+
+// Input schema with Zod
+const elevenlabsInputSchema = z.object({
+  text: z.string().describe("Text to convert to speech"),
+  voice_id: z.string().optional().describe("Voice ID to use"),
+  model_id: elevenLabsModelSchema
+    .default("eleven_multilingual_v2")
+    .describe("TTS model to use"),
+  stability: percentSchema.default(0.5).describe("Voice stability (0-1)"),
+  similarity_boost: percentSchema
+    .default(0.75)
+    .describe("Voice similarity boost (0-1)"),
+});
+
+// Output schema with Zod
+const elevenlabsOutputSchema = z.object({
+  audio: z.instanceof(Buffer),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<
+  typeof elevenlabsInputSchema,
+  typeof elevenlabsOutputSchema
+> = {
+  input: elevenlabsInputSchema,
+  output: elevenlabsOutputSchema,
+};
+
+export const definition: ModelDefinition<typeof schema> = {
+  type: "model",
+  name: "elevenlabs-tts",
+  description:
+    "ElevenLabs text-to-speech model for high-quality voice generation",
+  providers: ["elevenlabs"],
+  defaultProvider: "elevenlabs",
+  providerModels: {
+    elevenlabs: "eleven_multilingual_v2",
+  },
+  schema,
+};
+
+export default definition;

package/src/definitions/models/flux.ts

@@ -0,0 +1,56 @@
+/**
+ * Flux image generation model
+ * High-quality image generation from text
+ */
+
+import { z } from "zod";
+import { imageSizeSchema } from "../../core/schema/shared";
+import type { ModelDefinition, ZodSchema } from "../../core/schema/types";
+
+// Input schema with Zod
+const fluxInputSchema = z.object({
+  prompt: z.string().describe("Text description of the image"),
+  image_size: imageSizeSchema
+    .default("landscape_4_3")
+    .describe("Output image size/aspect"),
+  num_inference_steps: z
+    .number()
+    .int()
+    .default(28)
+    .describe("Number of inference steps"),
+  guidance_scale: z
+    .number()
+    .default(3.5)
+    .describe("Guidance scale for generation"),
+});
+
+// Output schema with Zod
+const fluxOutputSchema = z.object({
+  images: z.array(
+    z.object({
+      url: z.string(),
+    }),
+  ),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof fluxInputSchema, typeof fluxOutputSchema> = {
+  input: fluxInputSchema,
+  output: fluxOutputSchema,
+};
+
+export const definition: ModelDefinition<typeof schema> = {
+  type: "model",
+  name: "flux",
+  description:
+    "Flux Pro image generation model for high-quality images from text",
+  providers: ["fal", "replicate"],
+  defaultProvider: "fal",
+  providerModels: {
+    fal: "fal-ai/flux-pro/v1.1",
+    replicate: "black-forest-labs/flux-1.1-pro",
+  },
+  schema,
+};
+
+export default definition;

package/src/definitions/models/index.ts

@@ -0,0 +1,36 @@
+/**
+ * Model definitions index
+ */
+
+export { definition as elevenlabsTts } from "./elevenlabs";
+export { definition as flux } from "./flux";
+export { definition as kling } from "./kling";
+export { definition as llama } from "./llama";
+export { definition as nanoBananaPro } from "./nano-banana-pro";
+export { definition as sonauto } from "./sonauto";
+export { definition as soul } from "./soul";
+export { definition as wan } from "./wan";
+export { definition as whisper } from "./whisper";
+
+// All model definitions for auto-loading
+import { definition as elevenlabsDefinition } from "./elevenlabs";
+import { definition as fluxDefinition } from "./flux";
+import { definition as klingDefinition } from "./kling";
+import { definition as llamaDefinition } from "./llama";
+import { definition as nanoBananaProDefinition } from "./nano-banana-pro";
+import { definition as sonautoDefinition } from "./sonauto";
+import { definition as soulDefinition } from "./soul";
+import { definition as wanDefinition } from "./wan";
+import { definition as whisperDefinition } from "./whisper";
+
+export const allModels = [
+  klingDefinition,
+  fluxDefinition,
+  nanoBananaProDefinition,
+  wanDefinition,
+  whisperDefinition,
+  elevenlabsDefinition,
+  soulDefinition,
+  sonautoDefinition,
+  llamaDefinition,
+];

package/src/definitions/models/kling.ts

@@ -0,0 +1,56 @@
+/**
+ * Kling video generation model
+ * High-quality video generation from text/image
+ */
+
+import { z } from "zod";
+import {
+  aspectRatioSchema,
+  videoDurationSchema,
+} from "../../core/schema/shared";
+import type { ModelDefinition, ZodSchema } from "../../core/schema/types";
+
+// Input schema with Zod
+const klingInputSchema = z.object({
+  prompt: z.string().describe("Text description of the video"),
+  image_url: z
+    .string()
+    .url()
+    .optional()
+    .describe("Input image for image-to-video"),
+  duration: videoDurationSchema
+    .default(5)
+    .describe("Video duration in seconds"),
+  aspect_ratio: aspectRatioSchema
+    .default("16:9")
+    .describe("Output aspect ratio"),
+});
+
+// Output schema with Zod
+const klingOutputSchema = z.object({
+  video: z.object({
+    url: z.string(),
+  }),
+});
+
+// Schema object for the definition
+const schema: ZodSchema<typeof klingInputSchema, typeof klingOutputSchema> = {
+  input: klingInputSchema,
+  output: klingOutputSchema,
+};
+
+export const definition: ModelDefinition<typeof schema> = {
+  type: "model",
+  name: "kling",
+  description:
+    "Kling video generation model for high-quality video from text or image",
+  providers: ["fal", "replicate"],
+  defaultProvider: "fal",
+  providerModels: {
+    fal: "fal-ai/kling-video/v2.5-turbo/pro",
+    replicate: "fofr/kling-v1.5",
+  },
+  schema,
+};
+
+export default definition;

package/src/definitions/models/llama.ts

@@ -0,0 +1,54 @@
+/**
+ * Llama LLM model
+ * Fast inference via Groq
+ */
+
+import { z } from "zod";
+import type { ModelDefinition, ZodSchema } from "../../core/schema/types";
+
+// Llama model variants schema
+const llamaModelSchema = z.enum([
+  "llama-3.3-70b-versatile",
+  "llama-3.1-8b-instant",
+  "llama-3.1-70b-versatile",
+]);
+
+// Chat message schema
+const chatMessageSchema = z.object({
+  role: z.enum(["system", "user", "assistant"]),
+  content: z.string(),
+});
+
+// Input schema with Zod
+const llamaInputSchema = z.object({
+  messages: z.array(chatMessageSchema).describe("Chat messages array"),
+  model: llamaModelSchema
+    .default("llama-3.3-70b-versatile")
+    .describe("Llama model variant"),
+  temperature: z.number().default(1).describe("Sampling temperature"),
+  max_tokens: z.number().int().default(1024).describe("Maximum output tokens"),
+  stream: z.boolean().default(false).describe("Stream response"),
+});
+
+// Output schema with Zod
+const llamaOutputSchema = z.string().describe("Generated text response");
+
+// Schema object for the definition
+const schema: ZodSchema<typeof llamaInputSchema, typeof llamaOutputSchema> = {
+  input: llamaInputSchema,
+  output: llamaOutputSchema,
+};
+
+export const definition: ModelDefinition<typeof schema> = {
+  type: "model",
+  name: "llama",
+  description: "Meta Llama model for fast text generation via Groq",
+  providers: ["groq"],
+  defaultProvider: "groq",
+  providerModels: {
+    groq: "llama-3.3-70b-versatile",
+  },
+  schema,
+};
+
+export default definition;