varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/src/providers/replicate.ts

@@ -0,0 +1,147 @@
+/**
+ * Replicate provider for video and image generation
+ * Supports Minimax, Kling, Luma, Flux, and other models
+ */
+
+import Replicate from "replicate";
+import type { JobStatusUpdate, ProviderConfig } from "../core/schema/types";
+import { BaseProvider } from "./base";
+
+export class ReplicateProvider extends BaseProvider {
+  readonly name = "replicate";
+  private client: Replicate;
+
+  constructor(config?: ProviderConfig) {
+    super(config);
+    this.client = new Replicate({
+      auth: config?.apiKey || process.env.REPLICATE_API_TOKEN || "",
+    });
+  }
+
+  async submit(
+    model: string,
+    inputs: Record<string, unknown>,
+    _config?: ProviderConfig,
+  ): Promise<string> {
+    // Transform inputs for provider-specific field names
+    const transformedInputs = this.transformInputs(model, inputs);
+
+    const prediction = await this.client.predictions.create({
+      model: model as `${string}/${string}`,
+      input: transformedInputs,
+    });
+
+    console.log(`[replicate] job submitted: ${prediction.id}`);
+    return prediction.id;
+  }
+
+  /**
+   * Transform inputs for provider-specific field names
+   */
+  private transformInputs(
+    model: string,
+    inputs: Record<string, unknown>,
+  ): Record<string, unknown> {
+    // Nano Banana Pro: Replicate uses 'image_input' instead of 'image_urls'
+    if (model === "google/nano-banana-pro" && inputs.image_urls) {
+      const { image_urls, ...rest } = inputs;
+      return {
+        ...rest,
+        image_input: image_urls,
+      };
+    }
+    return inputs;
+  }
+
+  async getStatus(jobId: string): Promise<JobStatusUpdate> {
+    const prediction = await this.client.predictions.get(jobId);
+
+    const statusMap: Record<string, JobStatusUpdate["status"]> = {
+      starting: "queued",
+      processing: "processing",
+      succeeded: "completed",
+      failed: "failed",
+      canceled: "cancelled",
+    };
+
+    return {
+      status: statusMap[prediction.status] ?? "processing",
+      output: prediction.output,
+      error:
+        typeof prediction.error === "string" ? prediction.error : undefined,
+    };
+  }
+
+  async getResult(jobId: string): Promise<unknown> {
+    const prediction = await this.client.predictions.get(jobId);
+    return prediction.output;
+  }
+
+  override async cancel(jobId: string): Promise<void> {
+    await this.client.predictions.cancel(jobId);
+  }
+
+  // ============================================================================
+  // High-level convenience methods
+  // ============================================================================
+
+  async runModel(model: string, input: Record<string, unknown>) {
+    console.log(`[replicate] running ${model}...`);
+
+    const output = await this.client.run(model as `${string}/${string}`, {
+      input,
+    });
+
+    console.log(`[replicate] completed`);
+    return output;
+  }
+
+  async runVideo(options: { model: string; input: Record<string, unknown> }) {
+    return this.runModel(options.model, options.input);
+  }
+
+  async runImage(options: { model: string; input: Record<string, unknown> }) {
+    return this.runModel(options.model, options.input);
+  }
+
+  async listPredictions() {
+    return this.client.predictions.list();
+  }
+
+  async getPrediction(id: string) {
+    return this.client.predictions.get(id);
+  }
+}
+
+// Popular models registry
+export const MODELS = {
+  VIDEO: {
+    MINIMAX: "minimax/video-01",
+    KLING: "fofr/kling-v1.5",
+    LUMA: "fofr/ltx-video",
+    RUNWAY_GEN3: "replicate/runway-gen3-turbo",
+    WAN_2_5: "wan-video/wan-2.5-i2v",
+  },
+  IMAGE: {
+    FLUX_PRO: "black-forest-labs/flux-1.1-pro",
+    FLUX_DEV: "black-forest-labs/flux-dev",
+    FLUX_SCHNELL: "black-forest-labs/flux-schnell",
+    STABLE_DIFFUSION: "stability-ai/sdxl",
+    NANO_BANANA_PRO: "google/nano-banana-pro",
+  },
+};
+
+// Export singleton instance
+export const replicateProvider = new ReplicateProvider();
+
+// Re-export convenience functions for backward compatibility
+export const runModel = (model: string, input: Record<string, unknown>) =>
+  replicateProvider.runModel(model, input);
+export const runVideo = (options: {
+  model: string;
+  input: Record<string, unknown>;
+}) => replicateProvider.runVideo(options);
+export const runImage = (options: {
+  model: string;
+  input: Record<string, unknown>;
+}) => replicateProvider.runImage(options);

package/src/providers/storage.ts

@@ -0,0 +1,206 @@
+/**
+ * Storage provider for Cloudflare R2 / S3 compatible storage
+ */
+
+import {
+  GetObjectCommand,
+  PutObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import type { JobStatusUpdate, ProviderConfig } from "../core/schema/types";
+import { BaseProvider } from "./base";
+
+export interface StorageConfig extends ProviderConfig {
+  endpoint?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  bucket?: string;
+  publicUrl?: string;
+}
+
+export class StorageProvider extends BaseProvider {
+  readonly name = "storage";
+  private client: S3Client;
+  private bucket: string;
+  private publicUrl: string;
+
+  constructor(config?: StorageConfig) {
+    super(config);
+
+    this.client = new S3Client({
+      region: "auto",
+      endpoint: config?.endpoint || process.env.CLOUDFLARE_R2_API_URL,
+      credentials: {
+        accessKeyId:
+          config?.accessKeyId || process.env.CLOUDFLARE_ACCESS_KEY_ID || "",
+        secretAccessKey:
+          config?.secretAccessKey || process.env.CLOUDFLARE_ACCESS_SECRET || "",
+      },
+    });
+
+    this.bucket = config?.bucket || process.env.CLOUDFLARE_R2_BUCKET || "m";
+    this.publicUrl = config?.publicUrl || "https://s3.varg.ai";
+  }
+
+  async submit(
+    _model: string,
+    _inputs: Record<string, unknown>,
+    _config?: ProviderConfig,
+  ): Promise<string> {
+    const jobId = `storage_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+    return jobId;
+  }
+
+  async getStatus(_jobId: string): Promise<JobStatusUpdate> {
+    return { status: "completed" };
+  }
+
+  async getResult(_jobId: string): Promise<unknown> {
+    return null;
+  }
+
+  // ============================================================================
+  // Storage Operations
+  // ============================================================================
+
+  /**
+   * Upload a file from local path to storage
+   */
+  async uploadLocalFile(filePath: string, objectKey: string): Promise<string> {
+    console.log(`[storage] uploading ${filePath} to ${objectKey}`);
+
+    const file = Bun.file(filePath);
+    const buffer = await file.arrayBuffer();
+
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: objectKey,
+        Body: new Uint8Array(buffer),
+      }),
+    );
+
+    return this.getPublicUrl(objectKey);
+  }
+
+  /**
+   * Upload content from a URL to storage
+   */
+  async uploadFromUrl(url: string, objectKey: string): Promise<string> {
+    console.log(`[storage] uploading from URL to ${objectKey}`);
+
+    const response = await fetch(url);
+    const buffer = await response.arrayBuffer();
+
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: objectKey,
+        Body: new Uint8Array(buffer),
+      }),
+    );
+
+    return this.getPublicUrl(objectKey);
+  }
+
+  /**
+   * Upload raw buffer to storage
+   */
+  async uploadBuffer(
+    buffer: ArrayBuffer | Uint8Array,
+    objectKey: string,
+    contentType?: string,
+  ): Promise<string> {
+    console.log(`[storage] uploading buffer to ${objectKey}`);
+
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: objectKey,
+        Body: buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer),
+        ContentType: contentType,
+      }),
+    );
+
+    return this.getPublicUrl(objectKey);
+  }
+
+  /**
+   * Generate a presigned URL for temporary access
+   */
+  async generatePresignedUrl(
+    objectKey: string,
+    expiresIn = 3600,
+  ): Promise<string> {
+    const command = new GetObjectCommand({
+      Bucket: this.bucket,
+      Key: objectKey,
+    });
+
+    return getSignedUrl(this.client, command, { expiresIn });
+  }
+
+  /**
+   * Get the public URL for an object
+   */
+  getPublicUrl(objectKey: string): string {
+    const endpoint = process.env.CLOUDFLARE_R2_API_URL || "";
+
+    if (endpoint.includes("localhost")) {
+      return `${endpoint}/${objectKey}`;
+    }
+
+    return `${this.publicUrl}/${objectKey}`;
+  }
+
+  /**
+   * Download a file from storage
+   */
+  async downloadToFile(objectKey: string, outputPath: string): Promise<string> {
+    console.log(`[storage] downloading ${objectKey} to ${outputPath}`);
+
+    const command = new GetObjectCommand({
+      Bucket: this.bucket,
+      Key: objectKey,
+    });
+
+    const response = await this.client.send(command);
+    const body = response.Body;
+
+    if (!body) {
+      throw new Error("Empty response body");
+    }
+
+    // Convert stream to buffer
+    const chunks: Uint8Array[] = [];
+    // @ts-expect-error - body is a readable stream
+    for await (const chunk of body) {
+      chunks.push(chunk);
+    }
+    const buffer = Buffer.concat(chunks);
+
+    await Bun.write(outputPath, buffer);
+    console.log(`[storage] saved to ${outputPath}`);
+
+    return outputPath;
+  }
+}
+
+// Export singleton instance
+export const storageProvider = new StorageProvider();
+
+// Re-export convenience functions for backward compatibility
+export const uploadFile = (filePath: string, objectKey: string) =>
+  storageProvider.uploadLocalFile(filePath, objectKey);
+export const uploadFromUrl = (url: string, objectKey: string) =>
+  storageProvider.uploadFromUrl(url, objectKey);
+export const uploadBuffer = (
+  buffer: ArrayBuffer | Uint8Array,
+  objectKey: string,
+  contentType?: string,
+) => storageProvider.uploadBuffer(buffer, objectKey, contentType);
+export const generatePresignedUrl = (objectKey: string, expiresIn?: number) =>
+  storageProvider.generatePresignedUrl(objectKey, expiresIn);
+export const getPublicUrl = (objectKey: string) =>
+  storageProvider.getPublicUrl(objectKey);

package/src/react/cli.ts

@@ -0,0 +1,52 @@
+#!/usr/bin/env bun
+
+import { parseArgs } from "node:util";
+import { render } from "./render";
+import type { VargElement } from "./types";
+
+const { values, positionals } = parseArgs({
+  args: Bun.argv.slice(2),
+  options: {
+    output: { type: "string", short: "o" },
+    cache: { type: "string", short: "c", default: ".cache/ai" },
+    quiet: { type: "boolean", short: "q", default: false },
+  },
+  allowPositionals: true,
+});
+
+const [file] = positionals;
+
+if (!file) {
+  console.error("usage: bun react/cli.ts <component.tsx> [-o output.mp4]");
+  process.exit(1);
+}
+
+const resolvedPath = Bun.resolveSync(file, process.cwd());
+const mod = await import(resolvedPath);
+const component: VargElement = mod.default;
+
+if (!component || component.type !== "render") {
+  console.error("error: default export must be a <Render> element");
+  process.exit(1);
+}
+
+const outputPath =
+  values.output ??
+  `output/${file
+    .replace(/\.tsx?$/, "")
+    .split("/")
+    .pop()}.mp4`;
+
+if (!values.quiet) {
+  console.log(`rendering ${file} → ${outputPath}`);
+}
+
+const buffer = await render(component, {
+  output: outputPath,
+  cache: values.cache,
+  quiet: values.quiet,
+});
+
+if (!values.quiet) {
+  console.log(`done! ${buffer.byteLength} bytes → ${outputPath}`);
+}

package/src/react/elements.ts

@@ -0,0 +1,146 @@
+import type {
+  AnimateProps,
+  CaptionsProps,
+  ClipProps,
+  ImageProps,
+  MusicProps,
+  OverlayProps,
+  PackshotProps,
+  RenderProps,
+  SliderProps,
+  SpeechProps,
+  SplitProps,
+  SubtitleProps,
+  SwipeProps,
+  TalkingHeadProps,
+  TitleProps,
+  VargElement,
+  VargNode,
+  VideoProps,
+} from "./types";
+
+function normalizeChildren(children: unknown): VargNode[] {
+  if (children === null || children === undefined) return [];
+  if (Array.isArray(children))
+    return children.flat().flatMap(normalizeChildren);
+  return [children as VargNode];
+}
+
+function createElement<T extends VargElement["type"]>(
+  type: T,
+  props: Record<string, unknown>,
+  children: unknown,
+): VargElement<T> {
+  const { children: _, ...restProps } = props;
+  return {
+    type,
+    props: restProps,
+    children: normalizeChildren(children ?? props.children),
+  };
+}
+
+export function Render(props: RenderProps): VargElement<"render"> {
+  return createElement(
+    "render",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Clip(props: ClipProps): VargElement<"clip"> {
+  return createElement(
+    "clip",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Overlay(props: OverlayProps): VargElement<"overlay"> {
+  return createElement(
+    "overlay",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Image(props: ImageProps): VargElement<"image"> {
+  return createElement("image", props as Record<string, unknown>, undefined);
+}
+
+export function Video(props: VideoProps): VargElement<"video"> {
+  return createElement("video", props as Record<string, unknown>, undefined);
+}
+
+export function Animate(props: AnimateProps): VargElement<"animate"> {
+  return createElement("animate", props as Record<string, unknown>, undefined);
+}
+
+export function Speech(props: SpeechProps): VargElement<"speech"> {
+  return createElement(
+    "speech",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function TalkingHead(
+  props: TalkingHeadProps,
+): VargElement<"talking-head"> {
+  return createElement(
+    "talking-head",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Title(props: TitleProps): VargElement<"title"> {
+  return createElement(
+    "title",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Subtitle(props: SubtitleProps): VargElement<"subtitle"> {
+  return createElement(
+    "subtitle",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Music(props: MusicProps): VargElement<"music"> {
+  return createElement("music", props as Record<string, unknown>, undefined);
+}
+
+export function Captions(props: CaptionsProps): VargElement<"captions"> {
+  return createElement("captions", props as Record<string, unknown>, undefined);
+}
+
+export function Split(props: SplitProps): VargElement<"split"> {
+  return createElement(
+    "split",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Slider(props: SliderProps): VargElement<"slider"> {
+  return createElement(
+    "slider",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Swipe(props: SwipeProps): VargElement<"swipe"> {
+  return createElement(
+    "swipe",
+    props as Record<string, unknown>,
+    props.children,
+  );
+}
+
+export function Packshot(props: PackshotProps): VargElement<"packshot"> {
+  return createElement("packshot", props as Record<string, unknown>, undefined);
+}

package/src/react/examples/branching.tsx

@@ -0,0 +1,66 @@
+import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
+import { fal } from "../../ai-sdk/providers/fal-provider";
+import { Animate, Clip, Image, Render, Speech, Title } from "..";
+
+// Non-linear tree: multiple clips with independent branches
+// Clip 1: TalkingHead (Image -> Animate + Speech)
+// Clip 2: Split comparison (2 independent Images)
+// Clip 3: Product shot with music
+
+const character = Image({
+  prompt:
+    "friendly tech reviewer, young man with glasses, studio lighting, professional headshot",
+  model: fal.imageModel("flux-schnell"),
+});
+
+const _productBefore = Image({
+  prompt:
+    "old smartphone, cracked screen, slow, outdated design, on white background",
+  model: fal.imageModel("flux-schnell"),
+});
+
+const _productAfter = Image({
+  prompt:
+    "sleek new smartphone, edge-to-edge display, premium design, on white background",
+  model: fal.imageModel("flux-schnell"),
+});
+
+const packshot = Image({
+  prompt:
+    "smartphone floating with gradient background, product photography, premium feel",
+  model: fal.imageModel("flux-schnell"),
+});
+
+export default (
+  <Render width={1080} height={1920}>
+    {/* Clip 1: Talking head intro */}
+    <Clip duration={5}>
+      <Animate
+        image={character}
+        model={fal.videoModel("wan-2.5")}
+        motion="talking naturally, slight head movements, friendly expression"
+      />
+      <Speech voice="adam" model={elevenlabs.speechModel("turbo")}>
+        Hey everyone! Today we're looking at the biggest smartphone upgrade of
+        the year.
+      </Speech>
+    </Clip>
+
+    {/* Clip 2: Before/after comparison - branches into 2 images */}
+    <Clip duration={4} transition={{ name: "fade", duration: 0.5 }}>
+      <Image
+        prompt="split screen comparison layout"
+        model={fal.imageModel("flux-schnell")}
+      />
+      <Title position="top">Before vs After</Title>
+    </Clip>
+
+    {/* Clip 3: Product packshot */}
+    <Clip duration={3} transition={{ name: "fade", duration: 0.5 }}>
+      {packshot}
+      <Title position="bottom" color="#ffffff">
+        Available Now
+      </Title>
+    </Clip>
+  </Render>
+);

package/src/react/examples/captions-demo.tsx

@@ -0,0 +1,37 @@
+import { elevenlabs } from "../../ai-sdk/providers/elevenlabs-provider";
+import { Captions, Clip, Image, Render, render, Speech } from "..";
+
+async function main() {
+  const speech = Speech({
+    model: elevenlabs.speechModel("eleven_multilingual_v2"),
+    voice: "adam",
+    children:
+      "Hello world! This is a test of the captions system with word level timestamps.",
+  });
+
+  const video = (
+    <Render width={1080} height={1920}>
+      <Clip duration={5}>
+        <Image src="media/cyberpunk-street.png" resize="contain" />
+      </Clip>
+      <Clip duration={5}>
+        <Image src="media/cyberpunk-street.png" resize="cover" />
+      </Clip>
+      <Clip duration={5}>
+        <Image src="media/cyberpunk-street.png" />
+      </Clip>
+      <Captions src={speech} style="tiktok" />
+    </Render>
+  );
+
+  console.log("rendering captions demo with speech transcription...\n");
+
+  await render(video, {
+    output: "output/captions-demo.mp4",
+    cache: ".cache/ai",
+  });
+
+  console.log("\ndone! check output/captions-demo.mp4");
+}
+
+main().catch(console.error);