vargai 0.3.0

package/src/ai-sdk/file.ts

@@ -0,0 +1,238 @@
+import type { ImageModelV3File } from "@ai-sdk/provider";
+
+export class File {
+  private _data: Uint8Array | null = null;
+  private _url: string | null = null;
+  private _mediaType: string;
+  private _loader: (() => Promise<Uint8Array>) | null = null;
+
+  private constructor(
+    options:
+      | { data: Uint8Array; mediaType: string }
+      | { url: string; mediaType?: string }
+      | { loader: () => Promise<Uint8Array>; mediaType: string },
+  ) {
+    if ("data" in options) {
+      this._data = options.data;
+      this._mediaType = options.mediaType;
+    } else if ("url" in options) {
+      this._url = options.url;
+      this._mediaType = options.mediaType ?? inferMediaType(options.url);
+    } else {
+      this._loader = options.loader;
+      this._mediaType = options.mediaType;
+    }
+  }
+
+  static fromPath(path: string, mediaType?: string): File {
+    const resolvedMediaType = mediaType ?? inferMediaType(path);
+    return new File({
+      loader: async () => {
+        const file = Bun.file(path);
+        return new Uint8Array(await file.arrayBuffer());
+      },
+      mediaType: resolvedMediaType,
+    });
+  }
+
+  static fromUrl(url: string, mediaType?: string): File {
+    return new File({ url, mediaType });
+  }
+
+  static fromBuffer(data: Uint8Array, mediaType: string): File {
+    return new File({ data, mediaType });
+  }
+
+  static fromGenerated(generated: {
+    uint8Array: Uint8Array;
+    mediaType: string;
+  }): File {
+    return new File({
+      data: generated.uint8Array,
+      mediaType: generated.mediaType,
+    });
+  }
+
+  static fromArrayBuffer(buffer: ArrayBuffer, mediaType: string): File {
+    return new File({ data: new Uint8Array(buffer), mediaType });
+  }
+
+  static async fromBlob(blob: Blob, mediaType?: string): Promise<File> {
+    const data = new Uint8Array(await blob.arrayBuffer());
+    return new File({ data, mediaType: mediaType ?? blob.type });
+  }
+
+  static from(input: {
+    uint8Array: Uint8Array;
+    mimeType?: string;
+    mediaType?: string;
+  }): File;
+  static from(
+    input: string | Uint8Array | ArrayBuffer | Blob,
+    mediaType?: string,
+  ): File | Promise<File>;
+  static from(
+    input:
+      | string
+      | Uint8Array
+      | ArrayBuffer
+      | Blob
+      | { uint8Array: Uint8Array; mimeType?: string; mediaType?: string },
+    mediaType?: string,
+  ): File | Promise<File> {
+    if (typeof input === "object" && input !== null && "uint8Array" in input) {
+      const mime =
+        input.mimeType ?? input.mediaType ?? "application/octet-stream";
+      return File.fromBuffer(input.uint8Array, mime);
+    }
+    if (typeof input === "string" && /^https?:\/\//.test(input)) {
+      return File.fromUrl(input, mediaType);
+    }
+    if (typeof input === "string") {
+      return File.fromPath(input, mediaType);
+    }
+    if (input instanceof Blob) {
+      return File.fromBlob(input, mediaType);
+    }
+    if (input instanceof ArrayBuffer) {
+      return File.fromArrayBuffer(
+        input,
+        mediaType ?? "application/octet-stream",
+      );
+    }
+    return File.fromBuffer(input, mediaType ?? "application/octet-stream");
+  }
+
+  get mediaType(): string {
+    return this._mediaType;
+  }
+
+  isImage(): boolean {
+    return this._mediaType.startsWith("image/");
+  }
+
+  isAudio(): boolean {
+    return this._mediaType.startsWith("audio/");
+  }
+
+  isVideo(): boolean {
+    return this._mediaType.startsWith("video/");
+  }
+
+  async data(): Promise<Uint8Array> {
+    if (this._data) return this._data;
+    if (this._loader) {
+      this._data = await this._loader();
+      return this._data;
+    }
+    if (this._url) {
+      const response = await fetch(this._url);
+      this._data = new Uint8Array(await response.arrayBuffer());
+      return this._data;
+    }
+    throw new Error("File has no data source");
+  }
+
+  async arrayBuffer(): Promise<Uint8Array> {
+    return this.data();
+  }
+
+  async blob(): Promise<Blob> {
+    const data = await this.arrayBuffer();
+    return new Blob([data], { type: this._mediaType });
+  }
+
+  async url(uploader?: (blob: Blob) => Promise<string>): Promise<string> {
+    if (this._url && !this._data && !this._loader) {
+      return this._url;
+    }
+    const blob = await this.blob();
+    if (uploader) return uploader(blob);
+    try {
+      const { fal } = await import("@fal-ai/client");
+      return fal.storage.upload(blob);
+    } catch {
+      throw new Error("No uploader provided and @fal-ai/client not available.");
+    }
+  }
+
+  async base64(): Promise<string> {
+    const data = await this.arrayBuffer();
+    let binary = "";
+    for (let i = 0; i < data.byteLength; i++) {
+      binary += String.fromCharCode(data[i]!);
+    }
+    return btoa(binary);
+  }
+
+  async toInput(): Promise<ImageModelV3File> {
+    if (this._url && !this._data && !this._loader) {
+      return { type: "url", url: this._url };
+    }
+    const data = await this.arrayBuffer();
+    return { type: "file", mediaType: this._mediaType, data };
+  }
+
+  async toTemp(): Promise<string> {
+    const data = await this.data();
+    const ext = this.extensionFromMediaType();
+    const tmpDir = process.env.TMPDIR ?? "/tmp";
+    const filename = `varg-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`;
+    const path = `${tmpDir}/${filename}`;
+    await Bun.write(path, data);
+    return path;
+  }
+
+  static async toTemp(
+    file:
+      | { uint8Array: Uint8Array; mimeType?: string; mediaType?: string }
+      | File,
+  ): Promise<string> {
+    if (file instanceof File) {
+      return file.toTemp();
+    }
+    const f = File.from(file);
+    return f.toTemp();
+  }
+
+  private extensionFromMediaType(): string {
+    const extMap: Record<string, string> = {
+      "image/png": ".png",
+      "image/jpeg": ".jpg",
+      "image/gif": ".gif",
+      "image/webp": ".webp",
+      "audio/mpeg": ".mp3",
+      "audio/wav": ".wav",
+      "audio/mp4": ".m4a",
+      "video/mp4": ".mp4",
+      "video/webm": ".webm",
+      "video/quicktime": ".mov",
+    };
+    return extMap[this._mediaType] ?? "";
+  }
+}
+
+function inferMediaType(path: string): string {
+  const ext = path.split(".").pop()?.toLowerCase();
+  const mimeTypes: Record<string, string> = {
+    png: "image/png",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    gif: "image/gif",
+    webp: "image/webp",
+    svg: "image/svg+xml",
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    ogg: "audio/ogg",
+    m4a: "audio/mp4",
+    aac: "audio/aac",
+    mp4: "video/mp4",
+    webm: "video/webm",
+    mov: "video/quicktime",
+  };
+  return mimeTypes[ext ?? ""] ?? "application/octet-stream";
+}
+
+export function files(...paths: string[]): File[] {
+  return paths.map((p) => File.fromPath(p));
+}

package/src/ai-sdk/generate-element.ts

@@ -0,0 +1,92 @@
+import type { DataContent, ImageModel } from "ai";
+import { generateImage } from "ai";
+
+export type ElementType = "character" | "location" | "item";
+
+export interface Element {
+  type: ElementType;
+  text: string;
+  images: Uint8Array[];
+}
+
+export interface PromptWithElements {
+  text: string;
+  images: Uint8Array[];
+}
+
+export function scene(
+  strings: TemplateStringsArray,
+  ...elements: Element[]
+): PromptWithElements {
+  let text = "";
+  const images: Uint8Array[] = [];
+  let imageIndex = 1;
+
+  for (let i = 0; i < strings.length; i++) {
+    text += strings[i];
+    if (i < elements.length) {
+      const el = elements[i]!;
+      const count = el.images.length;
+
+      if (count === 1) {
+        text += `[image ${imageIndex}: ${el.type} - ${el.text}]`;
+      } else if (count > 1) {
+        text += `[images ${imageIndex}-${imageIndex + count - 1}: ${el.type} - ${el.text}]`;
+      } else {
+        text += `[${el.type}: ${el.text}]`;
+      }
+
+      images.push(...el.images);
+      imageIndex += count;
+    }
+  }
+
+  return { text: text.trim(), images };
+}
+
+export type GenerateElementPrompt =
+  | string
+  | {
+      text: string;
+      images?: Array<DataContent>;
+    };
+
+export interface GenerateElementOptions {
+  model: ImageModel;
+  type: ElementType;
+  prompt: GenerateElementPrompt;
+  n?: number;
+  size?: `${number}x${number}`;
+  aspectRatio?: `${number}:${number}`;
+  seed?: number;
+}
+
+export interface GenerateElementResult {
+  element: Element;
+}
+
+export async function generateElement(
+  options: GenerateElementOptions,
+): Promise<GenerateElementResult> {
+  const { model, type, prompt, n = 1, size, aspectRatio, seed } = options;
+
+  const text = typeof prompt === "string" ? prompt : prompt.text;
+  const images = typeof prompt === "string" ? undefined : prompt.images;
+
+  const { images: generatedImages } = await generateImage({
+    model,
+    prompt: images ? { text, images } : text,
+    n,
+    size,
+    aspectRatio,
+    seed,
+  });
+
+  return {
+    element: {
+      type,
+      text,
+      images: generatedImages.map((img) => img.uint8Array),
+    },
+  };
+}

package/src/ai-sdk/generate-music.ts

@@ -0,0 +1,46 @@
+import type { SharedV3ProviderOptions } from "@ai-sdk/provider";
+import type { MusicModelV3 } from "./music-model";
+
+export interface GenerateMusicOptions {
+  model: MusicModelV3;
+  prompt: string;
+  duration?: number;
+  providerOptions?: SharedV3ProviderOptions;
+  abortSignal?: AbortSignal;
+  headers?: Record<string, string | undefined>;
+}
+
+export interface GenerateMusicResult {
+  audio: {
+    uint8Array: Uint8Array;
+  };
+  warnings: Array<{ type: string; feature?: string; details?: string }>;
+  response: {
+    timestamp: Date;
+    modelId: string;
+    headers: Record<string, string> | undefined;
+  };
+}
+
+export async function generateMusic(
+  options: GenerateMusicOptions,
+): Promise<GenerateMusicResult> {
+  const { model, prompt, duration, providerOptions, abortSignal, headers } =
+    options;
+
+  const result = await model.doGenerate({
+    prompt,
+    duration,
+    providerOptions: providerOptions ?? {},
+    abortSignal,
+    headers,
+  });
+
+  return {
+    audio: {
+      uint8Array: result.audio,
+    },
+    warnings: result.warnings,
+    response: result.response,
+  };
+}

package/src/ai-sdk/generate-video.ts

@@ -0,0 +1,165 @@
+import type {
+  ImageModelV3File,
+  SharedV3ProviderOptions,
+  SharedV3Warning,
+} from "@ai-sdk/provider";
+import type { DataContent } from "ai";
+import type { VideoModelV3 } from "./video-model";
+
+export type GenerateVideoPrompt =
+  | string
+  | {
+      text?: string;
+      images?: Array<DataContent>;
+      audio?: DataContent;
+      video?: DataContent;
+    };
+
+export interface GenerateVideoOptions {
+  model: VideoModelV3;
+  prompt: GenerateVideoPrompt;
+  n?: number;
+  resolution?: `${number}x${number}`;
+  aspectRatio?: `${number}:${number}`;
+  duration?: number;
+  fps?: number;
+  seed?: number;
+  providerOptions?: SharedV3ProviderOptions;
+  abortSignal?: AbortSignal;
+  headers?: Record<string, string>;
+}
+
+export interface GeneratedVideo {
+  readonly base64: string;
+  readonly uint8Array: Uint8Array;
+  readonly mimeType: string;
+}
+
+export interface GenerateVideoResult {
+  readonly video: GeneratedVideo;
+  readonly videos: GeneratedVideo[];
+  readonly warnings: SharedV3Warning[];
+}
+
+class DefaultGeneratedVideo implements GeneratedVideo {
+  private _data: Uint8Array;
+  readonly mimeType = "video/mp4";
+
+  constructor(data: Uint8Array | string) {
+    if (typeof data === "string") {
+      this._data = Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
+    } else {
+      this._data = data;
+    }
+  }
+
+  get uint8Array(): Uint8Array {
+    return this._data;
+  }
+
+  get base64(): string {
+    let binary = "";
+    const bytes = this._data;
+    for (let i = 0; i < bytes.byteLength; i++) {
+      binary += String.fromCharCode(bytes[i]!);
+    }
+    return btoa(binary);
+  }
+}
+
+function toUint8Array(data: DataContent): Uint8Array {
+  if (typeof data === "string") {
+    return Uint8Array.from(atob(data), (c) => c.charCodeAt(0));
+  }
+  if (data instanceof ArrayBuffer) {
+    return new Uint8Array(data);
+  }
+  return data;
+}
+
+function normalizePrompt(prompt: GenerateVideoPrompt): {
+  prompt: string | undefined;
+  files: ImageModelV3File[] | undefined;
+} {
+  if (typeof prompt === "string") {
+    return { prompt, files: undefined };
+  }
+
+  const files: ImageModelV3File[] = [];
+
+  for (const img of prompt.images ?? []) {
+    files.push({
+      type: "file",
+      mediaType: "image/png",
+      data: toUint8Array(img),
+    });
+  }
+
+  if (prompt.audio) {
+    files.push({
+      type: "file",
+      mediaType: "audio/mpeg",
+      data: toUint8Array(prompt.audio),
+    });
+  }
+
+  if (prompt.video) {
+    files.push({
+      type: "file",
+      mediaType: "video/mp4",
+      data: toUint8Array(prompt.video),
+    });
+  }
+
+  return {
+    prompt: prompt.text,
+    files: files.length > 0 ? files : undefined,
+  };
+}
+
+export async function generateVideo(
+  options: GenerateVideoOptions,
+): Promise<GenerateVideoResult> {
+  const {
+    model,
+    prompt: promptArg,
+    n = 1,
+    resolution,
+    aspectRatio,
+    duration,
+    fps,
+    seed,
+    providerOptions = {},
+    abortSignal,
+    headers,
+  } = options;
+
+  const { prompt, files } = normalizePrompt(promptArg);
+
+  const result = await model.doGenerate({
+    prompt: prompt ?? "",
+    n,
+    resolution,
+    aspectRatio,
+    duration,
+    fps,
+    seed,
+    files,
+    providerOptions,
+    abortSignal,
+    headers,
+  });
+
+  const videos = result.videos.map((v) => new DefaultGeneratedVideo(v));
+  const warnings = result.warnings;
+
+  if (videos.length === 0) {
+    throw new Error("No videos generated");
+  }
+
+  return {
+    video: videos[0]!,
+    videos,
+    warnings,
+  };
+}

package/src/ai-sdk/index.ts

@@ -0,0 +1,72 @@
+export {
+  type CacheStorage,
+  clearCache,
+  type WithCacheOptions,
+  withCache,
+} from "./cache";
+export { File, files } from "./file";
+export { fileCache } from "./file-cache";
+export {
+  type Element,
+  type ElementType,
+  type GenerateElementOptions,
+  type GenerateElementResult,
+  generateElement,
+  scene,
+} from "./generate-element";
+export {
+  type GenerateMusicOptions,
+  type GenerateMusicResult,
+  generateMusic,
+} from "./generate-music";
+export {
+  type GenerateVideoOptions,
+  type GenerateVideoPrompt,
+  type GenerateVideoResult,
+  generateVideo,
+} from "./generate-video";
+export type {
+  MusicModelV3,
+  MusicModelV3CallOptions,
+  MusicModelV3ProviderMetadata,
+} from "./music-model";
+export {
+  type AudioTrack,
+  type Clip as EditlyClip,
+  type EditlyConfig,
+  editly,
+  type Layer as EditlyLayer,
+} from "./providers/editly";
+export {
+  createElevenLabs,
+  type ElevenLabsProvider,
+  elevenlabs,
+  VOICES,
+} from "./providers/elevenlabs-provider";
+export { createFal, type FalProvider, fal } from "./providers/fal-provider";
+export {
+  createHiggsfield,
+  type HiggsfieldImageModelSettings,
+  type HiggsfieldProvider,
+  type HiggsfieldProviderSettings,
+  higgsfield,
+} from "./providers/higgsfield";
+export {
+  createOpenAI,
+  type OpenAIProvider,
+  type OpenAIProviderSettings,
+  openai,
+} from "./providers/openai";
+export {
+  createReplicate,
+  type ReplicateProvider,
+  type ReplicateProviderSettings,
+  replicate,
+} from "./providers/replicate";
+export type {
+  VideoModelV3,
+  VideoModelV3CallOptions,
+  VideoModelV3File,
+  VideoModelV3ProviderMetadata,
+  VideoModelV3Usage,
+} from "./video-model";