vargai 0.4.0-alpha73 → 0.4.0-alpha74

package/package.json

@@ -71,7 +71,7 @@
     "zod": "^4.2.1"
   },
   "sideEffects": false,
-  "version": "0.4.0-alpha73",
+  "version": "0.4.0-alpha74",
   "exports": {
     ".": "./src/index.ts",
     "./ai": "./src/ai-sdk/index.ts",

package/src/ai-sdk/providers/elevenlabs.ts

@@ -9,8 +9,17 @@ import {
   type SpeechModelV3CallOptions,
 } from "@ai-sdk/provider";
 import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js";
+import type { ElevenLabsCharacterAlignment } from "../../speech/types";
 import type { MusicModelV3, MusicModelV3CallOptions } from "../music-model";
 
+/**
+ * Curated name → voice_id mapping for backward-compatible friendly names.
+ * These are convenience aliases only — any valid ElevenLabs voice_id can be
+ * passed directly as the `voice` parameter and it will be forwarded as-is.
+ *
+ * For the full catalog of 600+ voices, use voice_id strings directly or
+ * call the gateway's GET /v1/voices endpoint to browse/search.
+ */
 const VOICES: Record<string, string> = {
   rachel: "21m00Tcm4TlvDq8ikWAM",
   domi: "AZnzlk1XvdvUeBnXmlld",
@@ -105,11 +114,11 @@ class ElevenLabsSpeechModel implements SpeechModelV3 {
   readonly provider = "elevenlabs";
   readonly modelId: string;
 
-  private client: ElevenLabsClient;
+  private apiKey: string;
 
-  constructor(modelId: string, client: ElevenLabsClient) {
+  constructor(modelId: string, apiKey: string) {
     this.modelId = modelId;
-    this.client = client;
+    this.apiKey = apiKey;
   }
 
   async doGenerate(options: SpeechModelV3CallOptions) {
@@ -127,31 +136,80 @@ class ElevenLabsSpeechModel implements SpeechModelV3 {
       });
     }
 
-    const elevenLabsOptions = providerOptions?.elevenlabs ?? {};
-    const audio = await this.client.textToSpeech.convert(voiceId, {
-      text,
-      modelId: model,
-      outputFormat: "mp3_44100_128",
-      ...elevenLabsOptions,
-    } as Parameters<typeof this.client.textToSpeech.convert>[1]);
-
-    const reader = audio.getReader();
-    const chunks: Uint8Array[] = [];
-
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      chunks.push(value);
+    const elevenLabsOptions = (providerOptions?.elevenlabs ?? {}) as Record<
+      string,
+      unknown
+    >;
+
+    // Call the /with-timestamps endpoint via raw fetch.
+    // Returns JSON with base64 audio + character-level alignment.
+    const controller = new AbortController();
+    const timeoutMs = 120_000; // 2 minutes — generous for long-form TTS
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+    let response: Response;
+    try {
+      response = await fetch(
+        `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps?output_format=mp3_44100_128`,
+        {
+          method: "POST",
+          headers: {
+            "xi-api-key": this.apiKey,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            text,
+            model_id: model,
+            ...elevenLabsOptions,
+          }),
+          signal: controller.signal,
+        },
+      );
+    } catch (error) {
+      clearTimeout(timer);
+      if (error instanceof DOMException && error.name === "AbortError") {
+        throw new Error(
+          `ElevenLabs speech timed out after ${timeoutMs / 1000}s for voice ${voiceId}`,
+        );
+      }
+      throw error;
     }
+    clearTimeout(timer);
 
-    const totalLength = chunks.reduce((acc, c) => acc + c.length, 0);
-    const result = new Uint8Array(totalLength);
-    let offset = 0;
-    for (const chunk of chunks) {
-      result.set(chunk, offset);
-      offset += chunk.length;
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(
+        `ElevenLabs speech with timestamps failed (${response.status}): ${errorText}`,
+      );
     }
 
+    const json = (await response.json()) as {
+      audio_base64: string;
+      alignment?: ElevenLabsCharacterAlignment;
+      normalized_alignment?: ElevenLabsCharacterAlignment;
+    };
+
+    // Decode base64 audio to binary
+    const audioBytes = Buffer.from(json.audio_base64, "base64");
+    const result = new Uint8Array(
+      audioBytes.buffer,
+      audioBytes.byteOffset,
+      audioBytes.byteLength,
+    );
+
+    // Pack alignment data into providerMetadata so the AI SDK passes it through.
+    // biome-ignore lint/suspicious/noExplicitAny: JSON.parse returns any, matching JSONObject
+    const providerMetadata: Record<string, any> | undefined = json.alignment
+      ? JSON.parse(
+          JSON.stringify({
+            elevenlabs: {
+              alignment: json.alignment,
+              normalizedAlignment: json.normalized_alignment,
+            },
+          }),
+        )
+      : undefined;
+
     return {
       audio: result,
       warnings,
@@ -160,6 +218,7 @@ class ElevenLabsSpeechModel implements SpeechModelV3 {
         modelId: this.modelId,
         headers: undefined,
       },
+      providerMetadata,
     };
   }
 }
@@ -191,7 +250,7 @@ export function createElevenLabs(
   return {
     specificationVersion: "v3",
     speechModel(modelId = ELEVENLABS_DEFAULTS.speechModel) {
-      return new ElevenLabsSpeechModel(modelId, client);
+      return new ElevenLabsSpeechModel(modelId, apiKey);
     },
     musicModel(modelId = ELEVENLABS_DEFAULTS.musicModel) {
       return new ElevenLabsMusicModel(modelId, client);

package/src/core/schema/shared.ts

@@ -21,7 +21,15 @@ export type VideoDurationString = z.infer<typeof videoDurationStringSchema>;
 export const resolutionSchema = z.enum(["480p", "720p", "1080p"]);
 export type Resolution = z.infer<typeof resolutionSchema>;
 
-// ElevenLabs preset voices
+// Voice parameter: accepts any voice name or ElevenLabs voice_id string.
+// ElevenLabs has 1000+ voices — pass a voice_id directly for full catalog access.
+// Common names ("rachel", "adam", etc.) are resolved to voice_ids automatically.
+export const voiceSchema = z
+  .string()
+  .min(1, "Voice name or voice_id cannot be empty");
+export type Voice = z.infer<typeof voiceSchema>;
+
+/** @deprecated Use voiceSchema instead. Kept for backward compatibility. */
 export const voiceNameSchema = z.enum([
   "rachel",
   "domi",
@@ -36,6 +44,21 @@ export const voiceNameSchema = z.enum([
 ]);
 export type VoiceName = z.infer<typeof voiceNameSchema>;
 
+// Well-known voice names for quick reference in skills/prompts.
+// These are convenience aliases — any valid ElevenLabs voice_id also works.
+export const WELL_KNOWN_VOICE_NAMES = [
+  "rachel",
+  "domi",
+  "sarah",
+  "bella",
+  "antoni",
+  "elli",
+  "josh",
+  "arnold",
+  "adam",
+  "sam",
+] as const;
+
 // Simplified voice set (commonly used in skills)
 export const simpleVoiceSchema = z.enum(["rachel", "sam", "adam", "josh"]);
 export type SimpleVoice = z.infer<typeof simpleVoiceSchema>;

package/src/definitions/actions/voice.ts

@@ -4,15 +4,17 @@
  */
 
 import { z } from "zod";
-import { filePathSchema, voiceNameSchema } from "../../core/schema/shared";
+import { filePathSchema, voiceSchema } from "../../core/schema/shared";
 import type { ActionDefinition, ZodSchema } from "../../core/schema/types";
 import { elevenlabsProvider, VOICES } from "../../providers/elevenlabs";
 import { storageProvider } from "../../providers/storage";
 
-// Input schema with Zod
+// Input schema with Zod — accepts any voice name or ElevenLabs voice_id
 const voiceInputSchema = z.object({
   text: z.string().describe("Text to convert to speech"),
-  voice: voiceNameSchema.default("rachel").describe("Voice to use"),
+  voice: voiceSchema
+    .default("rachel")
+    .describe("Voice name or ElevenLabs voice_id"),
   output: filePathSchema.optional().describe("Output file path"),
 });
 
@@ -58,10 +60,11 @@ export interface VoiceResult {
   uploadUrl?: string;
 }
 
-// Voice name to ID mapping
+// Voice name to ID mapping. Unknown names pass through as voice_ids.
 const VOICE_MAP: Record<string, string> = {
   rachel: VOICES.RACHEL,
   domi: VOICES.DOMI,
+  sarah: VOICES.SARAH,
   bella: VOICES.BELLA,
   antoni: VOICES.ANTONI,
   elli: VOICES.ELLI,

package/src/providers/elevenlabs.ts

@@ -186,7 +186,11 @@ export class ElevenLabsProvider extends BaseProvider {
   }
 }
 
-// Popular voices
+/**
+ * Curated voice_id constants for common ElevenLabs voices.
+ * For the full catalog of 600+ voices, use voice_ids directly or
+ * call the gateway's GET /v1/voices endpoint to browse/search.
+ */
 export const VOICES = {
   RACHEL: "21m00Tcm4TlvDq8ikWAM",
   DOMI: "AZnzlk1XvdvUeBnXmlld",

package/src/react/examples/async/speech-segments-voiceover.tsx

@@ -0,0 +1,113 @@
+/**
+ * Speech segments demo — single continuous voiceover.
+ *
+ *   Scene 1: VEED lipsync talking head  (segment 0 for lipsync, audio muted)
+ *   Scene 2: image b-roll               (no per-clip audio)
+ *   Scene 3: VEED lipsync talking head   (segment 2 for lipsync, audio muted)
+ *
+ * One full voiceover plays at the Render level — smooth, continuous audio
+ * with no splicing artifacts. VEED videos use keepAudio: false so the
+ * baked-in lipsync audio doesn't double up with the voiceover.
+ *
+ * Segments are only used for:
+ *   - Feeding audio to VEED for lipsync generation
+ *   - Setting clip durations from segment timing
+ *
+ * Run: bun run src/react/examples/async/speech-segments-voiceover.tsx
+ * Output: output/speech-segments-voiceover.mp4
+ */
+
+import { elevenlabs } from "../../../ai-sdk/providers/elevenlabs";
+import { fal } from "../../../ai-sdk/providers/fal";
+import { Clip, Image, Render, render, Speech, Video } from "../..";
+
+// --- One speech call, three segments ---
+const { audio, segments } = await Speech({
+  model: elevenlabs.speechModel("eleven_v3"),
+  voice: "adam",
+  children: [
+    "Scientists always lied to you about bananas.",
+    "Bananas are normally dangerous, they can kill your gut health.",
+    'The actual issue is Banana bacteria called "alupios manurale" causing food poisoning symptoms.',
+  ],
+});
+
+console.log(`Total duration: ${audio.duration.toFixed(2)}s`);
+console.log(`Segments: ${segments.length}`);
+for (const [i, seg] of segments.entries()) {
+  console.log(
+    `  [${i}] ${seg.start.toFixed(2)}s -> ${seg.end.toFixed(2)}s (${seg.duration.toFixed(2)}s) "${seg.text}"`,
+  );
+}
+
+// --- Portrait for the talking head ---
+const portrait = Image({
+  prompt:
+    "Ultra-realistic studio portrait of a serious male scientist in his 40s, lab coat, glasses, concerned expression, dramatic lighting, dark background, documentary style",
+  model: fal.imageModel("nano-banana-pro"),
+  aspectRatio: "9:16",
+});
+
+// --- Scene 1: lipsync (segment audio for VEED, but muted in final video) ---
+const talking1 = Video({
+  model: fal.videoModel("veed-fabric-1.0"),
+  keepAudio: false, // muted — full voiceover handles audio
+  prompt: { images: [portrait], audio: segments[0] },
+  providerOptions: { fal: { resolution: "720p" } },
+});
+
+// --- Scene 3: lipsync (segment audio for VEED, but muted in final video) ---
+const talking3 = Video({
+  model: fal.videoModel("veed-fabric-1.0"),
+  keepAudio: false, // muted — full voiceover handles audio
+  prompt: { images: [portrait], audio: segments[2] },
+  providerOptions: { fal: { resolution: "720p" } },
+});
+
+const demo = (
+  <Render width={1080} height={1920}>
+    {/* Scene 1: talking head */}
+    <Clip duration={segments[0]!.duration}>{talking1}</Clip>
+
+    {/* Scene 2: banana b-roll (no per-clip audio — voiceover covers it) */}
+    <Clip duration={segments[1]!.duration}>
+      <Image
+        prompt="macro shot of a dangerous banana with dramatic dark lighting, bacteria visualization, medical documentary style, gut health danger concept"
+        model={fal.imageModel("nano-banana-pro")}
+        aspectRatio="9:16"
+        zoom="in"
+      />
+    </Clip>
+
+    {/* Scene 3: talking head */}
+    <Clip duration={segments[2]!.duration}>{talking3}</Clip>
+
+    {/* Full continuous voiceover — smooth, no splicing */}
+    {audio}
+  </Render>
+);
+
+async function main() {
+  if (!process.env.FAL_API_KEY && !process.env.FAL_KEY) {
+    console.error("ERROR: FAL_API_KEY/FAL_KEY not found in environment");
+    process.exit(1);
+  }
+  if (!process.env.ELEVENLABS_API_KEY) {
+    console.error("ERROR: ELEVENLABS_API_KEY not found in environment");
+    process.exit(1);
+  }
+
+  const result = await render(demo, {
+    output: "output/speech-segments-voiceover.mp4",
+    cache: ".cache/ai-speech-segments-voiceover",
+  });
+
+  console.log(
+    `Done: output/speech-segments-voiceover.mp4 (${(result.video.byteLength / 1024 / 1024).toFixed(2)} MB)`,
+  );
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/src/react/examples/async/speech-segments.tsx

@@ -0,0 +1,109 @@
+/**
+ * Speech segments demo — per-clip audio, hard cuts between scenes.
+ *
+ *   Scene 1: VEED lipsync talking head  (segment 0, keepAudio)
+ *   Scene 2: image b-roll + voiceover   (segment 1 as clip child)
+ *   Scene 3: VEED lipsync talking head   (segment 2, keepAudio)
+ *
+ * Each clip carries its own segment audio. Hard cuts between scenes —
+ * no crossfade transitions, so audio from adjacent clips never overlaps.
+ * This is the cleanest approach for per-clip audio.
+ *
+ * For smooth audio with crossfade transitions, use the single-voiceover
+ * pattern instead (see speech-segments-voiceover.tsx).
+ *
+ * Run: bun run src/react/examples/async/speech-segments.tsx
+ * Output: output/speech-segments.mp4
+ */
+
+import { elevenlabs } from "../../../ai-sdk/providers/elevenlabs";
+import { fal } from "../../../ai-sdk/providers/fal";
+import { Clip, Image, Render, render, Speech, Video } from "../..";
+
+// --- One speech call, three segments ---
+const { segments } = await Speech({
+  model: elevenlabs.speechModel("eleven_v3"),
+  voice: "adam",
+  children: [
+    "Scientists always lied to you about bananas.",
+    "Bananas are normally dangerous, they can kill your gut health.",
+    'The actual issue is Banana bacteria called "alupios manurale" causing food poisoning symptoms.',
+  ],
+});
+
+console.log(`Segments: ${segments.length}`);
+for (const [i, seg] of segments.entries()) {
+  console.log(
+    `  [${i}] ${seg.start.toFixed(2)}s -> ${seg.end.toFixed(2)}s (${seg.duration.toFixed(2)}s) "${seg.text}"`,
+  );
+}
+
+// --- Portrait for the talking head ---
+const portrait = Image({
+  prompt:
+    "Ultra-realistic studio portrait of a serious male scientist in his 40s, lab coat, glasses, concerned expression, dramatic lighting, dark background, documentary style",
+  model: fal.imageModel("nano-banana-pro"),
+  aspectRatio: "9:16",
+});
+
+// --- Scene 1: lipsync talking head ---
+const talking1 = Video({
+  model: fal.videoModel("veed-fabric-1.0"),
+  keepAudio: true,
+  prompt: { images: [portrait], audio: segments[0] },
+  providerOptions: { fal: { resolution: "720p" } },
+});
+
+// --- Scene 3: lipsync talking head ---
+const talking3 = Video({
+  model: fal.videoModel("veed-fabric-1.0"),
+  keepAudio: true,
+  prompt: { images: [portrait], audio: segments[2] },
+  providerOptions: { fal: { resolution: "720p" } },
+});
+
+const demo = (
+  <Render width={1080} height={1920}>
+    {/* Scene 1: talking head */}
+    <Clip duration={segments[0]!.duration}>{talking1}</Clip>
+
+    {/* Scene 2: b-roll + segment voiceover */}
+    <Clip duration={segments[1]!.duration}>
+      <Image
+        prompt="macro shot of a dangerous banana with dramatic dark lighting, bacteria visualization, medical documentary style, gut health danger concept"
+        model={fal.imageModel("nano-banana-pro")}
+        aspectRatio="9:16"
+        zoom="in"
+      />
+      {segments[1]}
+    </Clip>
+
+    {/* Scene 3: talking head */}
+    <Clip duration={segments[2]!.duration}>{talking3}</Clip>
+  </Render>
+);
+
+async function main() {
+  if (!process.env.FAL_API_KEY && !process.env.FAL_KEY) {
+    console.error("ERROR: FAL_API_KEY/FAL_KEY not found in environment");
+    process.exit(1);
+  }
+  if (!process.env.ELEVENLABS_API_KEY) {
+    console.error("ERROR: ELEVENLABS_API_KEY not found in environment");
+    process.exit(1);
+  }
+
+  const result = await render(demo, {
+    output: "output/speech-segments.mp4",
+    cache: ".cache/ai-speech-segments",
+  });
+
+  console.log(
+    `Done: output/speech-segments.mp4 (${(result.video.byteLength / 1024 / 1024).toFixed(2)} MB)`,
+  );
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/src/react/index.ts

@@ -1,6 +1,7 @@
 export type { CacheStorage } from "../ai-sdk/cache";
 export { File } from "../ai-sdk/file";
 export type { SizeValue } from "../ai-sdk/providers/editly/types";
+export type { Segment, WordTiming } from "../speech/types";
 export { assets } from "./assets";
 export {
   Captions,

package/src/react/renderers/captions.ts

@@ -251,39 +251,53 @@ export async function renderCaptions(
           : await renderSpeech(props.src, ctx);
       audioPath = await ctx.backend.resolvePath(speechFile);
 
-      const transcribeTaskId = ctx.progress
-        ? addTask(ctx.progress, "transcribe", "groq-whisper")
-        : null;
-      if (transcribeTaskId && ctx.progress)
-        startTask(ctx.progress, transcribeTaskId);
-
-      const audioData =
-        audioPath.startsWith("http://") || audioPath.startsWith("https://")
-          ? await fetch(audioPath).then((res) => res.arrayBuffer())
-          : await Bun.file(audioPath).arrayBuffer();
-
-      const result = await transcribe({
-        model: groq.transcription("whisper-large-v3"),
-        audio: new Uint8Array(audioData),
-        providerOptions: {
-          groq: {
-            responseFormat: "verbose_json",
-            timestampGranularities: ["word"],
-          },
-        },
-      });
-
-      if (transcribeTaskId && ctx.progress)
-        completeTask(ctx.progress, transcribeTaskId);
-
-      const rawBody = (result.responses[0] as { body?: unknown })?.body;
-      const parsed = groqResponseSchema.safeParse(rawBody);
-      const words = parsed.success ? parsed.data.words : undefined;
-
-      if (!words || words.length === 0) {
-        srtContent = `1\n00:00:00,000 --> 00:00:05,000\n${result.text}\n`;
+      // Check if the speech element already has word-level timing from ElevenLabs.
+      // If so, skip the Whisper transcription step entirely (saves time and cost).
+      const nativeWords =
+        props.src instanceof ResolvedElement ? props.src.meta.words : undefined;
+
+      if (nativeWords && nativeWords.length > 0) {
+        // Use native ElevenLabs word timing — same shape as GroqWord
+        srtContent = convertToSRT(nativeWords);
       } else {
-        srtContent = convertToSRT(words);
+        // Fallback: transcribe with Groq Whisper to get word-level timestamps
+        const transcribeTaskId = ctx.progress
+          ? addTask(ctx.progress, "transcribe", "groq-whisper")
+          : null;
+        if (transcribeTaskId && ctx.progress)
+          startTask(ctx.progress, transcribeTaskId);
+
+        let result: Awaited<ReturnType<typeof transcribe>>;
+        try {
+          const audioData =
+            audioPath.startsWith("http://") || audioPath.startsWith("https://")
+              ? await fetch(audioPath).then((res) => res.arrayBuffer())
+              : await Bun.file(audioPath).arrayBuffer();
+
+          result = await transcribe({
+            model: groq.transcription("whisper-large-v3"),
+            audio: new Uint8Array(audioData),
+            providerOptions: {
+              groq: {
+                responseFormat: "verbose_json",
+                timestampGranularities: ["word"],
+              },
+            },
+          });
+        } finally {
+          if (transcribeTaskId && ctx.progress)
+            completeTask(ctx.progress, transcribeTaskId);
+        }
+
+        const rawBody = (result.responses[0] as { body?: unknown })?.body;
+        const parsed = groqResponseSchema.safeParse(rawBody);
+        const words = parsed.success ? parsed.data.words : undefined;
+
+        if (!words || words.length === 0) {
+          srtContent = `1\n00:00:00,000 --> 00:00:05,000\n${result.text}\n`;
+        } else {
+          srtContent = convertToSRT(words);
+        }
       }
 
       srtPath = `/tmp/varg-captions-${Date.now()}.srt`;

package/src/react/renderers/resolve-lazy.ts

@@ -11,6 +11,7 @@
  * Must be called BEFORE renderRoot() so the render pipeline sees a fully
  * static VargElement tree.
  */
+import { ResolvedElement } from "../resolved-element";
 import type { VargElement, VargNode } from "../types";
 
 export async function resolveLazy(node: VargNode): Promise<VargNode> {
@@ -53,6 +54,13 @@ export async function resolveLazy(node: VargNode): Promise<VargNode> {
     }
   }
 
+  // Preserve ResolvedElement instances (segments, pre-resolved speech/images).
+  // Spreading would destroy the prototype and break `instanceof` checks
+  // used by the clip renderer to detect pre-resolved elements.
+  if (element instanceof ResolvedElement) {
+    return element;
+  }
+
   // Return a new element with resolved children (don't mutate the original).
   // IMPORTANT: strip .then from the spread to prevent Promise.all from
   // treating the result as a thenable (Image/Speech/etc. elements are thenable).