videoclaw 3.0.0-alpha.1

package/src/video/audio-platform/native-gemini-tts.ts

@@ -0,0 +1,226 @@
+/**
+ * SCAFFOLD: Gemini TTS request/response per docs; verify against a real key.
+ *
+ * Gemini TTS backend — wraps the generativelanguage.googleapis.com
+ * `gemini-2.5-flash-preview-tts:generateContent` endpoint. This is an
+ * API-KEY product (NOT Vertex), so it resolves a key from the existing
+ * Gemini key pool (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY) via
+ * `fetchGeminiWithPool`. When a caller passes `input.env` carrying a key the
+ * process.env-backed pool cannot see, that key is forwarded as the pool's
+ * `keyOverride` so library callers authenticate without mutating process.env.
+ *
+ * The model returns RAW PCM (signed 16-bit little-endian, 24kHz, mono) as
+ * base64 in candidates[0].content.parts[].inlineData.data. We wrap that PCM
+ * in a 44-byte WAV header and write the result to `outputPath` (a .wav).
+ *
+ * Duration is computed deterministically from the PCM byte count — no ffprobe:
+ *   durationMs = round(pcmBytes / (24000 * 2) * 1000)
+ *
+ * Availability: a Gemini key is resolvable (pool size > 0 or one of the env
+ * vars is set). The backend is inert (unavailable) without a key and never
+ * throws on the default path; on a missing/empty audio response it throws
+ * VclawError('tts_failed', ...).
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { fetchGeminiWithPool } from '../gemini-key-pool.js';
+import { VclawError } from '../errors.js';
+import type { TtsBackend, TtsGenInput, TtsGenResult } from './types.js';
+
+const TTS_MODEL = 'gemini-2.5-flash-preview-tts';
+const TTS_SAMPLE_RATE_HZ = 24_000;
+const TTS_BYTES_PER_SAMPLE = 2; // s16le
+const TTS_CHANNELS = 1; // mono
+const DEFAULT_VOICE = 'Kore';
+
+/** Chars-per-second heuristic for dry-run duration estimation (~14 cps speech). */
+const DRYRUN_CHARS_PER_SEC = 14;
+
+/**
+ * Gemini API key sources, in the same precedence as the global key pool
+ * (gemini-key-pool.ts). Used to resolve an explicit key from a caller-supplied
+ * `input.env` so the pool (which only reads process.env) can be bypassed.
+ */
+const GEMINI_KEY_VARS = ['GEMINI_API_KEYS', 'GOOGLE_API_KEYS', 'GOOGLE_API_KEY'] as const;
+
+/**
+ * Resolve the first key from a caller-supplied env that differs from
+ * process.env. Returns undefined when no env was passed, or when the passed env
+ * adds no key beyond what process.env already exposes (in which case the global
+ * pool handles key selection). Splits on the same delimiters the pool uses and
+ * returns the first token.
+ */
+function resolveKeyOverrideFromEnv(env: NodeJS.ProcessEnv | undefined): string | undefined {
+  if (!env || env === process.env) return undefined;
+  for (const varName of GEMINI_KEY_VARS) {
+    const raw = env[varName];
+    if (typeof raw !== 'string' || raw.trim() === '') continue;
+    // process.env already carries this exact value → let the pool handle it.
+    if (process.env[varName] === raw) continue;
+    const first = raw.split(/[,;\n\s]+/).map((t) => t.trim()).find((t) => t.length > 0);
+    if (first) return first;
+  }
+  return undefined;
+}
+
+function buildTtsUrl(key: string): string {
+  return `https://generativelanguage.googleapis.com/v1beta/models/${TTS_MODEL}:generateContent?key=${encodeURIComponent(key)}`;
+}
+
+/**
+ * durationMs from raw PCM byte count (s16le 24kHz mono).
+ * pcmBytes / (sampleRate * bytesPerSample * channels) * 1000
+ */
+function pcmDurationMs(pcmBytes: number): number {
+  return Math.round((pcmBytes / (TTS_SAMPLE_RATE_HZ * TTS_BYTES_PER_SAMPLE * TTS_CHANNELS)) * 1000);
+}
+
+/** Wrap raw PCM (s16le 24kHz mono) bytes in a canonical 44-byte WAV header. */
+function wrapPcmInWav(pcm: Buffer): Buffer {
+  const byteRate = TTS_SAMPLE_RATE_HZ * TTS_CHANNELS * TTS_BYTES_PER_SAMPLE;
+  const blockAlign = TTS_CHANNELS * TTS_BYTES_PER_SAMPLE;
+  const header = Buffer.alloc(44);
+  header.write('RIFF', 0, 'ascii');
+  header.writeUInt32LE(36 + pcm.length, 4); // ChunkSize
+  header.write('WAVE', 8, 'ascii');
+  header.write('fmt ', 12, 'ascii');
+  header.writeUInt32LE(16, 16); // Subchunk1Size (PCM)
+  header.writeUInt16LE(1, 20); // AudioFormat = PCM
+  header.writeUInt16LE(TTS_CHANNELS, 22);
+  header.writeUInt32LE(TTS_SAMPLE_RATE_HZ, 24);
+  header.writeUInt32LE(byteRate, 28);
+  header.writeUInt16LE(blockAlign, 32);
+  header.writeUInt16LE(TTS_BYTES_PER_SAMPLE * 8, 34); // BitsPerSample
+  header.write('data', 36, 'ascii');
+  header.writeUInt32LE(pcm.length, 40); // Subchunk2Size
+  return Buffer.concat([header, pcm]);
+}
+
+/** Defensive extraction of the first base64 inlineData audio payload. */
+function extractAudioBase64(parsed: unknown): string | undefined {
+  if (typeof parsed !== 'object' || parsed === null) return undefined;
+  const candidates = (parsed as { candidates?: unknown }).candidates;
+  if (!Array.isArray(candidates) || candidates.length === 0) return undefined;
+  const parts = (candidates[0] as { content?: { parts?: unknown } })?.content?.parts;
+  if (!Array.isArray(parts)) return undefined;
+  for (const part of parts) {
+    const data = (part as { inlineData?: { data?: unknown } })?.inlineData?.data;
+    if (typeof data === 'string' && data.length > 0) return data;
+  }
+  return undefined;
+}
+
+export const geminiTts: TtsBackend = {
+  id: 'gemini-tts',
+  kind: 'tts',
+  outputExtension: 'wav',
+  displayName: 'Gemini TTS (gemini-2.5-flash-preview-tts)',
+  // API-key product, not Vertex. Availability is resolved from the Gemini key
+  // pool (see isTtsBackendAvailable), so requiredEnv stays empty and the
+  // registry gates on requiresGeminiKey instead of this backend's id.
+  requiredEnv: [],
+  requiresVertex: false,
+  requiresGeminiKey: true,
+  summary:
+    'Single-speaker text-to-speech via the Gemini API (gemini-2.5-flash-preview-tts). Returns 24kHz mono PCM wrapped as WAV. Requires a Gemini API key (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY).',
+
+  async generate(input: TtsGenInput): Promise<TtsGenResult> {
+    const voice = input.voice && input.voice.trim() ? input.voice.trim() : DEFAULT_VOICE;
+
+    if (!input.text || !input.text.trim()) {
+      throw new VclawError('tts_failed', 'Cannot synthesize narration from empty text.', {
+        backendId: 'gemini-tts',
+      });
+    }
+
+    // Dry-run: estimate duration from text length; do NOT touch network or do
+    // real synthesis. Write a placeholder WAV (44-byte header, no samples) so
+    // downstream existsSync() checks stay consistent.
+    if (input.dryRun) {
+      const estSec = Math.max(1, input.text.trim().length / DRYRUN_CHARS_PER_SEC);
+      const durationMs = Math.round(estSec * 1000);
+      await mkdir(dirname(input.outputPath), { recursive: true });
+      await writeFile(input.outputPath, wrapPcmInWav(Buffer.alloc(0)));
+      return { path: input.outputPath, durationMs, backendId: 'gemini-tts' };
+    }
+
+    const requestBody = JSON.stringify({
+      contents: [{ parts: [{ text: input.text }] }],
+      generationConfig: {
+        responseModalities: ['AUDIO'],
+        speechConfig: {
+          voiceConfig: {
+            prebuiltVoiceConfig: { voiceName: voice },
+          },
+        },
+      },
+    });
+
+    // Resolve a key from input.env when it carries one the global pool (which
+    // only reads process.env) cannot see, so a library caller passing
+    // env: { GEMINI_API_KEYS: '...' } actually authenticates instead of hitting
+    // 'No Gemini API keys configured'. Absent that, the pool handles selection.
+    const keyOverride = resolveKeyOverrideFromEnv(input.env);
+
+    const poolOptions: {
+      fetcher?: typeof fetch;
+      keyOverride?: string;
+    } = {};
+    if (input.fetcher) poolOptions.fetcher = input.fetcher;
+    if (keyOverride) poolOptions.keyOverride = keyOverride;
+
+    const response = await fetchGeminiWithPool(
+      buildTtsUrl,
+      {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: requestBody,
+      },
+      poolOptions,
+    );
+
+    if (!response.ok) {
+      const body = await response.text().catch(() => '');
+      throw new VclawError(
+        'tts_failed',
+        `Gemini TTS request failed with HTTP ${response.status}.`,
+        { backendId: 'gemini-tts', status: response.status, body: body.slice(0, 500) },
+      );
+    }
+
+    let parsed: unknown;
+    try {
+      parsed = await response.json();
+    } catch (err) {
+      throw new VclawError('tts_failed', 'Gemini TTS returned a non-JSON response.', {
+        backendId: 'gemini-tts',
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+
+    const b64 = extractAudioBase64(parsed);
+    if (!b64) {
+      throw new VclawError('tts_failed', 'Gemini TTS response contained no audio data.', {
+        backendId: 'gemini-tts',
+      });
+    }
+
+    const pcm = Buffer.from(b64, 'base64');
+    if (pcm.length === 0) {
+      throw new VclawError('tts_failed', 'Gemini TTS returned an empty audio payload.', {
+        backendId: 'gemini-tts',
+      });
+    }
+
+    const wav = wrapPcmInWav(pcm);
+    await mkdir(dirname(input.outputPath), { recursive: true });
+    await writeFile(input.outputPath, wav);
+
+    return {
+      path: input.outputPath,
+      durationMs: pcmDurationMs(pcm.length),
+      backendId: 'gemini-tts',
+    };
+  },
+};

package/src/video/audio-platform/native-lyria.ts

@@ -0,0 +1,231 @@
+/**
+ * Music generation via Google Vertex AI Lyria (predict endpoint).
+ *
+ * VERIFIED LIVE against `lyria-002` (Lyria 2, GA) on 2026-06-01 — the request
+ * body and response field below match the real API. INERT / unavailable
+ * without a Vertex project (GOOGLE_CLOUD_PROJECT or VERTEX_PROJECT must be set).
+ *
+ *   POST https://{REGION}-aiplatform.googleapis.com/v1/projects/{PROJECT}/
+ *        locations/{REGION}/publishers/google/models/{MODEL}:predict
+ *   Authorization: Bearer <token>
+ *   Body: { instances: [{ prompt: string }], parameters: { sampleCount: 1 } }
+ *   Response: { predictions: [{ bytesBase64Encoded: string (WAV), mimeType }] }
+ *
+ * MODEL (via LYRIA_MODEL env, default lyria-002):
+ *   - lyria-002             Lyria 2, GA, 30s, text-only             ($0.06/30s)
+ *   - lyria-3-clip-preview  Lyria 3, 30s, text+image  (PREVIEW — needs project allowlist)
+ *   - lyria-3-pro-preview   Lyria 3 Pro, ~184s full song, text+image (PREVIEW — allowlist)
+ * NOTE: the Lyria 3 image-input (multimodal) request shape is NOT yet wired or
+ * verified here — it needs preview allowlist access; add + verify once granted.
+ *
+ * Auth (tried in order): GOOGLE_ACCESS_TOKEN env -> GCE metadata server ->
+ * `gcloud auth print-access-token` (local dev, dependency-free). For prod/CI,
+ * set GOOGLE_ACCESS_TOKEN or run on a GCE/Cloud Run service account.
+ * requiresVertex: true.
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { VclawError } from '../errors.js';
+import type { MusicBackend, MusicGenInput, MusicGenResult } from './types.js';
+
+const execFileAsync = promisify(execFile);
+
+const DEFAULT_LYRIA_REGION = 'us-central1';
+const DEFAULT_LYRIA_MODEL = 'lyria-002';
+
+/**
+ * True duration (ms) of a RIFF/WAVE buffer, read from its fmt byteRate + data
+ * chunk size. Lyria returns a full WAV, so this reports the real clip length
+ * (~32.8s for lyria-002) rather than the requested-duration default. Returns
+ * null for a non-WAV / unparseable buffer (caller falls back to the request).
+ */
+export function wavDurationMs(buf: Buffer): number | null {
+  if (
+    buf.length < 44 ||
+    buf.toString('ascii', 0, 4) !== 'RIFF' ||
+    buf.toString('ascii', 8, 12) !== 'WAVE'
+  ) {
+    return null;
+  }
+  let byteRate = 0;
+  let dataSize = 0;
+  let off = 12;
+  while (off + 8 <= buf.length) {
+    const id = buf.toString('ascii', off, off + 4);
+    const size = buf.readUInt32LE(off + 4);
+    if (id === 'fmt ' && off + 24 <= buf.length) {
+      byteRate = buf.readUInt32LE(off + 16); // fmt body: +8 fmt/chan, +12 rate, +16 byteRate
+    } else if (id === 'data') {
+      dataSize = size;
+      break;
+    }
+    off += 8 + size + (size % 2); // chunks are word-aligned
+  }
+  if (byteRate > 0 && dataSize > 0) return Math.round((dataSize / byteRate) * 1000);
+  return null;
+}
+
+/**
+ * Resolve the Vertex project id from explicit env or well-known fallback vars.
+ */
+function resolveVertexProject(env: NodeJS.ProcessEnv): string | undefined {
+  return env['GOOGLE_CLOUD_PROJECT'] ?? env['VERTEX_PROJECT'] ?? env['GCLOUD_PROJECT'];
+}
+
+/**
+ * Obtain a Vertex bearer token. Tries GOOGLE_APPLICATION_CREDENTIALS / ADC
+ * via the metadata server token endpoint; falls back to the
+ * GOOGLE_ACCESS_TOKEN env var for local-dev / CI injection.
+ *
+ * SCAFFOLD: In production this should use the Google Auth Library
+ * (e.g. google-auth-library) to obtain a proper access token via ADC.
+ * The current implementation accepts a pre-resolved token via env var
+ * GOOGLE_ACCESS_TOKEN for testability without a real service account.
+ */
+async function resolveVertexToken(
+  env: NodeJS.ProcessEnv,
+  fetcher: typeof fetch,
+): Promise<string> {
+  const explicit = env['GOOGLE_ACCESS_TOKEN'];
+  if (explicit) return explicit;
+
+  // Attempt GCE metadata server (works on Cloud Run / GCE VMs).
+  try {
+    const metaResp = await fetcher(
+      'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token',
+      { headers: { 'Metadata-Flavor': 'Google' } },
+    );
+    if (metaResp.ok) {
+      const json = (await metaResp.json()) as { access_token?: string };
+      if (json.access_token) return json.access_token;
+    }
+  } catch {
+    // Not on GCE — fall through.
+  }
+
+  // Local-dev fallback: mint a token via the gcloud CLI if present. Keeps the
+  // Lyria backend dependency-free (no google-auth-library) while removing the
+  // need to refresh GOOGLE_ACCESS_TOKEN by hand. videoclaw already shells out
+  // to provider CLIs elsewhere (e.g. the veo-useapi Bun transport).
+  try {
+    const { stdout } = await execFileAsync('gcloud', ['auth', 'print-access-token'], {
+      timeout: 15_000,
+    });
+    const tok = stdout.trim();
+    if (tok) return tok;
+  } catch {
+    // gcloud absent / not authenticated — fall through to the clear error.
+  }
+
+  throw new VclawError(
+    'env_var_missing',
+    'No Vertex access token available. Set GOOGLE_ACCESS_TOKEN, run on a GCE/Cloud Run service account, or `gcloud auth login` (the CLI fallback).',
+    { backend: 'lyria' },
+  );
+}
+
+export const lyriaBackend: MusicBackend = {
+  id: 'lyria',
+  kind: 'music',
+  displayName: 'Lyria (Vertex AI)',
+  requiredEnv: [],
+  requiresVertex: true,
+  summary:
+    'Music generation via Google Vertex AI Lyria. Default lyria-002 (verified); LYRIA_MODEL can target lyria-3-clip-preview / lyria-3-pro-preview (preview, allowlist). Requires GOOGLE_CLOUD_PROJECT / VERTEX_PROJECT + a Vertex token.',
+
+  async generate(input: MusicGenInput): Promise<MusicGenResult> {
+    const env = input.env ?? process.env;
+    const fetcher = input.fetcher ?? fetch;
+    const durationSec = input.durationSec ?? 30;
+
+    if (!input.prompt.trim()) {
+      throw new VclawError('music_gen_failed', 'Cannot generate music with an empty prompt.', {
+        backend: 'lyria',
+      });
+    }
+
+    const project = resolveVertexProject(env);
+    if (!project) {
+      throw new VclawError(
+        'env_var_missing',
+        'GOOGLE_CLOUD_PROJECT (or VERTEX_PROJECT) is not set. The Lyria backend requires a Vertex project.',
+        { backend: 'lyria' },
+      );
+    }
+
+    const region = env['VERTEX_REGION'] ?? DEFAULT_LYRIA_REGION;
+    const model = env['LYRIA_MODEL'] ?? DEFAULT_LYRIA_MODEL;
+
+    if (input.dryRun) {
+      // Write an empty placeholder so downstream existsSync() stays consistent
+      // with every other audio backend's dry-run path (lyria3/elevenlabs/etc.).
+      const dir = dirname(input.outputPath);
+      if (dir) await mkdir(dir, { recursive: true });
+      await writeFile(input.outputPath, Buffer.alloc(0));
+      return { path: input.outputPath, durationMs: durationSec * 1000, backendId: 'lyria' };
+    }
+
+    const token = await resolveVertexToken(env, fetcher);
+
+    const endpoint =
+      `https://${region}-aiplatform.googleapis.com/v1/projects/${project}` +
+      `/locations/${region}/publishers/google/models/${model}:predict`;
+
+    // Verified against lyria-002 (Lyria 2). Lyria 3 adds optional image input.
+    const body = {
+      instances: [{ prompt: input.prompt }],
+      parameters: { sampleCount: 1 },
+    };
+
+    const resp = await fetcher(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (!resp.ok) {
+      let detail = '';
+      try {
+        detail = await resp.text();
+      } catch {
+        /* ignore */
+      }
+      throw new VclawError(
+        'music_gen_failed',
+        `Vertex Lyria predict request failed: ${resp.status} ${resp.statusText}`,
+        { status: resp.status, detail: detail.slice(0, 300), backend: 'lyria' },
+      );
+    }
+
+    // Verified response shape (lyria-002): predictions[0].bytesBase64Encoded (WAV).
+    const json = (await resp.json()) as {
+      predictions?: Array<{ bytesBase64Encoded?: string; mimeType?: string }>;
+    };
+
+    const prediction = json.predictions?.[0];
+    const b64 = prediction?.bytesBase64Encoded;
+    if (!b64) {
+      throw new VclawError(
+        'music_gen_failed',
+        'Vertex Lyria response did not include bytesBase64Encoded audio data.',
+        { backend: 'lyria' },
+      );
+    }
+
+    const bytes = Buffer.from(b64, 'base64');
+    const dir = dirname(input.outputPath);
+    if (dir) await mkdir(dir, { recursive: true });
+    await writeFile(input.outputPath, bytes);
+
+    // Report the WAV's true length when parseable (Lyria clips are ~32.8s),
+    // falling back to the requested duration only if the header is unreadable.
+    const durationMs = wavDurationMs(bytes) ?? durationSec * 1000;
+    return { path: input.outputPath, durationMs, backendId: 'lyria' };
+  },
+};