videoclaw 3.0.0-alpha.1

package/src/video/assemble/music.ts

@@ -0,0 +1,261 @@
+/**
+ * Background-music generation adapter (fetch-based).
+ *
+ * Ported from `skills/video-replicator/scripts/generate_music.py`:
+ *  - Provider:     Kie.ai (Suno AI v1 API)
+ *  - API base:     https://api.kie.ai
+ *  - auth header:  Authorization: Bearer <KIE_API_KEY>
+ *  - submit:       POST /api/v1/generate
+ *                    body { prompt, customMode, instrumental, model, callBackUrl }
+ *                    -> { code: 200, data: { taskId } }
+ *  - poll:         GET  /api/v1/generate/record-info?taskId=<id>
+ *                    -> { code: 200, data: { status, response: { sunoData: [...] } } }
+ *                    status === "SUCCESS"  -> tracks ready (track.audioUrl)
+ *                    status === "FAILED"   -> generation failed
+ *  - download:     GET <track.audioUrl> -> raw audio bytes (mp3)
+ *
+ * This is a poll-based provider: submit a job, poll record-info until the
+ * job reaches SUCCESS/FAILED (or we exhaust attempts), then download the
+ * first track's audio. The retry/prompt-simplification ladder from the
+ * Python source is intentionally NOT ported (out of scope for 3f); only the
+ * core submit -> poll -> download path is here.
+ *
+ * Pure module: no CLI wiring (3i). Mockable via `fetch`; the poll interval
+ * is injectable so tests don't wait.
+ */
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { VclawError } from '../errors.js';
+
+export const KIE_API_BASE = 'https://api.kie.ai';
+
+/** Matches the Python defaults (config.py + generate_music.py payload). */
+export const DEFAULT_MUSIC_MODEL = 'V5';
+export const DEFAULT_DURATION_SEC = 30;
+/** Max prompt length for non-custom mode (Python truncates to this). */
+export const MAX_PROMPT_CHARS = 500;
+/** config.py MUSIC_POLL_MAX_ATTEMPTS / MUSIC_POLL_INTERVAL (5 min at 5s). */
+export const DEFAULT_POLL_MAX_ATTEMPTS = 60;
+export const DEFAULT_POLL_INTERVAL_MS = 5000;
+
+export interface GenerateMusicInput {
+  /** Music description (BPM / genre / mood). Truncated to 500 chars. */
+  prompt: string;
+  /** Desired duration in seconds (default 30). Forwarded to manifest metadata. */
+  durationSec?: number;
+  /** Absolute path the downloaded audio is written to. */
+  outputPath: string;
+  /** Suno model (default "V5"). */
+  model?: string;
+  /** Instrumental (no vocals) — default true, matching the Python payload. */
+  instrumental?: boolean;
+  /** Read from KIE_API_KEY when omitted. */
+  apiKey?: string;
+  /** Injectable fetch for offline tests; defaults to the global `fetch`. */
+  fetcher?: typeof fetch;
+  /** Skip all network + file I/O; return a synthetic result for dry runs. */
+  dryRun?: boolean;
+  /** Max number of record-info polls before timing out (default 60). */
+  maxPollAttempts?: number;
+  /** Delay between polls in ms (default 5000; set 0 in tests). */
+  pollIntervalMs?: number;
+}
+
+export interface GenerateMusicResult {
+  /** Path the audio was written to (== input.outputPath). */
+  path: string;
+  /** Duration in milliseconds. Prefers the track's reported duration, else
+   *  the requested `durationSec`. */
+  durationMs: number;
+}
+
+/** One track entry from the Kie.ai `sunoData` array. */
+interface SunoTrack {
+  audioUrl?: string;
+  title?: string;
+  /** Seconds, as a number per Kie.ai responses. */
+  duration?: number;
+  tags?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Resolve the Kie.ai API key from the explicit input or KIE_API_KEY.
+ * Throws `env_var_missing` if absent (matches the Python env var).
+ */
+export function resolveMusicApiKey(explicit?: string): string {
+  const key = explicit ?? process.env.KIE_API_KEY;
+  if (!key) {
+    throw new VclawError(
+      'env_var_missing',
+      'KIE_API_KEY is not set. Export it (e.g. `export KIE_API_KEY=...`) before generating music. Sign up at https://kie.ai.',
+      { envVar: 'KIE_API_KEY' },
+    );
+  }
+  return key;
+}
+
+function sleep(ms: number): Promise<void> {
+  if (ms <= 0) return Promise.resolve();
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function safeReadText(resp: Response): Promise<string> {
+  try {
+    return await resp.text();
+  } catch {
+    return '';
+  }
+}
+
+/**
+ * Generate background music via Kie.ai Suno. Submits the job, polls
+ * record-info to completion, downloads the first track, and writes it to
+ * `outputPath`. Throws `music_gen_failed` on any non-2xx, API error code,
+ * FAILED status, or poll timeout.
+ */
+export async function generateMusic(input: GenerateMusicInput): Promise<GenerateMusicResult> {
+  const durationSec = input.durationSec ?? DEFAULT_DURATION_SEC;
+
+  if (!input.prompt.trim()) {
+    throw new VclawError('music_gen_failed', 'Cannot generate music with an empty prompt.', {});
+  }
+
+  if (input.dryRun) {
+    return { path: input.outputPath, durationMs: durationSec * 1000 };
+  }
+
+  const apiKey = resolveMusicApiKey(input.apiKey);
+  const model = input.model ?? DEFAULT_MUSIC_MODEL;
+  const instrumental = input.instrumental ?? true;
+  const maxPollAttempts = input.maxPollAttempts ?? DEFAULT_POLL_MAX_ATTEMPTS;
+  const pollIntervalMs = input.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+  const doFetch = input.fetcher ?? fetch;
+
+  const headers = {
+    Authorization: `Bearer ${apiKey}`,
+    'Content-Type': 'application/json',
+  };
+
+  // --- submit ---
+  const submitBody = {
+    prompt: input.prompt.slice(0, MAX_PROMPT_CHARS),
+    customMode: false,
+    instrumental,
+    model,
+    // Dummy callback: we poll instead of receiving a webhook (matches Python).
+    callBackUrl: 'https://httpbin.org/post',
+  };
+
+  const submitResp = await doFetch(`${KIE_API_BASE}/api/v1/generate`, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(submitBody),
+  });
+
+  if (!submitResp.ok) {
+    const detail = await safeReadText(submitResp);
+    throw new VclawError(
+      'music_gen_failed',
+      `Kie.ai generate request failed: ${submitResp.status} ${submitResp.statusText}`,
+      { status: submitResp.status, detail: detail.slice(0, 300) },
+    );
+  }
+
+  const submitJson = (await submitResp.json()) as {
+    code?: number;
+    msg?: string;
+    data?: { taskId?: string };
+  };
+
+  if (submitJson.code !== 200) {
+    throw new VclawError('music_gen_failed', `Kie.ai API error: ${submitJson.msg ?? 'Unknown error'}`, {
+      code: submitJson.code,
+    });
+  }
+
+  const taskId = submitJson.data?.taskId;
+  if (!taskId) {
+    throw new VclawError('music_gen_failed', 'Kie.ai response did not include a taskId.', {});
+  }
+
+  // --- poll ---
+  let track: SunoTrack | undefined;
+  for (let attempt = 0; attempt < maxPollAttempts; attempt++) {
+    await sleep(pollIntervalMs);
+
+    const statusUrl = new URL(`${KIE_API_BASE}/api/v1/generate/record-info`);
+    statusUrl.searchParams.set('taskId', taskId);
+
+    const statusResp = await doFetch(statusUrl.toString(), { method: 'GET', headers });
+    if (!statusResp.ok) {
+      // Transient: keep polling (matches Python `continue`).
+      continue;
+    }
+
+    const statusJson = (await statusResp.json()) as {
+      code?: number;
+      data?: {
+        status?: string;
+        errorMessage?: string;
+        response?: { sunoData?: SunoTrack[] };
+      };
+    };
+
+    if (statusJson.code !== 200) {
+      // Still warming up; poll again.
+      continue;
+    }
+
+    const data = statusJson.data ?? {};
+    const genStatus = data.status ?? '';
+
+    if (genStatus === 'SUCCESS') {
+      const tracks = data.response?.sunoData ?? [];
+      if (tracks.length > 0) {
+        track = tracks[0];
+        break;
+      }
+    }
+
+    if (genStatus === 'FAILED' || JSON.stringify(data).includes('GENERATE_AUDIO_FAILED')) {
+      throw new VclawError('music_gen_failed', `Music generation failed: ${data.errorMessage ?? 'GENERATE_AUDIO_FAILED'}`, {
+        taskId,
+        status: genStatus,
+      });
+    }
+    // otherwise: pending — loop again.
+  }
+
+  if (!track) {
+    throw new VclawError('music_gen_failed', `Music generation timed out after ${maxPollAttempts} polls.`, {
+      taskId,
+    });
+  }
+
+  const audioUrl = track.audioUrl;
+  if (!audioUrl) {
+    throw new VclawError('music_gen_failed', 'Completed track did not include an audioUrl.', { taskId });
+  }
+
+  // --- download ---
+  const audioResp = await doFetch(audioUrl, { method: 'GET' });
+  if (!audioResp.ok) {
+    const detail = await safeReadText(audioResp);
+    throw new VclawError(
+      'music_gen_failed',
+      `Failed to download music track: ${audioResp.status} ${audioResp.statusText}`,
+      { status: audioResp.status, detail: detail.slice(0, 300) },
+    );
+  }
+
+  const bytes = new Uint8Array(await audioResp.arrayBuffer());
+  const dir = dirname(input.outputPath);
+  if (dir) await mkdir(dir, { recursive: true });
+  await writeFile(input.outputPath, bytes);
+
+  // Prefer the track's reported duration (seconds); fall back to requested.
+  const trackDurationSec = typeof track.duration === 'number' ? track.duration : durationSec;
+
+  return { path: input.outputPath, durationMs: Math.round(trackDurationSec * 1000) };
+}

package/src/video/assemble/narration-fit.ts

@@ -0,0 +1,90 @@
+/**
+ * Pure narration/video fit planner.
+ *
+ * Adapted from the useful timing rule in Google's story-generator skill:
+ * speed voiceover only within a natural threshold, otherwise keep narration
+ * natural and loop/cap the visual bed to the narration length.
+ */
+
+export interface NarrationFitInput {
+  /** Original narration duration in milliseconds. */
+  voiceDurationMs: number;
+  /** Available/generated video duration in milliseconds. */
+  videoDurationMs: number;
+  /** Maximum acceptable voice speed-up. Defaults to 1.25x. */
+  maxTempo?: number;
+  /** Safety pad removed from the visual target before fitting. Defaults to 500ms. */
+  tailPadMs?: number;
+}
+
+export interface NarrationFitWarning {
+  code: 'voice-speedup-applied' | 'voice-too-long-loop-video';
+  message: string;
+}
+
+export interface NarrationFitPlan {
+  /** FFmpeg atempo value for narration. */
+  tempo: number;
+  /** Whether the visual bed must be looped/extended to preserve natural speech. */
+  loopVideo: boolean;
+  /** Exact mixed output duration to cap with ffmpeg -t. */
+  targetDurationMs: number;
+  warnings: NarrationFitWarning[];
+}
+
+function assertPositiveDuration(name: string, value: number): void {
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new Error(`${name} must be a positive finite duration in milliseconds`);
+  }
+}
+
+function roundTempo(value: number): number {
+  return Math.round(value * 100) / 100;
+}
+
+export function planNarrationFit(input: NarrationFitInput): NarrationFitPlan {
+  assertPositiveDuration('voiceDurationMs', input.voiceDurationMs);
+  assertPositiveDuration('videoDurationMs', input.videoDurationMs);
+  const maxTempo = input.maxTempo ?? 1.25;
+  const tailPadMs = input.tailPadMs ?? 500;
+  if (!Number.isFinite(maxTempo) || maxTempo < 1) {
+    throw new Error('maxTempo must be a finite number >= 1');
+  }
+  if (!Number.isFinite(tailPadMs) || tailPadMs < 0) {
+    throw new Error('tailPadMs must be a finite number >= 0');
+  }
+
+  if (input.voiceDurationMs <= input.videoDurationMs) {
+    return {
+      tempo: 1,
+      loopVideo: false,
+      targetDurationMs: input.voiceDurationMs,
+      warnings: [],
+    };
+  }
+
+  const fitTargetMs = Math.max(1000, input.videoDurationMs - tailPadMs);
+  const requiredTempo = input.voiceDurationMs / fitTargetMs;
+  if (requiredTempo <= maxTempo) {
+    const tempo = roundTempo(Math.max(1, requiredTempo));
+    return {
+      tempo,
+      loopVideo: false,
+      targetDurationMs: Math.round(input.voiceDurationMs / tempo),
+      warnings: [{
+        code: 'voice-speedup-applied',
+        message: `Narration speed-fit applied at ${tempo.toFixed(2)}x.`,
+      }],
+    };
+  }
+
+  return {
+    tempo: 1,
+    loopVideo: true,
+    targetDurationMs: input.voiceDurationMs,
+    warnings: [{
+      code: 'voice-too-long-loop-video',
+      message: `Required narration speed-up ${requiredTempo.toFixed(2)}x exceeds the ${maxTempo.toFixed(2)}x threshold; keep speech natural and loop the visual bed.`,
+    }],
+  };
+}

package/src/video/assemble/overlay.ts

@@ -0,0 +1,225 @@
+/**
+ * Motion-graphics overlays for the assemble layer (the After-Effects-shaped gap):
+ * composite a graphic onto footage, burn an animated alert, or a lower-third title
+ * — all time-gated with fades. These are PURE FFmpeg arg-builders (the tested
+ * surface; ffmpeg is never spawned here), mirroring stitch.ts. They pair with
+ * gen-image: generate a "SYSTEM COMPROMISED" / dashboard graphic, then overlay it.
+ *
+ * Conventions: `-y` is auto-prepended by runFfmpeg, so it is NOT included here.
+ * Time gating uses `enable='between(t,start,end)'`; fades use the `fade` filter on
+ * the overlay's alpha, so the composite eases in/out rather than popping.
+ *
+ * VALIDATION STATUS: {@link buildOverlayArgs} (graphic composite) is real-render
+ * validated — pair it with gen-image to overlay a generated "SYSTEM COMPROMISED" /
+ * dashboard graphic with no font dependency (the recommended path). The two text
+ * builders ({@link buildAlertTextArgs}, {@link buildLowerThirdArgs}) use the
+ * `drawtext` filter, which requires an ffmpeg built with libfreetype; their arg
+ * shapes are unit-tested but NOT render-validated on every host. On a drawtext-less
+ * ffmpeg, render the text to a PNG (gen-image / a title-card) and composite it via
+ * buildOverlayArgs instead.
+ */
+
+export type OverlayPosition =
+  | 'top-left'
+  | 'top-right'
+  | 'bottom-left'
+  | 'bottom-right'
+  | 'center'
+  | 'bottom-center'
+  | 'top-center'
+  | 'full';
+
+export const OVERLAY_POSITION_IDS: OverlayPosition[] = [
+  'top-left', 'top-right', 'bottom-left', 'bottom-right',
+  'center', 'bottom-center', 'top-center', 'full',
+];
+
+/** overlay x/y expressions for a position, given a pixel `margin`. `full` is 0,0
+ * (the caller scales the graphic to the frame). Uses overlay-filter vars
+ * (main_w/main_h = base, overlay_w/overlay_h = the graphic). */
+function overlayXY(position: OverlayPosition, margin: number): { x: string; y: string } {
+  const m = String(margin);
+  switch (position) {
+    case 'top-left': return { x: m, y: m };
+    case 'top-right': return { x: `main_w-overlay_w-${m}`, y: m };
+    case 'bottom-left': return { x: m, y: `main_h-overlay_h-${m}` };
+    case 'bottom-right': return { x: `main_w-overlay_w-${m}`, y: `main_h-overlay_h-${m}` };
+    case 'center': return { x: '(main_w-overlay_w)/2', y: '(main_h-overlay_h)/2' };
+    case 'bottom-center': return { x: '(main_w-overlay_w)/2', y: `main_h-overlay_h-${m}` };
+    case 'top-center': return { x: '(main_w-overlay_w)/2', y: m };
+    case 'full': return { x: '0', y: '0' };
+  }
+}
+
+export interface BuildOverlayOptions {
+  position?: OverlayPosition;
+  /** Visible window in seconds. Defaults to the whole clip (no `enable` gate). */
+  startSec?: number;
+  endSec?: number;
+  fadeInSec?: number;
+  fadeOutSec?: number;
+  /** 0..1 overall opacity. Default 1. */
+  opacity?: number;
+  /** Inset from the frame edge in px (ignored for `center`/`full`). Default 48. */
+  margin?: number;
+  /** H.264 CRF. Default 18 (overlays want to preserve text edges). */
+  crf?: number;
+}
+
+/**
+ * Composite a graphic (PNG/MP4-with-alpha) onto a base video (PURE). Fades the
+ * overlay's alpha in/out and gates it to [startSec, endSec] when given. `full`
+ * scales the graphic to the frame; other positions overlay at native size.
+ */
+export function buildOverlayArgs(
+  baseVideo: string,
+  graphic: string,
+  outputPath: string,
+  opts: BuildOverlayOptions = {},
+): string[] {
+  const position = opts.position ?? 'bottom-center';
+  const margin = opts.margin ?? 48;
+  const { x, y } = overlayXY(position, margin);
+  const start = opts.startSec;
+  const end = opts.endSec;
+
+  const ovChain: string[] = ['format=rgba'];
+  if (typeof opts.opacity === 'number' && opts.opacity < 1) {
+    ovChain.push(`colorchannelmixer=aa=${opts.opacity}`);
+  }
+  if (opts.fadeInSec && opts.fadeInSec > 0) {
+    ovChain.push(`fade=t=in:st=${start ?? 0}:d=${opts.fadeInSec}:alpha=1`);
+  }
+  if (opts.fadeOutSec && opts.fadeOutSec > 0 && typeof end === 'number') {
+    ovChain.push(`fade=t=out:st=${Math.max(0, end - opts.fadeOutSec)}:d=${opts.fadeOutSec}:alpha=1`);
+  }
+
+  const enable = typeof start === 'number' && typeof end === 'number'
+    ? `:enable='between(t,${start},${end})'`
+    : '';
+  // For `full`, scale the graphic to the base frame with scale2ref (a runtime
+  // reference to the base video's real WxH) — NOT `scale=main_w:main_h`, whose
+  // main_w/main_h are overlay-filter-only vars that evaluate to 0 inside `scale`
+  // (→ scale=0:0, which ffmpeg rejects with "Output size 0x0 is not allowed").
+  // Other positions overlay the graphic at its native size.
+  const filter =
+    position === 'full'
+      ? `[1:v]${ovChain.join(',')}[ovpre];` +
+        `[ovpre][0:v]scale2ref=w=main_w:h=main_h[ov][base];` +
+        `[base][ov]overlay=0:0${enable}[v]`
+      : `[1:v]${ovChain.join(',')}[ov];` +
+        `[0:v][ov]overlay=${x}:${y}${enable}[v]`;
+
+  return [
+    '-i', baseVideo,
+    '-i', graphic,
+    '-filter_complex', filter,
+    '-map', '[v]', '-map', '0:a?',
+    '-c:v', 'libx264', '-preset', 'fast', '-crf', String(opts.crf ?? 18),
+    '-c:a', 'copy', '-movflags', '+faststart',
+    outputPath,
+  ];
+}
+
+export interface BuildAlertTextOptions {
+  startSec?: number;
+  endSec?: number;
+  /** Pulse frequency in Hz (the flash rate). Default 2. */
+  pulseHz?: number;
+  /** Text color. Default red. */
+  color?: string;
+  fontSize?: number;
+  /** Draw a flashing border box. Default true. */
+  border?: boolean;
+  crf?: number;
+}
+
+/**
+ * Burn an animated alert (e.g. "SYSTEM COMPROMISED") onto a base video WITHOUT a
+ * pre-rendered graphic (PURE). The text alpha pulses via a sine expression and an
+ * optional border box flashes in sync, gated to [startSec, endSec]. This is the
+ * templated motion-graphic the reference advert uses for its breach moment.
+ */
+export function buildAlertTextArgs(
+  baseVideo: string,
+  text: string,
+  outputPath: string,
+  opts: BuildAlertTextOptions = {},
+): string[] {
+  const start = opts.startSec ?? 0;
+  const end = opts.endSec;
+  const hz = opts.pulseHz ?? 2;
+  const color = opts.color ?? 'red';
+  const fontSize = opts.fontSize ?? 72;
+  const gate = typeof end === 'number' ? `between(t,${start},${end})` : `gte(t,${start})`;
+  // Pulsing alpha: 0 outside the window, 0.55..1.0 sine inside.
+  const alpha = `if(${gate}\\,0.55+0.45*sin(2*PI*${hz}*t)\\,0)`;
+  const safeText = text.replace(/:/g, '\\:').replace(/'/g, "\\'");
+
+  const filters: string[] = [];
+  if (opts.border !== false) {
+    // A thick frame border that shares the pulsing alpha.
+    filters.push(
+      `drawbox=x=24:y=24:w=iw-48:h=ih-48:color=${color}:t=8:enable='${gate}'`,
+    );
+  }
+  filters.push(
+    `drawtext=text='${safeText}':fontcolor=${color}:fontsize=${fontSize}:` +
+      `x=(w-text_w)/2:y=(h-text_h)/2:alpha='${alpha}'`,
+  );
+
+  return [
+    '-i', baseVideo,
+    '-vf', filters.join(','),
+    '-c:v', 'libx264', '-preset', 'fast', '-crf', String(opts.crf ?? 18),
+    '-c:a', 'copy', '-movflags', '+faststart',
+    outputPath,
+  ];
+}
+
+export interface BuildLowerThirdOptions {
+  startSec?: number;
+  endSec?: number;
+  fadeInSec?: number;
+  fadeOutSec?: number;
+  fontSize?: number;
+  color?: string;
+  /** Background box color@opacity, e.g. `black@0.55`. Default `black@0.55`. */
+  boxColor?: string;
+  /** Inset from the bottom-left in px. Default 64. */
+  margin?: number;
+  crf?: number;
+}
+
+/**
+ * Burn a lower-third title (text on a translucent box, bottom-left) onto a base
+ * video (PURE), gated to [startSec, endSec] with optional fades. The standard
+ * name/role caption the reference uses to introduce characters.
+ */
+export function buildLowerThirdArgs(
+  baseVideo: string,
+  text: string,
+  outputPath: string,
+  opts: BuildLowerThirdOptions = {},
+): string[] {
+  const start = opts.startSec;
+  const end = opts.endSec;
+  const margin = opts.margin ?? 64;
+  const fontSize = opts.fontSize ?? 40;
+  const safeText = text.replace(/:/g, '\\:').replace(/'/g, "\\'");
+  const enable = typeof start === 'number' && typeof end === 'number'
+    ? `:enable='between(t,${start},${end})'`
+    : '';
+
+  const drawtext =
+    `drawtext=text='${safeText}':fontcolor=${opts.color ?? 'white'}:fontsize=${fontSize}:` +
+    `x=${margin}:y=h-${margin}-text_h:box=1:boxcolor=${opts.boxColor ?? 'black@0.55'}:boxborderw=18${enable}`;
+
+  return [
+    '-i', baseVideo,
+    '-vf', drawtext,
+    '-c:v', 'libx264', '-preset', 'fast', '-crf', String(opts.crf ?? 18),
+    '-c:a', 'copy', '-movflags', '+faststart',
+    outputPath,
+  ];
+}