videoclaw 3.0.0-alpha.1

package/src/video/music-video.ts

@@ -0,0 +1,324 @@
+/**
+ * music-video.ts — the vocal-synced, beat-exact music-video ASSEMBLER (the crown
+ * jewel of the music-video lane). It turns a config (song + performer clips +
+ * B-roll pools) into a finished cut whose performers land on their own vocals and
+ * whose lips stay locked to the muxed song, with zero cumulative drift.
+ *
+ * This is a FULLY LOCAL assembler — no provider, no spend, only ffmpeg (already a
+ * dependency). It composes the lane's pure pieces:
+ *   buildVocalMap (transcript → rap/hook/instrumental/outro)  [or an explicit map]
+ *     → planVocalSync (performer-on-vocal, time-aligned, de-patterned B-roll)
+ *       → buildSegmentCutArgs (frame-exact `-frames:v` cut per segment)
+ *         → concat demuxer (-c copy) → ONE grade pass + mux the song.
+ *
+ * {@link buildMusicVideoPlan} is PURE (clip durations are injected via the config)
+ * so the whole plan — every segment, every ffmpeg invocation — is unit-testable
+ * offline. {@link runMusicVideo} is the thin execution shell: it writes the concat
+ * list, spawns each step, then asserts the built master matches the planned
+ * duration within one frame ({@link assertNoDrift}) so sync can never silently
+ * regress. Dry-run returns the plan without spawning.
+ */
+import { promises as fs } from 'node:fs';
+import * as path from 'node:path';
+import {
+  buildVocalMap,
+  validateVocalMap,
+  type WhisperSegment,
+} from './vocal-map.js';
+import {
+  planVocalSync,
+  plannedDuration,
+  type BrollPools,
+  type PlannedSegment,
+  type VocalSection,
+  type VocalSectionType,
+} from './vocal-sync-plan.js';
+import {
+  buildSegmentCutArgs,
+  assertNoDrift,
+  CUT_DEFAULT_WIDTH,
+  CUT_DEFAULT_HEIGHT,
+} from './assemble/cut-segment.js';
+import { TARGET_FPS } from './assemble/animate-slides.js';
+import { resolveGradeFilter, buildConcatListContent } from './assemble/stitch.js';
+import { runFfmpeg, ffprobeDuration } from './assemble/ffmpeg.js';
+
+/** A clip the assembler can cut from: an id plus its file and probed length. */
+export interface ClipEntry {
+  id: string;
+  path: string;
+  /** Source clip duration in seconds (so the planner stays pure). */
+  durationSec: number;
+}
+
+/** The music-video assembly config (an on-disk artifact or built in memory). */
+export interface MusicVideoConfig {
+  /** Path to the song audio that is muxed as the master's full audio track. */
+  song: string;
+  /** Song length in seconds (the cut covers [0, songEnd]). */
+  songEnd: number;
+  /** Output frame rate (default 24). */
+  fps?: number;
+  /** Output width (default 1280). */
+  width?: number;
+  /** Output height (default 720). */
+  height?: number;
+  /** Grade id resolved via {@link resolveGradeFilter} (one pass over the whole cut). */
+  grade?: string;
+  /**
+   * Explicit vocal map (already carrying `performerClip`s). Wins over `transcript`.
+   * Use this for a hand-authored map.
+   */
+  vocalMap?: VocalSection[];
+  /** Whisper segments to auto-classify into a vocal map (when `vocalMap` is absent). */
+  transcript?: WhisperSegment[];
+  /** Override the auto hook detection (passed to {@link buildVocalMap}). */
+  hookKeywords?: string[];
+  /**
+   * Performer clip id per vocal type, attached to the AUTO-built map
+   * (rap → rapper clip, hook → singer clip). Ignored when `vocalMap` is explicit.
+   */
+  performers?: { rap?: string; hook?: string };
+  /** B-roll clip ids grouped by role. */
+  pools: BrollPools;
+  /** Clip registry: every id referenced by `pools`/`performers`/`vocalMap`. */
+  clips: ClipEntry[];
+  /** Beat timestamps (seconds); empty = cut on the section cadence only. */
+  beats?: number[];
+  /** Seconds-per-cut overrides by section type. */
+  cutLengths?: Partial<Record<VocalSectionType, number>>;
+  /** Output master path (default `<workDir>/master.mp4`). */
+  output?: string;
+}
+
+/** One ordered ffmpeg invocation in the assembly plan (args only, no spawn). */
+export interface MusicVideoStep {
+  kind: 'segment-cut' | 'concat-demuxer' | 'grade-mux';
+  /** Everything AFTER `ffmpeg -y` (so it matches {@link runFfmpeg}). */
+  args: string[];
+  /** Human label for logs/plan output. */
+  label: string;
+  /** For the concat step: the concat-list file content to write before spawning. */
+  concatListContent?: string;
+  /** For the concat step: where that list file is written. */
+  concatListPath?: string;
+}
+
+export interface MusicVideoPlan {
+  schemaVersion: 1;
+  vocalMap: VocalSection[];
+  segments: PlannedSegment[];
+  plannedDurationSec: number;
+  width: number;
+  height: number;
+  fps: number;
+  grade: string;
+  steps: MusicVideoStep[];
+  segmentDir: string;
+  concatVideo: string;
+  output: string;
+}
+
+/** Attach the per-type performer clip to every vocal section of an auto-built map. */
+function attachPerformers(
+  map: VocalSection[],
+  performers: { rap?: string; hook?: string } | undefined,
+): VocalSection[] {
+  if (!performers) return map;
+  return map.map((s) => {
+    const clip = s.type === 'rap' ? performers.rap : s.type === 'hook' ? performers.hook : undefined;
+    return clip ? { ...s, performerClip: clip } : s;
+  });
+}
+
+/**
+ * Build the full, frame-exact assembly plan. PURE — no fs, no spawn. `workDir` is
+ * only used to compute step output paths; nothing is written here. Throws if a
+ * referenced clip id is missing from the registry or the vocal map is invalid.
+ */
+export function buildMusicVideoPlan(config: MusicVideoConfig, workDir: string): MusicVideoPlan {
+  const fps = config.fps ?? TARGET_FPS;
+  const width = config.width ?? CUT_DEFAULT_WIDTH;
+  const height = config.height ?? CUT_DEFAULT_HEIGHT;
+  const grade = resolveGradeFilter(config.grade);
+
+  // 1. resolve the vocal map (explicit wins; else auto-classify + attach performers).
+  let vocalMap: VocalSection[];
+  if (config.vocalMap && config.vocalMap.length > 0) {
+    vocalMap = config.vocalMap;
+  } else {
+    if (!config.transcript) {
+      throw new Error('buildMusicVideoPlan: config needs either vocalMap or transcript.');
+    }
+    const auto = buildVocalMap(config.transcript, {
+      songEnd: config.songEnd,
+      ...(config.hookKeywords ? { hookKeywords: config.hookKeywords } : {}),
+    });
+    vocalMap = attachPerformers(auto, config.performers);
+  }
+  const issues = validateVocalMap(vocalMap, config.songEnd);
+  if (issues.length > 0) {
+    throw new Error(`buildMusicVideoPlan: invalid vocal map: ${issues.join('; ')}`);
+  }
+
+  // 2. clip registry → duration lookup + path lookup (pure planner inputs).
+  const byId = new Map(config.clips.map((c) => [c.id, c]));
+  const clipDuration = (id: string): number => {
+    const c = byId.get(id);
+    if (!c) throw new Error(`buildMusicVideoPlan: clip "${id}" referenced but not in the registry.`);
+    return c.durationSec;
+  };
+  const clipPath = (id: string): string => {
+    const c = byId.get(id);
+    if (!c) throw new Error(`buildMusicVideoPlan: clip "${id}" referenced but not in the registry.`);
+    return c.path;
+  };
+
+  // 3. plan the vocal-synced segment list.
+  const segments = planVocalSync({
+    vocalMap,
+    beats: config.beats ?? [],
+    songEnd: config.songEnd,
+    pools: config.pools,
+    clipDuration,
+    fps,
+    ...(config.cutLengths ? { cutLengths: config.cutLengths } : {}),
+  });
+  if (segments.length === 0) {
+    throw new Error('buildMusicVideoPlan: planner produced no segments (check pools/vocal map).');
+  }
+
+  // 4. compile the ffmpeg steps: frame-exact cut per segment → concat → grade + mux.
+  const segmentDir = path.join(workDir, 'segments');
+  const concatVideo = path.join(workDir, 'concat-silent.mp4');
+  const concatListPath = path.join(workDir, 'concat-list.txt');
+  const output = config.output ?? path.join(workDir, 'master.mp4');
+
+  const steps: MusicVideoStep[] = [];
+  const segmentPaths: string[] = [];
+  segments.forEach((seg, i) => {
+    const outputPath = path.join(segmentDir, `seg-${String(i).padStart(4, '0')}.mp4`);
+    segmentPaths.push(outputPath);
+    steps.push({
+      kind: 'segment-cut',
+      label: `cut ${i} ${seg.performer ? 'performer' : 'b-roll'} ${seg.clip} @${seg.inSec}s ×${seg.durationSec}s`,
+      args: buildSegmentCutArgs({
+        clipPath: clipPath(seg.clip),
+        inSeconds: seg.inSec,
+        durationSec: seg.durationSec,
+        outputPath,
+        width,
+        height,
+        fps,
+        keepAudio: false,
+      }),
+    });
+  });
+
+  steps.push({
+    kind: 'concat-demuxer',
+    label: `concat ${segmentPaths.length} segments (-c copy)`,
+    concatListContent: buildConcatListContent(segmentPaths),
+    concatListPath,
+    args: ['-nostdin', '-f', 'concat', '-safe', '0', '-i', concatListPath, '-c', 'copy', concatVideo],
+  });
+
+  // ONE grade pass + mux the song as the full audio track. Re-encode video only
+  // when a grade is applied; otherwise copy the concatenated video stream.
+  const gradeMuxArgs: string[] = ['-nostdin', '-i', concatVideo, '-i', config.song];
+  if (grade) {
+    gradeMuxArgs.push('-vf', grade, '-c:v', 'libx264', '-crf', '18', '-preset', 'medium', '-pix_fmt', 'yuv420p');
+  } else {
+    gradeMuxArgs.push('-c:v', 'copy');
+  }
+  gradeMuxArgs.push(
+    '-map', '0:v:0', '-map', '1:a:0',
+    '-c:a', 'aac', '-b:a', '192k',
+    '-shortest', output,
+  );
+  steps.push({
+    kind: 'grade-mux',
+    label: grade ? `grade (${config.grade}) + mux song` : 'mux song (no grade)',
+    args: gradeMuxArgs,
+  });
+
+  return {
+    schemaVersion: 1,
+    vocalMap,
+    segments,
+    plannedDurationSec: plannedDuration(segments),
+    width,
+    height,
+    fps,
+    grade,
+    steps,
+    segmentDir,
+    concatVideo,
+    output,
+  };
+}
+
+export interface RunMusicVideoOptions {
+  /** Plan only — do not write files or spawn ffmpeg. */
+  dryRun?: boolean;
+  /** Override the ffmpeg binary. */
+  ffmpegBin?: string;
+  /** Override the ffprobe binary (drift check). */
+  ffprobeBin?: string;
+  /** Skip the post-build drift assertion (e.g. when ffprobe is unavailable). */
+  skipDriftCheck?: boolean;
+}
+
+export interface RunMusicVideoResult {
+  output: string;
+  plannedDurationSec: number;
+  builtDurationSec: number;
+  segmentCount: number;
+  performerCuts: number;
+  brollCuts: number;
+  grade: string;
+  dryRun: boolean;
+}
+
+/**
+ * Execute the assembly plan: mkdir the work dirs, run every step in order, write
+ * the concat list before its step, then assert the built master matches the plan
+ * within one frame. On `dryRun`, returns the planned metrics without touching disk.
+ */
+export async function runMusicVideo(
+  plan: MusicVideoPlan,
+  opts: RunMusicVideoOptions = {},
+): Promise<RunMusicVideoResult> {
+  const performerCuts = plan.segments.filter((s) => s.performer).length;
+  const base: Omit<RunMusicVideoResult, 'builtDurationSec' | 'dryRun'> = {
+    output: plan.output,
+    plannedDurationSec: plan.plannedDurationSec,
+    segmentCount: plan.segments.length,
+    performerCuts,
+    brollCuts: plan.segments.length - performerCuts,
+    grade: plan.grade,
+  };
+
+  if (opts.dryRun) {
+    return { ...base, builtDurationSec: plan.plannedDurationSec, dryRun: true };
+  }
+
+  await fs.mkdir(plan.segmentDir, { recursive: true });
+  await fs.mkdir(path.dirname(plan.output), { recursive: true });
+
+  for (const step of plan.steps) {
+    if (step.concatListContent && step.concatListPath) {
+      await fs.writeFile(step.concatListPath, step.concatListContent, 'utf8');
+    }
+    await runFfmpeg(step.args, { ...(opts.ffmpegBin ? { ffmpegBin: opts.ffmpegBin } : {}) });
+  }
+
+  let builtDurationSec = plan.plannedDurationSec;
+  if (!opts.skipDriftCheck) {
+    const ms = await ffprobeDuration(plan.output, { ...(opts.ffprobeBin ? { ffprobeBin: opts.ffprobeBin } : {}) });
+    builtDurationSec = ms / 1000;
+    assertNoDrift(plan.plannedDurationSec, builtDurationSec, plan.fps);
+  }
+
+  return { ...base, builtDurationSec, dryRun: false };
+}

package/src/video/narrate.ts

@@ -0,0 +1,221 @@
+/**
+ * Narration (TTS) generation.
+ *
+ * `generateNarration` synthesizes a single narration clip for a project from a
+ * text script via a TTS backend (from the audio-platform registry), writes the
+ * audio to `projects/<slug>/artifacts/audio/narration.wav`, and persists a
+ * `narration.json` artifact describing it. When a `videoDurationMs` is supplied,
+ * the artifact also embeds a pure `planNarrationFit()` plan (atempo / loop-video
+ * timing) so a downstream assemble step can fit the narration to the video bed.
+ *
+ * Pure-ish: `env` and `fetcher` are threaded through to the backend so tests
+ * run fully offline with no real keys or network. Throws `tts_failed` when no
+ * requested/available TTS backend can run.
+ */
+
+import { mkdir, readFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { dirname, join, relative, resolve } from 'node:path';
+import { resolveProjectWorkspace } from './workspace.js';
+import { appendProjectEvent } from './events.js';
+import { writeTextFileAtomic } from './atomic-write.js';
+import { VclawError } from './errors.js';
+import {
+  getTtsBackend,
+  listAvailableTtsBackends,
+  isTtsBackendAvailable,
+} from './audio-platform/registry.js';
+import type { TtsBackend, TtsGenResult } from './audio-platform/types.js';
+import { planNarrationFit } from './assemble/narration-fit.js';
+import type { NarrationFitPlan } from './assemble/narration-fit.js';
+
+export const NARRATION_SCHEMA_VERSION = 1;
+
+export interface NarrationArtifact {
+  schemaVersion: typeof NARRATION_SCHEMA_VERSION;
+  projectSlug: string;
+  generatedAt: string;
+  backendId: string;
+  voice?: string;
+  text: string;
+  /** Project-relative path to the generated narration audio. */
+  path: string;
+  durationMs: number;
+  /**
+   * Backends that failed (in order) before `backendId` succeeded, when the
+   * automatic fallback chain was used. Omitted when the first backend succeeded
+   * or an explicit `--backend` was given (explicit selection never falls back).
+   */
+  fallbackFrom?: string[];
+  /** Present only when a videoDurationMs was supplied. */
+  fit?: NarrationFitPlan;
+}
+
+export interface GenerateNarrationOptions {
+  workspaceRoot: string;
+  slug: string;
+  text: string;
+  voice?: string;
+  /** Restrict to this backend id (defaults to the first AVAILABLE backend). */
+  backendId?: string;
+  /** When given, embed a planNarrationFit() plan in the artifact. */
+  videoDurationMs?: number;
+  dryRun?: boolean;
+  env?: NodeJS.ProcessEnv;
+  fetcher?: typeof fetch;
+}
+
+export function narrationArtifactPathFor(root: string, slug: string): string {
+  return join(resolveProjectWorkspace(slug, root).projectDir, 'artifacts', 'narration.json');
+}
+
+export function narrationAudioDirFor(root: string, slug: string): string {
+  return join(resolveProjectWorkspace(slug, root).projectDir, 'artifacts', 'audio');
+}
+
+export async function readNarrationArtifact(
+  root: string,
+  slug: string,
+): Promise<NarrationArtifact | null> {
+  const path = narrationArtifactPathFor(root, slug);
+  if (!existsSync(path)) return null;
+  return JSON.parse(await readFile(path, 'utf-8')) as NarrationArtifact;
+}
+
+async function writeNarrationArtifact(
+  root: string,
+  slug: string,
+  artifact: NarrationArtifact,
+): Promise<string> {
+  const path = narrationArtifactPathFor(root, slug);
+  await mkdir(dirname(path), { recursive: true });
+  await writeTextFileAtomic(path, `${JSON.stringify(artifact, null, 2)}\n`);
+  return path;
+}
+
+/**
+ * Generate a single narration clip + persist the `narration.json` artifact.
+ * Returns the written artifact. Throws `tts_failed` when no requested/available
+ * TTS backend can run.
+ */
+export async function generateNarration(
+  options: GenerateNarrationOptions,
+): Promise<NarrationArtifact> {
+  const { workspaceRoot, slug, text } = options;
+  const env = options.env ?? process.env;
+
+  if (!text.trim()) {
+    throw new VclawError('tts_failed', 'Cannot generate narration with empty text.', { slug });
+  }
+
+  const available = listAvailableTtsBackends(env);
+
+  // Resolve the candidate chain. An explicit `--backend` is honored strictly
+  // (looked up + gated on availability) and NEVER falls back. Without one, we
+  // try every available backend in registry order (gemini-tts first, then
+  // elevenlabs-tts) so narration still succeeds when the preferred backend
+  // fails at RUNTIME — e.g. gemini-tts 403s when the Gemini `generativelanguage`
+  // API isn't enabled on the key's project even though the key is present.
+  let candidates: TtsBackend[];
+  if (options.backendId) {
+    const named = getTtsBackend(options.backendId); // throws on unknown id
+    if (!isTtsBackendAvailable(named, env)) {
+      throw new VclawError(
+        'tts_failed',
+        `TTS backend '${named.id}' is not available. Set a Gemini API key (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY) or ELEVENLABS_API_KEY.`,
+        { slug, requested: named.id, available: available.map((b) => b.id) },
+      );
+    }
+    candidates = [named];
+  } else {
+    candidates = available;
+  }
+
+  if (candidates.length === 0) {
+    throw new VclawError(
+      'tts_failed',
+      'No TTS backend is available. Set a Gemini API key (GEMINI_API_KEYS / GOOGLE_API_KEYS / GOOGLE_API_KEY) or ELEVENLABS_API_KEY.',
+      { slug, requested: options.backendId ?? null, available: available.map((b) => b.id) },
+    );
+  }
+
+  const workspace = resolveProjectWorkspace(slug, workspaceRoot);
+  const projectRoot = resolve(workspace.projectDir);
+  const audioDir = narrationAudioDirFor(workspaceRoot, slug);
+  await mkdir(audioDir, { recursive: true });
+
+  // Try each candidate in order; the first that synthesizes wins. A failure is
+  // recorded (for the `fallbackFrom` trail) and the next candidate is tried.
+  let result: TtsGenResult | undefined;
+  const failedBackends: string[] = [];
+  const attemptErrors: Array<{ backendId: string; message: string }> = [];
+  for (const backend of candidates) {
+    // Extension follows the backend's actual output format (wav for gemini-tts,
+    // mp3 for elevenlabs-tts) so the file name matches its bytes.
+    const outputPath = join(audioDir, `narration.${backend.outputExtension ?? 'wav'}`);
+    try {
+      result = await backend.generate({
+        text,
+        ...(options.voice ? { voice: options.voice } : {}),
+        outputPath,
+        ...(options.dryRun !== undefined ? { dryRun: options.dryRun } : {}),
+        env,
+        ...(options.fetcher ? { fetcher: options.fetcher } : {}),
+      });
+      break;
+    } catch (err) {
+      failedBackends.push(backend.id);
+      attemptErrors.push({
+        backendId: backend.id,
+        message: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  if (!result) {
+    throw new VclawError(
+      'tts_failed',
+      `All TTS backend(s) failed: ${attemptErrors.map((e) => `${e.backendId} (${e.message})`).join('; ')}`,
+      { slug, attempts: attemptErrors, candidates: candidates.map((b) => b.id) },
+    );
+  }
+
+  const fallbackFrom = failedBackends.length > 0 ? failedBackends : undefined;
+  const rel = relative(projectRoot, resolve(result.path)).replaceAll('\\', '/');
+
+  let fit: NarrationFitPlan | undefined;
+  if (options.videoDurationMs !== undefined) {
+    fit = planNarrationFit({
+      voiceDurationMs: result.durationMs,
+      videoDurationMs: options.videoDurationMs,
+    });
+  }
+
+  const artifact: NarrationArtifact = {
+    schemaVersion: NARRATION_SCHEMA_VERSION,
+    projectSlug: slug,
+    generatedAt: new Date().toISOString(),
+    backendId: result.backendId,
+    ...(options.voice ? { voice: options.voice } : {}),
+    text,
+    path: rel,
+    durationMs: result.durationMs,
+    ...(fallbackFrom ? { fallbackFrom } : {}),
+    ...(fit ? { fit } : {}),
+  };
+  await writeNarrationArtifact(workspaceRoot, slug, artifact);
+
+  if (existsSync(workspace.eventsPath) || existsSync(workspace.eventsDir)) {
+    await appendProjectEvent(workspace, {
+      type: 'narration.generated',
+      payload: {
+        backendId: result.backendId,
+        durationMs: result.durationMs,
+        dryRun: options.dryRun ?? false,
+        ...(fallbackFrom ? { fallbackFrom } : {}),
+      },
+    });
+  }
+
+  return artifact;
+}