videoclaw 3.0.0-alpha.1

package/src/video/assemble/audio-concat.ts

@@ -0,0 +1,165 @@
+/**
+ * Narration combined-track concat (resolves the `concatenate_audio_files`
+ * 3b deferral in `tts.ts`).
+ *
+ * Ported from `skills/video-replicator/scripts/generate_tts.py`
+ * (`generate_silence` + `concatenate_audio_files`): concatenate per-scene
+ * narration audio into a single track, optionally padding each scene's audio
+ * with trailing silence to match its scene's on-screen duration (so the
+ * combined narration stays aligned to the slide timeline).
+ *
+ * Repo pattern (sub-slice 3h): the PLAN — every ffmpeg arg array and concat
+ * list — is built by the pure {@link planNarrationConcat} and is the tested
+ * surface; {@link runNarrationConcat} is the thin real-spawn executor and is
+ * never run in tests (use `dryRun`).
+ *
+ * Encoding mirrors the Python: silence is anullsrc 44.1kHz stereo, per-scene
+ * pad-concat is stream-copy, and the final combined track is always
+ * libmp3lame 128k (an .mp3, regardless of the per-scene input format).
+ */
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { runFfmpeg, type RunFfmpegOptions } from './ffmpeg.js';
+import { VclawError } from '../errors.js';
+
+/** Escape a path for an ffmpeg concat-demuxer list line: file '<path>'. */
+export function concatListLine(path: string): string {
+  return `file '${resolve(path).replace(/'/g, `'\\''`)}'`;
+}
+
+export interface NarrationConcatInput {
+  /** Per-scene audio paths, in scene order. */
+  files: string[];
+  /** Combined output path (.mp3 — the track is always re-encoded libmp3lame). */
+  output: string;
+  /** Working directory for silence/padded intermediates + concat lists. */
+  tempDir: string;
+  /**
+   * Pad each scene's audio with trailing silence up to its target duration.
+   * Requires both duration arrays; scenes beyond `sceneDurationsSec`, scenes
+   * with unknown (<=0) actual duration, and scenes already at/over target are
+   * used as-is (the Python's rules).
+   */
+  padToDuration?: boolean;
+  /** Measured per-scene audio durations, seconds (parallel to `files`). */
+  actualDurationsSec?: number[];
+  /** Target per-scene on-screen durations, seconds (parallel to `files`). */
+  sceneDurationsSec?: number[];
+}
+
+export interface NarrationPadStep {
+  sceneIndex: number;
+  gapSeconds: number;
+  /** ffmpeg args generating the silence gap file. */
+  silencePath: string;
+  silenceArgs: string[];
+  /** Concat-demuxer list (content + path) joining original + silence. */
+  listPath: string;
+  listContent: string;
+  /** ffmpeg args stream-copy-concatenating original + silence. */
+  paddedPath: string;
+  concatArgs: string[];
+}
+
+export interface NarrationConcatPlan {
+  /** Per-scene padding work (empty when padding is off / not needed). */
+  padSteps: NarrationPadStep[];
+  /** The file list actually concatenated (originals with padded substitutions). */
+  effectiveFiles: string[];
+  finalListPath: string;
+  finalListContent: string;
+  finalArgs: string[];
+  output: string;
+}
+
+/**
+ * Build the full concat plan — pure (no I/O): all ffmpeg arg arrays and
+ * concat-list contents, with padded intermediates substituted into the final
+ * list. Throws only on invalid input (no files).
+ */
+export function planNarrationConcat(input: NarrationConcatInput): NarrationConcatPlan {
+  if (!Array.isArray(input.files) || input.files.length === 0) {
+    throw new VclawError(
+      'unexpected_internal_error',
+      'planNarrationConcat: input.files must be a non-empty array',
+    );
+  }
+
+  const padSteps: NarrationPadStep[] = [];
+  const effectiveFiles = [...input.files];
+
+  if (input.padToDuration && input.sceneDurationsSec && input.actualDurationsSec) {
+    for (let i = 0; i < input.files.length; i += 1) {
+      if (i >= input.sceneDurationsSec.length) continue;
+      const target = input.sceneDurationsSec[i];
+      const actual = input.actualDurationsSec[i] ?? 0;
+      if (!(actual > 0) || !(target > actual)) continue;
+
+      const gapSeconds = target - actual;
+      const silencePath = join(input.tempDir, `silence_${i}.mp3`);
+      const paddedPath = join(input.tempDir, `padded_${i}.mp3`);
+      const listPath = join(input.tempDir, `concat_${i}.txt`);
+      padSteps.push({
+        sceneIndex: i,
+        gapSeconds,
+        silencePath,
+        silenceArgs: [
+          '-f', 'lavfi',
+          '-i', 'anullsrc=r=44100:cl=stereo',
+          '-t', gapSeconds.toFixed(3),
+          '-c:a', 'libmp3lame',
+          '-b:a', '128k',
+          silencePath,
+        ],
+        listPath,
+        listContent: `${concatListLine(input.files[i])}\n${concatListLine(silencePath)}\n`,
+        paddedPath,
+        concatArgs: ['-f', 'concat', '-safe', '0', '-i', listPath, '-c', 'copy', paddedPath],
+      });
+      effectiveFiles[i] = paddedPath;
+    }
+  }
+
+  const finalListPath = `${input.output}.concat.txt`;
+  return {
+    padSteps,
+    effectiveFiles,
+    finalListPath,
+    finalListContent: effectiveFiles.map((f) => `${concatListLine(f)}\n`).join(''),
+    finalArgs: [
+      '-f', 'concat', '-safe', '0',
+      '-i', finalListPath,
+      '-c:a', 'libmp3lame',
+      '-b:a', '128k',
+      input.output,
+    ],
+    output: input.output,
+  };
+}
+
+/**
+ * Execute a {@link NarrationConcatPlan}: write the concat lists, run each pad
+ * step, then the final concat. Real-spawn path — unit tests must pass
+ * `dryRun: true` (lists are still written so the command is inspectable).
+ */
+export async function runNarrationConcat(
+  plan: NarrationConcatPlan,
+  opts: RunFfmpegOptions = {},
+): Promise<{ output: string; commands: string[] }> {
+  const commands: string[] = [];
+  await mkdir(dirname(plan.finalListPath), { recursive: true });
+
+  for (const step of plan.padSteps) {
+    await mkdir(dirname(step.listPath), { recursive: true });
+    const silence = await runFfmpeg(step.silenceArgs, opts);
+    commands.push(silence.command);
+    await writeFile(step.listPath, step.listContent, 'utf8');
+    const padded = await runFfmpeg(step.concatArgs, opts);
+    commands.push(padded.command);
+  }
+
+  await writeFile(plan.finalListPath, plan.finalListContent, 'utf8');
+  const final = await runFfmpeg(plan.finalArgs, opts);
+  commands.push(final.command);
+  return { output: plan.output, commands };
+}

package/src/video/assemble/audio-mix-plan.ts

@@ -0,0 +1,150 @@
+/**
+ * audio-mix-plan.ts — PURE multi-layer audio mix-plan helper.
+ *
+ * Composes project audio inputs (music, narration, dialogue, sfx) into the
+ * {@link AudioLayer}[] shape consumed by {@link buildMultiLayerMixArgs} in
+ * stitch.ts. No filesystem access, no network calls, no side-effects.
+ *
+ * stitch.ts's `AudioLayer.role` union natively includes `'sfx'`, so this module
+ * re-exports the base type directly — `'sfx'` layers produced here are mixed in
+ * (never used as a ducking sidechain) by `buildMultiLayerMixArgs`.
+ */
+
+import type { AudioLayer as BaseAudioLayer } from './stitch.js';
+
+/**
+ * Re-export of stitch.ts's {@link AudioLayer} (whose `role` union already
+ * includes `'sfx'`). Output of {@link buildAudioLayers} is therefore directly
+ * assignable to {@link buildMultiLayerMixArgs}'s `layers` parameter.
+ */
+export type AudioLayer = BaseAudioLayer;
+
+// ---------------------------------------------------------------------------
+// Public inputs / option shapes
+// ---------------------------------------------------------------------------
+
+/**
+ * All optional audio paths for a single project mix.
+ * Absent fields are simply skipped when building the layer list.
+ */
+export interface AudioMixInputs {
+  /** Path to background-music bed (looped at low volume). */
+  musicPath?: string;
+  /** Path to voice-over narration track. */
+  narrationPath?: string;
+  /** Paths to on-screen dialogue tracks (one entry per track). */
+  dialoguePaths?: string[];
+  /** Paths to sound-effect tracks. */
+  sfxPaths?: string[];
+}
+
+export interface AudioMixPlanOptions {
+  /**
+   * Volume for music layers (0..1).
+   * Passed through to {@link BuildMultiLayerMixOptions}.musicVolume.
+   * Default: 0.05 (matches stitch.ts DEFAULT_MUSIC_VOLUME).
+   */
+  musicVolume?: number;
+  /**
+   * Volume for sfx layers (0..1).
+   * Default: 0.6.
+   */
+  sfxVolume?: number;
+  /**
+   * When true, music will be sidechained under narration/dialogue.
+   * Forwarded to {@link BuildMultiLayerMixOptions}.duckMusicUnderVoice.
+   * Default: false.
+   */
+  duckMusicUnderVoice?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// buildAudioLayers
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert a flat {@link AudioMixInputs} map into the ordered {@link AudioLayer}[]
+ * that {@link buildMultiLayerMixArgs} consumes.
+ *
+ * Layer ordering: music → narration → dialogue (in order) → sfx (in order).
+ * Absent inputs produce no layers. Deterministic and side-effect-free.
+ */
+export function buildAudioLayers(
+  inputs: AudioMixInputs,
+  opts: AudioMixPlanOptions = {},
+): AudioLayer[] {
+  const { musicVolume = 0.05, sfxVolume = 0.6 } = opts;
+  const layers: AudioLayer[] = [];
+
+  if (inputs.musicPath) {
+    layers.push({
+      trackPath: inputs.musicPath,
+      role: 'music',
+      volume: musicVolume,
+      loop: true,
+    });
+  }
+
+  if (inputs.narrationPath) {
+    layers.push({
+      trackPath: inputs.narrationPath,
+      role: 'narration',
+    });
+  }
+
+  for (const dp of inputs.dialoguePaths ?? []) {
+    layers.push({
+      trackPath: dp,
+      role: 'dialogue',
+    });
+  }
+
+  for (const sp of inputs.sfxPaths ?? []) {
+    layers.push({
+      trackPath: sp,
+      role: 'sfx',
+      volume: sfxVolume,
+    });
+  }
+
+  return layers;
+}
+
+// ---------------------------------------------------------------------------
+// planMultiLayerMix
+// ---------------------------------------------------------------------------
+
+export interface MultiLayerMixPlan {
+  /** Ordered layers ready to pass to buildMultiLayerMixArgs. */
+  layers: AudioLayer[];
+  /**
+   * Whether music ducking is requested.
+   * Pass as BuildMultiLayerMixOptions.duckMusicUnderVoice.
+   */
+  duckMusicUnderVoice: boolean;
+}
+
+/**
+ * Convenience wrapper that returns both the layer list and the ducking flag
+ * so a future assemble-wiring is a single destructure.
+ *
+ * ```ts
+ * const { layers, duckMusicUnderVoice } = planMultiLayerMix(videoPath, inputs, opts);
+ * const args = buildMultiLayerMixArgs(videoPath, layers, outputPath, {
+ *   duckMusicUnderVoice,
+ *   musicVolume: opts?.musicVolume,
+ * });
+ * ```
+ *
+ * **Not wired into assemble.ts** — called only by future callers.
+ */
+export function planMultiLayerMix(
+  _videoPath: string,
+  inputs: AudioMixInputs,
+  opts: AudioMixPlanOptions = {},
+): MultiLayerMixPlan {
+  return {
+    layers: buildAudioLayers(inputs, opts),
+    duckMusicUnderVoice: opts.duckMusicUnderVoice ?? false,
+  };
+}

package/src/video/assemble/audio-utils.ts

@@ -0,0 +1,101 @@
+/**
+ * Pure audio helpers for the assemble stage (no I/O).
+ *
+ * Ported from the validation/derivation logic in
+ * `skills/video-replicator/scripts/generate_tts.py`. Everything here is
+ * deterministic + unit-testable; network and filesystem work lives in
+ * `tts.ts` / `tts-elevenlabs.ts`.
+ */
+import { VclawError } from '../errors.js';
+
+/**
+ * ElevenLabs `output_format` values we accept. Each maps to a container/
+ * codec + sample-rate + bitrate. The Python CLI takes a free string, but we
+ * validate against the documented ElevenLabs set so a typo surfaces as a
+ * clean `invalid_audio_format` error rather than an opaque API 4xx.
+ */
+export const VALID_OUTPUT_FORMATS = [
+  'mp3_22050_32',
+  'mp3_44100_32',
+  'mp3_44100_64',
+  'mp3_44100_96',
+  'mp3_44100_128',
+  'mp3_44100_192',
+  'pcm_8000',
+  'pcm_16000',
+  'pcm_22050',
+  'pcm_24000',
+  'pcm_44100',
+  'pcm_48000',
+  'ulaw_8000',
+  'alaw_8000',
+  'opus_48000_32',
+  'opus_48000_64',
+  'opus_48000_96',
+  'opus_48000_128',
+  'opus_48000_192',
+] as const;
+
+export type OutputFormat = typeof VALID_OUTPUT_FORMATS[number];
+
+export const DEFAULT_OUTPUT_FORMAT: OutputFormat = 'mp3_44100_128';
+
+/**
+ * Validate an ElevenLabs output-format string. Returns the normalized format
+ * on success; throws `VclawError('invalid_audio_format', ...)` otherwise.
+ */
+export function validateOutputFormat(format: string): OutputFormat {
+  if ((VALID_OUTPUT_FORMATS as readonly string[]).includes(format)) {
+    return format as OutputFormat;
+  }
+  throw new VclawError(
+    'invalid_audio_format',
+    `Unsupported audio output format: "${format}". Valid formats: ${VALID_OUTPUT_FORMATS.join(', ')}`,
+    { format, valid: VALID_OUTPUT_FORMATS },
+  );
+}
+
+/**
+ * The file extension that corresponds to a given output format
+ * (mp3 -> mp3, pcm/ulaw/alaw -> wav, opus -> opus).
+ */
+export function extensionForFormat(format: string): string {
+  if (format.startsWith('mp3_')) return 'mp3';
+  if (format.startsWith('opus_')) return 'opus';
+  // pcm / ulaw / alaw are raw; we wrap them as wav on disk.
+  return 'wav';
+}
+
+/**
+ * Derive a per-scene audio filename. Scene indexes are 1-based in the
+ * transcript / on disk to mirror the Python pipeline's `scene_NN` naming.
+ */
+export function sceneAudioFilename(sceneIndex: number, format: string): string {
+  const padded = String(sceneIndex).padStart(2, '0');
+  return `scene_${padded}.${extensionForFormat(format)}`;
+}
+
+/** Filename for the combined narration track. */
+export function combinedNarrationFilename(format: string): string {
+  return `narration.${extensionForFormat(format)}`;
+}
+
+/**
+ * Format a duration (milliseconds) as `MM:SS.mmm` for human-readable logs
+ * and report entries.
+ */
+export function formatDuration(durationMs: number): string {
+  if (!Number.isFinite(durationMs) || durationMs < 0) {
+    throw new VclawError('invalid_audio_format', `Invalid duration: ${durationMs}`, {
+      durationMs,
+    });
+  }
+  const totalSeconds = durationMs / 1000;
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = Math.floor(totalSeconds % 60);
+  const millis = Math.round(durationMs % 1000);
+  const mm = String(minutes).padStart(2, '0');
+  const ss = String(seconds).padStart(2, '0');
+  const mmm = String(millis).padStart(3, '0');
+  return `${mm}:${ss}.${mmm}`;
+}

package/src/video/assemble/cut-segment.ts

@@ -0,0 +1,121 @@
+/**
+ * cut-segment.ts — FRAME-EXACT extraction of one montage segment from a source
+ * clip. PURE arg-builder (no I/O, no spawn) so it is unit-testable like
+ * {@link buildAnimateArgs}; `runFfmpeg` prepends the `ffmpeg -y` and spawns.
+ *
+ * WHY THIS EXISTS — the load-bearing fix for audio-synced montages:
+ *  - Cutting each segment with `-t <seconds>` makes ffmpeg quantize to whole
+ *    frames and bias SHORT (~0.01–0.02s lost per cut). Over ~88 cuts that
+ *    accumulates LINEARLY to ~2s, so a beat-/vocal-synced edit drifts AHEAD of
+ *    its muxed audio by the end (lips/hits land early). The cure is to emit an
+ *    EXACT integer frame count via `-frames:v round(dur*fps)` (same trick as
+ *    {@link buildAnimateArgs}) so cumulative video-time == planned time.
+ *  - `-nostdin` so ffmpeg never consumes a caller's stdin (a `while read` loop
+ *    fed from a seg list otherwise loses bytes and corrupts the next path).
+ *  - Full-frame `scale=…:force_original_aspect_ratio=increase,crop=W:H` (never a
+ *    2.39 crop — that chops heads on close-ups) + `setsar=1` so heterogeneous
+ *    source clips (different fps/SAR) concat cleanly.
+ *  - `-ss` BEFORE `-i` (fast input seek; ffmpeg's default `accurate_seek` makes
+ *    the first output frame land on the exact requested frame).
+ *
+ * The companion {@link assertNoDrift} lets the assembler verify, after building,
+ * that the concatenated video length equals the planner total within one frame.
+ */
+import { TARGET_FPS } from './animate-slides.js';
+
+export const CUT_DEFAULT_WIDTH = 1280;
+export const CUT_DEFAULT_HEIGHT = 720;
+
+export interface BuildSegmentCutArgsInput {
+  /** Source clip to cut from. */
+  clipPath: string;
+  /** Input-seek point in seconds (the lip-sync scrub offset / B-roll in-point). */
+  inSeconds: number;
+  /** Desired segment duration; quantized to an exact whole number of frames. */
+  durationSec: number;
+  /** Where the cut segment is written. */
+  outputPath: string;
+  /** Output frame width (default 1280). */
+  width?: number;
+  /** Output frame height (default 720). */
+  height?: number;
+  /** Output frame rate (default 24, matching {@link TARGET_FPS}). */
+  fps?: number;
+  /** x264 CRF (default 18). */
+  crf?: number;
+  /** x264 preset (default 'veryfast' — segments are re-graded later in one pass). */
+  preset?: string;
+  /** Keep source audio? Default false (`-an`); montage audio is muxed at the end. */
+  keepAudio?: boolean;
+}
+
+/** Exact whole-frame count for a segment duration (>= 1). */
+export function segmentFrameCount(durationSec: number, fps: number = TARGET_FPS): number {
+  return Math.max(1, Math.round(durationSec * fps));
+}
+
+/**
+ * Build the FFmpeg args for one frame-exact segment cut. PURE — the returned
+ * array is everything AFTER `ffmpeg -y`. Outputs exactly
+ * `segmentFrameCount(durationSec, fps)` frames starting at `inSeconds`.
+ */
+export function buildSegmentCutArgs(input: BuildSegmentCutArgsInput): string[] {
+  const width = input.width ?? CUT_DEFAULT_WIDTH;
+  const height = input.height ?? CUT_DEFAULT_HEIGHT;
+  const fps = input.fps ?? TARGET_FPS;
+  const crf = input.crf ?? 18;
+  const preset = input.preset ?? 'veryfast';
+  const frames = segmentFrameCount(input.durationSec, fps);
+
+  const vf =
+    `scale=${width}:${height}:force_original_aspect_ratio=increase,` +
+    `crop=${width}:${height},fps=${fps},setsar=1`;
+
+  const args = [
+    '-nostdin',
+    '-ss',
+    input.inSeconds.toFixed(4),
+    '-i',
+    input.clipPath,
+    '-vf',
+    vf,
+    '-frames:v',
+    String(frames),
+  ];
+  if (!input.keepAudio) args.push('-an');
+  args.push(
+    '-c:v',
+    'libx264',
+    '-crf',
+    String(crf),
+    '-preset',
+    preset,
+    '-pix_fmt',
+    'yuv420p',
+    input.outputPath,
+  );
+  return args;
+}
+
+/**
+ * Assert the built (concatenated) video length matches the planned length to
+ * within one frame. Throws with a descriptive message otherwise. Use this as
+ * the post-build gate on any audio-synced assembly so cumulative drift can never
+ * silently regress.
+ */
+export function assertNoDrift(
+  plannedSeconds: number,
+  builtSeconds: number,
+  fps: number = TARGET_FPS,
+): void {
+  const frameSec = 1 / fps;
+  const driftSec = Math.abs(plannedSeconds - builtSeconds);
+  if (driftSec > frameSec + 1e-6) {
+    const driftFrames = (driftSec * fps).toFixed(2);
+    throw new Error(
+      `assembly drift: built ${builtSeconds.toFixed(3)}s vs planned ` +
+        `${plannedSeconds.toFixed(3)}s (${driftSec.toFixed(3)}s / ${driftFrames} frames > 1 frame). ` +
+        `Every segment must be cut frame-exact (-frames:v), not with -t.`,
+    );
+  }
+}