videoclaw 3.0.0-alpha.1

package/src/video/assemble/text-card.ts

@@ -0,0 +1,285 @@
+/**
+ * text-card.ts — music-video title overlays that work on ANY ffmpeg and ANY
+ * script (Devanagari/Arabic included).
+ *
+ * Two halves:
+ *  1. A text→PNG renderer ({@link renderTextCard}) that rasterizes a stack of
+ *     lines to a transparent full-frame PNG via Pillow + RAQM (HarfBuzz shaping),
+ *     using MEASURE-INK-BOTTOM-AND-STACK so a tall Devanagari glyph's vowel marks
+ *     can never overlap the line beneath it. This sidesteps ffmpeg `drawtext`,
+ *     which (a) needs a libfreetype build and (b) can't shape complex scripts.
+ *  2. A PURE compositor ({@link buildTitleCompositeArgs}) that burns one or MORE
+ *     timed cards onto a base video.
+ *
+ * WHY a new compositor instead of {@link buildOverlayArgs}: that one takes the
+ * graphic as a plain `-i` input with no `-loop 1`. A static PNG is then a single
+ * frame at t=0, so a delayed `fade=in:st=N` (N>0) never reaches its window and
+ * the card stays invisible — the exact bug a hand-built run hit. Looping each
+ * image input fixes the fade, and chaining lets a lower-third AND an end card
+ * coexist. `buildTitleCompositeArgs` is the tested core (ffmpeg is never spawned
+ * here); `-y` is auto-prepended by runFfmpeg.
+ */
+import { spawn } from 'node:child_process';
+import { mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { TARGET_FPS } from './animate-slides.js';
+
+/** One timed full-frame title card (the text is pre-positioned inside the PNG). */
+export interface TitleCard {
+  /** Full-frame transparent PNG (same WxH as the base video). */
+  pngPath: string;
+  /** When the card becomes visible, seconds. */
+  startSec: number;
+  /** When it disappears, seconds. */
+  endSec: number;
+  /** Alpha fade-in duration (default 0.6). */
+  fadeInSec?: number;
+  /** Alpha fade-out duration (default 0.6). 0 = no fade-out (e.g. an end card that holds to EOF). */
+  fadeOutSec?: number;
+}
+
+export interface BuildTitleCompositeInput {
+  baseVideo: string;
+  cards: TitleCard[];
+  outputPath: string;
+  /** Image input framerate for the looped PNGs (default 24). */
+  fps?: number;
+  /** H.264 CRF (default 16 — preserve text edges). */
+  crf?: number;
+  preset?: string;
+}
+
+/**
+ * Build the FFmpeg args to composite N timed title cards onto `baseVideo`. PURE.
+ * Each PNG is a looped input (`-loop 1 -framerate fps`) so alpha fades on a
+ * delayed start actually animate; cards chain via successive `overlay=0:0`
+ * filters gated with `enable='between(t,start,end)'`. `-shortest` bounds the
+ * infinite looped images to the base video.
+ */
+export function buildTitleCompositeArgs(input: BuildTitleCompositeInput): string[] {
+  if (input.cards.length === 0) {
+    throw new Error('buildTitleCompositeArgs: at least one card is required.');
+  }
+  const fps = input.fps ?? TARGET_FPS;
+  const crf = input.crf ?? 16;
+  const preset = input.preset ?? 'medium';
+
+  const args: string[] = ['-i', input.baseVideo];
+  for (const card of input.cards) {
+    args.push('-loop', '1', '-framerate', String(fps), '-i', card.pngPath);
+  }
+
+  // Per-card alpha chain, then chained overlays onto the base.
+  const filters: string[] = [];
+  input.cards.forEach((card, i) => {
+    const inputIdx = i + 1; // input 0 is the base video
+    const fadeIn = card.fadeInSec ?? 0.6;
+    const fadeOut = card.fadeOutSec ?? 0.6;
+    const chain = ['format=rgba'];
+    if (fadeIn > 0) chain.push(`fade=t=in:st=${card.startSec}:d=${fadeIn}:alpha=1`);
+    if (fadeOut > 0) {
+      chain.push(`fade=t=out:st=${Math.max(0, card.endSec - fadeOut)}:d=${fadeOut}:alpha=1`);
+    }
+    filters.push(`[${inputIdx}:v]${chain.join(',')}[c${i}]`);
+  });
+  let prev = '[0:v]';
+  input.cards.forEach((card, i) => {
+    const out = i === input.cards.length - 1 ? '[v]' : `[v${i}]`;
+    filters.push(`${prev}[c${i}]overlay=0:0:enable='between(t,${card.startSec},${card.endSec})'${out}`);
+    prev = `[v${i}]`;
+  });
+
+  args.push(
+    '-filter_complex', filters.join(';'),
+    '-map', '[v]', '-map', '0:a?',
+    '-c:v', 'libx264', '-preset', preset, '-crf', String(crf), '-pix_fmt', 'yuv420p',
+    '-c:a', 'copy', '-shortest', '-movflags', '+faststart',
+    input.outputPath,
+  );
+  return args;
+}
+
+// ─── Text → PNG renderer (Pillow + RAQM) ──────────────────────────────────────
+
+/** One line in a card. `font` is an alias (see FONT_ALIASES) or an absolute font path. */
+export interface TextCardLine {
+  text: string;
+  font: string;
+  size: number;
+  /** Hex like "#f5f0e6". */
+  color: string;
+  /** Extra px between glyphs (Didot/caps look). Default 0. */
+  tracking?: number;
+}
+
+export interface TextCardSpec {
+  width: number;
+  height: number;
+  outputPath: string;
+  /** Where the stack sits. 'bottom-left' = lower-third; 'center' = end card. */
+  anchor: 'bottom-left' | 'center';
+  /** Explicit top Y for the text block. For a 'center' anchor this keeps the
+   *  horizontal centering but overrides the default vertical centering — used to
+   *  drop an end card into the lower third so it clears a centered subject. */
+  y?: number;
+  lines: TextCardLine[];
+  /** Base vertical gap added below each line's measured ink bottom. Default 28. */
+  gap?: number;
+  /** Soft drop shadow for legibility over footage. Default true. */
+  shadow?: boolean;
+  /** Optional ember accent rule to the left of a bottom-left stack. */
+  accentRule?: { color: string; width?: number };
+}
+
+/** Font aliases → macOS system font files (override by passing an absolute path as `font`). */
+export const FONT_ALIASES: Record<string, string> = {
+  didot: '/System/Library/Fonts/Supplemental/Didot.ttc',
+  devanagari: '/System/Library/Fonts/Kohinoor.ttc',
+  kohinoor: '/System/Library/Fonts/Kohinoor.ttc',
+  avenir: '/System/Library/Fonts/Avenir Next.ttc',
+  georgia: '/System/Library/Fonts/Supplemental/Georgia.ttf',
+  helvetica: '/System/Library/Fonts/HelveticaNeue.ttc',
+};
+
+/** The embedded renderer — productized render_overlays.py. Reads a spec JSON (argv[1]). */
+const TEXT_CARD_PY = String.raw`
+import sys, json
+from PIL import Image, ImageDraw, ImageFont, ImageFilter
+spec = json.load(open(sys.argv[1]))
+W, H = spec["width"], spec["height"]
+RAQM = ImageFont.Layout.RAQM
+def hx(c):
+    c = c.lstrip("#"); return tuple(int(c[i:i+2], 16) for i in (0, 2, 4))
+def font(path, size):
+    return ImageFont.truetype(path, size, layout_engine=RAQM)
+def tw(d, text, f, tr):
+    if tr == 0: return d.textbbox((0, 0), text, font=f)[2]
+    return sum(d.textbbox((0, 0), ch, font=f)[2] + tr for ch in text) - tr
+def put(d, xy, text, f, rgba, tr):
+    x, y = xy
+    if tr == 0:
+        d.text((x, y), text, font=f, fill=rgba); return
+    for ch in text:
+        d.text((x, y), ch, font=f, fill=rgba); x += d.textbbox((0,0), ch, font=f)[2] + tr
+img = Image.new("RGBA", (W, H), (0, 0, 0, 0))
+shadow_on = spec.get("shadow", True)
+gap = spec.get("gap", 28)
+anchor = spec["anchor"]
+# render each line into a temp layer, measure ink bottom, stack below previous
+layers = []
+_md = ImageDraw.Draw(img)
+for ln in spec["lines"]:
+    f = font(ln["font"], ln["size"]); tr = ln.get("tracking", 0)
+    w = tw(_md, ln["text"], f, tr)
+    layers.append((ln, f, tr, w))
+total_w = max((w for _, _, _, w in layers), default=0)
+# Fit-to-width: never let a line overflow the frame. If the widest line exceeds
+# the safe width (frame minus side margins), shrink EVERY line's font by the
+# same factor (preserving the size hierarchy + tracking) until it fits. Without
+# this a long end-card tagline rendered past the canvas edge and was cropped.
+safe_w = int(W * (1 - 2 * spec.get("safeMarginRatio", 0.07)))
+if total_w > safe_w and total_w > 0:
+    fit = safe_w / total_w
+    rescaled = []
+    for (ln, f, tr, w) in layers:
+        ns = max(8, int(ln["size"] * fit)); ntr = (tr * ns) // ln["size"] if ln["size"] else 0
+        nf = font(ln["font"], ns)
+        rescaled.append((ln, nf, ntr, tw(_md, ln["text"], nf, ntr)))
+    layers = rescaled
+    total_w = max((w for _, _, _, w in layers), default=0)
+# starting x
+if anchor == "bottom-left":
+    x0 = spec.get("x", 150); rule_w = (spec.get("accentRule") or {}).get("width", 7)
+    text_x = x0 + rule_w + 26
+    # bottom-anchored: estimate total height first via a dry stack
+top = spec.get("y")
+def stack(start_top):
+    yb = start_top; first_top = None; last_bottom = None
+    for (ln, f, tr, w) in layers:
+        layer = Image.new("RGBA", (W, H), (0, 0, 0, 0)); d = ImageDraw.Draw(layer)
+        x = (W - w) // 2 if anchor == "center" else text_x
+        y = yb + (0 if last_bottom is None else gap)
+        if shadow_on:
+            sh = Image.new("RGBA", (W, H), (0, 0, 0, 0)); sd = ImageDraw.Draw(sh)
+            put(sd, (x, y), ln["text"], f, (0, 0, 0, 170), tr)
+            img.alpha_composite(sh.filter(ImageFilter.GaussianBlur(7)))
+        put(d, (x, y), ln["text"], f, hx(ln["color"]) + (255,), tr)
+        img.alpha_composite(layer)
+        bb = layer.split()[3].getbbox()
+        if bb:
+            if first_top is None: first_top = bb[1]
+            last_bottom = bb[3]; yb = bb[3]
+    return first_top, last_bottom
+if anchor == "center" and top is None:
+    # two-pass: dry-run to center the block vertically (clear img between passes)
+    tmp = img.copy(); ft, lb = stack(0)
+    img.paste(tmp); img.alpha_composite  # noop guard
+    block_h = (lb - ft) if (ft is not None and lb is not None) else 0
+    img = Image.new("RGBA", (W, H), (0, 0, 0, 0))
+    stack(max(0, (H - block_h) // 2 - (ft or 0)))
+else:
+    if top is None: top = H - 360 if anchor == "bottom-left" else (H // 2 - 200)
+    ft, lb = stack(top)
+# ember accent rule for the lower-third
+ar = spec.get("accentRule")
+if ar and anchor == "bottom-left":
+    a = img.split()[3].getbbox()
+    if a:
+        d = ImageDraw.Draw(img); rw = ar.get("width", 7)
+        d.rectangle([spec.get("x",150), a[1], spec.get("x",150)+rw, a[3]], fill=hx(ar["color"]) + (255,))
+img.save(spec["outputPath"])
+print(spec["outputPath"])
+`;
+
+function resolveFonts(lines: TextCardLine[]): TextCardLine[] {
+  return lines.map((ln) => ({
+    ...ln,
+    font: ln.font.startsWith('/') ? ln.font : (FONT_ALIASES[ln.font.toLowerCase()] ?? ln.font),
+  }));
+}
+
+export interface RenderTextCardOptions {
+  /** python3 binary (default 'python3'). */
+  python?: string;
+  env?: NodeJS.ProcessEnv;
+}
+
+/** True iff python3 + Pillow + RAQM (Devanagari/complex-script shaping) are available. */
+export async function pillowAvailable(options: RenderTextCardOptions = {}): Promise<boolean> {
+  const python = options.python ?? 'python3';
+  return new Promise((resolve) => {
+    const child = spawn(python, ['-c', "from PIL import features; raise SystemExit(0 if features.check('raqm') else 1)"], {
+      stdio: 'ignore', env: options.env ?? process.env,
+    });
+    child.on('error', () => resolve(false));
+    child.on('exit', (code) => resolve(code === 0));
+  });
+}
+
+/**
+ * Render a {@link TextCardSpec} to a transparent PNG at `spec.outputPath` (Pillow
+ * + RAQM, measure-and-stack). Spawns python3. Call {@link pillowAvailable} first
+ * to fail clearly on a host without the dep. Returns the output path.
+ */
+export async function renderTextCard(spec: TextCardSpec, options: RenderTextCardOptions = {}): Promise<string> {
+  const python = options.python ?? 'python3';
+  const resolved: TextCardSpec = { ...spec, lines: resolveFonts(spec.lines) };
+  const dir = await mkdtemp(join(tmpdir(), 'vclaw-textcard-'));
+  const scriptPath = join(dir, 'render.py');
+  const specPath = join(dir, 'spec.json');
+  try {
+    await writeFile(scriptPath, TEXT_CARD_PY, 'utf8');
+    await writeFile(specPath, JSON.stringify(resolved), 'utf8');
+    await new Promise<void>((resolve, reject) => {
+      const child = spawn(python, [scriptPath, specPath], { stdio: ['ignore', 'ignore', 'pipe'], env: options.env ?? process.env });
+      let stderr = '';
+      child.stderr?.on('data', (d) => (stderr += String(d)));
+      child.on('error', reject);
+      child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`renderTextCard failed (exit ${code}): ${stderr.slice(0, 400)}`))));
+    });
+    return spec.outputPath;
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}

package/src/video/assemble/title-card.ts

@@ -0,0 +1,186 @@
+/**
+ * Title-card image generation for the assemble stage (sub-slice 3d).
+ *
+ * Source of truth: `skills/video-replicator/scripts/generate_title_card.py`
+ * (the `process_image` / text-overlay path). The Python pipeline composes a
+ * 16:9 card (1280x720), draws a semi-transparent dark band across the bottom,
+ * then renders an upper-cased title (58px, white, with a 2px outline) and an
+ * optional subtitle (28px, light blue #C8DCFF) centered in that band.
+ *
+ * We reproduce that purely in-process with `sharp`: build a base layer (a solid
+ * colour, or a center-cropped/resized background image), then composite an SVG
+ * holding the band + text on top. SVG-via-sharp is the most portable text path
+ * here — no native font installation step, generic-family fallbacks
+ * (sans-serif) resolve on every platform, and the geometry is declarative so it
+ * matches the Python layout 1:1.
+ *
+ * Pure module: no CLI wiring (that lands in sub-slice 3i), no network, no
+ * FFmpeg hold-video step (the Python Phase-2 `create_hold_video` is a separate
+ * concern handled by the stitch sub-slice).
+ */
+import sharp from 'sharp';
+import { mkdir, stat, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { VclawError } from '../errors.js';
+
+export interface GenerateTitleCardInput {
+  /** Main heading text. Rendered upper-cased to match the Python pipeline. */
+  title: string;
+  /** Optional secondary line below the title. */
+  subtitle?: string;
+  /** Where to write the composed PNG. */
+  outputPath: string;
+  /** Card width in pixels (default: 1280, matching LANDSCAPE_WIDTH). */
+  width?: number;
+  /** Card height in pixels (default: 720, matching LANDSCAPE_HEIGHT). */
+  height?: number;
+  /**
+   * Background: a hex colour (e.g. `#101018`) for a solid fill, or a path to an
+   * image to center-crop + resize to the card dimensions. Default: solid
+   * `#101018` (a dark slate consistent with the Python "dark" style band).
+   */
+  background?: string;
+  /** Title colour (default: white, matching TITLE_CARD_TITLE_COLOR). */
+  textColor?: string;
+  /** Title font size in px (default: 58, matching TITLE_CARD_TITLE_FONT_SIZE). */
+  fontSize?: number;
+  /** Skip composition + file write; return dimensions only. */
+  dryRun?: boolean;
+}
+
+export interface GenerateTitleCardResult {
+  /** Path written to (empty string on dry-run). */
+  path: string;
+  width: number;
+  height: number;
+}
+
+// Python config defaults (skills/.../config.py).
+const DEFAULT_WIDTH = 1280;
+const DEFAULT_HEIGHT = 720;
+const DEFAULT_TITLE_FONT_SIZE = 58;
+const SUBTITLE_FONT_SIZE = 28;
+const DEFAULT_TITLE_COLOR = 'white';
+const SUBTITLE_COLOR = '#C8DCFF';
+const TITLE_OUTLINE_COLOR = 'black';
+const TITLE_OUTLINE_WIDTH = 2;
+// Band spans from BAND_TOP to the bottom edge in the Python 720px layout; we
+// express it as a fraction so non-default heights keep the same proportion.
+const BAND_TOP_FRACTION = 560 / 720;
+const BAND_OPACITY = 160 / 255; // 0..1 alpha for the SVG fill.
+// Text baselines are offset from the band top (Python: +30 title, +95 subtitle,
+// measured against the 720px layout). Keep them proportional to height.
+const TITLE_OFFSET_FRACTION = 30 / 720;
+const SUBTITLE_OFFSET_FRACTION = 95 / 720;
+const FONT_FAMILY = "'Arial', 'Helvetica', sans-serif";
+
+const HEX_RE = /^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{4}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})$/;
+
+/** Minimal XML escaping for text injected into the SVG. */
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+}
+
+/**
+ * Compose a title-card PNG.
+ *
+ * Throws `VclawError('invalid_video_format', ...)` for non-positive dimensions
+ * and `VclawError('unexpected_internal_error', ...)` if sharp composition or
+ * the file write fails.
+ */
+export async function generateTitleCard(
+  input: GenerateTitleCardInput,
+): Promise<GenerateTitleCardResult> {
+  const width = input.width ?? DEFAULT_WIDTH;
+  const height = input.height ?? DEFAULT_HEIGHT;
+
+  if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
+    throw new VclawError(
+      'invalid_video_format',
+      `Title-card dimensions must be positive integers (got ${width}x${height}).`,
+      { width, height },
+    );
+  }
+
+  if (input.dryRun) {
+    return { path: '', width, height };
+  }
+
+  const titleColor = input.textColor ?? DEFAULT_TITLE_COLOR;
+  const titleFontSize = input.fontSize ?? DEFAULT_TITLE_FONT_SIZE;
+  const subtitleFontSize = Math.round(
+    SUBTITLE_FONT_SIZE * (titleFontSize / DEFAULT_TITLE_FONT_SIZE),
+  );
+
+  const bandTop = Math.round(height * BAND_TOP_FRACTION);
+  const bandHeight = height - bandTop;
+  const titleY = bandTop + Math.round(height * TITLE_OFFSET_FRACTION) + titleFontSize;
+  const subtitleY =
+    bandTop + Math.round(height * SUBTITLE_OFFSET_FRACTION) + subtitleFontSize;
+  const centerX = width / 2;
+
+  const titleText = escapeXml(input.title.toUpperCase());
+  const subtitleText = input.subtitle ? escapeXml(input.subtitle) : undefined;
+
+  // SVG overlay: band rectangle + outlined title + optional subtitle. Text is
+  // center-anchored so it lands centered like the Python textbbox math.
+  const titleNode = `<text x="${centerX}" y="${titleY}" text-anchor="middle" font-family="${FONT_FAMILY}" font-size="${titleFontSize}" font-weight="bold" fill="${titleColor}" stroke="${TITLE_OUTLINE_COLOR}" stroke-width="${TITLE_OUTLINE_WIDTH}" paint-order="stroke fill">${titleText}</text>`;
+  const subtitleNode = subtitleText
+    ? `<text x="${centerX}" y="${subtitleY}" text-anchor="middle" font-family="${FONT_FAMILY}" font-size="${subtitleFontSize}" fill="${SUBTITLE_COLOR}">${subtitleText}</text>`
+    : '';
+
+  const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">
+  <rect x="0" y="${bandTop}" width="${width}" height="${bandHeight}" fill="black" fill-opacity="${BAND_OPACITY}" />
+  ${titleNode}
+  ${subtitleNode}
+</svg>`;
+
+  let base: sharp.Sharp;
+  const background = input.background ?? '#101018';
+  if (HEX_RE.test(background)) {
+    base = sharp({
+      create: {
+        width,
+        height,
+        channels: 4,
+        background,
+      },
+    });
+  } else {
+    // Treat as an image path: center-crop + resize to the card (cover fit).
+    try {
+      await stat(background);
+    } catch {
+      throw new VclawError(
+        'image_not_found',
+        `Title-card background image not found: "${background}".`,
+        { background },
+      );
+    }
+    base = sharp(background).resize(width, height, { fit: 'cover', position: 'center' });
+  }
+
+  try {
+    const composed = await base
+      .composite([{ input: Buffer.from(svg), top: 0, left: 0 }])
+      .png()
+      .toBuffer();
+
+    await mkdir(dirname(input.outputPath), { recursive: true });
+    await writeFile(input.outputPath, composed);
+  } catch (err) {
+    if (err instanceof VclawError) throw err;
+    throw new VclawError(
+      'unexpected_internal_error',
+      `Failed to generate title card at "${input.outputPath}": ${(err as Error).message}`,
+      { outputPath: input.outputPath },
+    );
+  }
+
+  return { path: input.outputPath, width, height };
+}