privateer-agent 0.12.13 → 0.12.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,13 +15,22 @@
15
15
  // something finished. ffmpeg is spawned WITHOUT a shell and every argument is built
16
16
  // from validated parameters, so nothing a prompt injects can become a command.
17
17
  //
18
+ // FILTERGRAPHS ARE THE SECOND INJECTION SURFACE, and a shell-free spawn does nothing
19
+ // about it: `-filter_complex` takes one string that ffmpeg itself parses, where `:`,
20
+ // `,`, `;`, `'`, `[` and `\` are all syntax. So no caller-supplied text is ever
21
+ // interpolated into a graph raw. Numbers go through `bounded()`, colours through a
22
+ // closed pattern, positions through a fixed list of nine names (never an x/y
23
+ // expression), paths through `filterPath()`, and prose — a caption — is written to a
24
+ // temp file that drawtext READS, so its content is never parsed as a filter at all.
25
+ //
18
26
  // ffmpeg is not a dependency we ship. When it's missing every operation says so once,
19
27
  // clearly, with the install line for the platform — rather than surfacing ENOENT.
20
28
 
21
29
  import { Type } from "typebox";
22
30
  import { spawn } from "node:child_process";
23
- import { existsSync, mkdirSync, statSync } from "node:fs";
24
- import { dirname, isAbsolute, resolve } from "node:path";
31
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
32
+ import { tmpdir } from "node:os";
33
+ import { dirname, isAbsolute, join, resolve } from "node:path";
25
34
 
26
35
  /** Tool names this module registers, for allow-list construction. */
27
36
  export const COMPOSE_TOOL_NAMES = ["video_compose"] as const;
@@ -32,7 +41,19 @@ const FFMPEG_TIMEOUT_MS = Number(process.env.PRIVATEER_FFMPEG_TIMEOUT_MS) || 10
32
41
  // would eat the model's context.
33
42
  const STDERR_TAIL_CHARS = 1200;
34
43
 
35
- const OPERATIONS = ["probe", "concat", "slideshow", "mux_audio", "trim", "extract_frame", "gif"] as const;
44
+ const OPERATIONS = [
45
+ "probe",
46
+ "concat",
47
+ "slideshow",
48
+ "mux_audio",
49
+ "mix_audio",
50
+ "overlay_text",
51
+ "overlay_image",
52
+ "burn_subtitles",
53
+ "trim",
54
+ "extract_frame",
55
+ "gif",
56
+ ] as const;
36
57
  type Operation = (typeof OPERATIONS)[number];
37
58
 
38
59
  function text(t: string) {
@@ -218,12 +239,87 @@ function normalizeVideo(index: number, label: string, w: number, h: number, fps:
218
239
  );
219
240
  }
220
241
 
242
+ const AFORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo";
243
+
221
244
  function normalizeAudio(index: number, label: string): string {
222
- return `[${index}:a]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,aresample=async=1[${label}]`;
245
+ return `[${index}:a]${AFORMAT},aresample=async=1[${label}]`;
246
+ }
247
+
248
+ // A number from the model, bounded and named. Every numeric parameter that reaches a
249
+ // filter string goes through here: it is both the input validation the model gets a
250
+ // useful message from, and the guarantee that nothing but a finite number is ever
251
+ // interpolated into a filtergraph.
252
+ function bounded(v: unknown, label: string, min: number, max: number, dflt: number): number {
253
+ if (v == null) return dflt;
254
+ const n = Number(v);
255
+ if (!Number.isFinite(n) || n < min || n > max) {
256
+ throw new ToolError(`\`${label}\` must be a number between ${min} and ${max} (got ${JSON.stringify(v)}).`);
257
+ }
258
+ return n;
259
+ }
260
+
261
+ // Colours reach ffmpeg as filter option values, so they are matched against a closed
262
+ // shape rather than passed through: a name, or #rrggbb(aa). Opacity is a separate
263
+ // numeric parameter, which is why `black@0.5` is not accepted here — it would be a
264
+ // second, unvalidated way to write a filter option.
265
+ const COLOR_RE = /^(?:#[0-9a-fA-F]{6}(?:[0-9a-fA-F]{2})?|[a-zA-Z]{3,20})$/;
266
+
267
+ function colorValue(v: string | undefined, dflt: string, label: string): string {
268
+ const c = String(v ?? dflt).trim();
269
+ if (!COLOR_RE.test(c)) {
270
+ throw new ToolError(`\`${label}\` must be a colour name (e.g. white) or #rrggbb (got ${JSON.stringify(v)}).`);
271
+ }
272
+ return c;
273
+ }
274
+
275
+ // Quote a path for use as a filter OPTION VALUE. Single quotes protect the separators
276
+ // the filtergraph parser would otherwise act on, and `\` and `:` are escaped inside them
277
+ // so a Windows path (C:\Users\…) survives — verified against ffmpeg 7 with a path
278
+ // containing a colon.
279
+ function filterPath(p: string): string {
280
+ return `'${p.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/:/g, "\\:")}'`;
223
281
  }
224
282
 
225
283
  // ── Operations ───────────────────────────────────────────────────────────────
226
284
 
285
+ /** One audio track placed on the timeline by `mix_audio`. */
286
+ interface MixTrack {
287
+ path: string;
288
+ atSeconds?: number;
289
+ gain?: number;
290
+ loop?: boolean;
291
+ duck?: boolean;
292
+ }
293
+
294
+ /** How type is drawn — shared by an `overlay_text` cue and a whole subtitle file. */
295
+ interface TextStyle {
296
+ position?: string;
297
+ fontSize?: number;
298
+ color?: string;
299
+ box?: boolean;
300
+ boxColor?: string;
301
+ boxOpacity?: number;
302
+ marginPx?: number;
303
+ }
304
+
305
+ /** One line of burnt-in type placed by `overlay_text`. */
306
+ interface TextCue extends TextStyle {
307
+ text: string;
308
+ fromSeconds?: number;
309
+ toSeconds?: number;
310
+ }
311
+
312
+ /** One still composited onto the picture by `overlay_image`. */
313
+ interface ImageLayer {
314
+ path: string;
315
+ fromSeconds?: number;
316
+ toSeconds?: number;
317
+ position?: string;
318
+ widthPercent?: number;
319
+ opacity?: number;
320
+ marginPx?: number;
321
+ }
322
+
227
323
  interface Params {
228
324
  operation: Operation;
229
325
  inputs?: string[];
@@ -240,6 +336,14 @@ interface Params {
240
336
  volume?: number;
241
337
  loopAudio?: boolean;
242
338
  keepOriginalAudio?: boolean;
339
+ tracks?: (MixTrack | string)[];
340
+ voiceTrack?: number;
341
+ texts?: TextCue[];
342
+ fontFile?: string;
343
+ images?: (ImageLayer | string)[];
344
+ subtitles?: string;
345
+ style?: TextStyle;
346
+ maxCharsPerLine?: number;
243
347
  }
244
348
 
245
349
  async function opProbe(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
@@ -422,6 +526,618 @@ async function opMuxAudio(cwd: string, p: Params, signal?: AbortSignal): Promise
422
526
  return `Muxed ${p.audio} onto ${p.input}${p.loopAudio ? " (looped)" : ""}${p.keepOriginalAudio ? " (mixed with the original audio)" : ""} → ${output} (${result.durationSec.toFixed(2)}s)`;
423
527
  }
424
528
 
529
+ // Ducking: press the bed down while the narration is speaking, let it back up between
530
+ // lines. A compressor keyed off the voice does this on its own, which is why the level
531
+ // isn't a parameter — the alternative is the model authoring a volume envelope per line
532
+ // and re-authoring it every time a word changes.
533
+ //
534
+ // `ratio=12` with a low threshold is a firm duck rather than a gentle one (a marketing
535
+ // bed should get out of the way, not merely dip), and `release=350` is long enough that
536
+ // the bed doesn't pump between words. attack=15ms keeps the first syllable clear.
537
+ const DUCK_FILTER = "sidechaincompress=threshold=0.03:ratio=12:attack=15:release=350:detection=rms";
538
+
539
+ // Several audio tracks, each placed at a moment and at a level, mixed onto one video.
540
+ //
541
+ // WHY THIS EXISTS ALONGSIDE mux_audio. mux_audio lays ONE track over the whole video at
542
+ // one volume, which is the right tool for "put this narration on" and useless for a cut
543
+ // that has narration AND a bed AND effects that land on specific frames. Without
544
+ // placement an effect can only start at zero, and without ducking a bed loud enough to
545
+ // hear is loud enough to bury the voice — those two gaps are most of the distance
546
+ // between "clips with audio" and something that sounds finished.
547
+ async function opMixAudio(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
548
+ if (!p.input) throw new ToolError("`mix_audio` needs `input` (the video the tracks go onto).");
549
+ const raw = p.tracks ?? [];
550
+ if (raw.length === 0) {
551
+ throw new ToolError(
552
+ "`mix_audio` needs at least one entry in `tracks`, each with a `path` (plus optional atSeconds, gain, loop, duck). " +
553
+ "For one track at one level over the whole video, mux_audio is simpler.",
554
+ );
555
+ }
556
+ // A bare string is accepted as a track at 0s and full level: models write
557
+ // `tracks: ["vo.mp3"]` often enough that refusing it teaches nothing.
558
+ const tracks: MixTrack[] = raw.map((t, i) => {
559
+ const track = typeof t === "string" ? { path: t } : ((t ?? {}) as MixTrack);
560
+ if (!track.path) throw new ToolError(`tracks[${i}] has no \`path\`.`);
561
+ return track;
562
+ });
563
+
564
+ // Argument validation runs BEFORE anything touches the disk: told that `duck` needs a
565
+ // `voiceTrack`, a model fixes the call; told that a file it hasn't written yet is
566
+ // missing, it goes looking for the wrong problem.
567
+ const voice = p.voiceTrack;
568
+ if (voice != null && (!Number.isInteger(Number(voice)) || Number(voice) < 0 || Number(voice) >= tracks.length)) {
569
+ throw new ToolError(
570
+ `\`voiceTrack\` must be the index of one of the ${tracks.length} track(s) — 0 to ${tracks.length - 1} — got ${JSON.stringify(p.voiceTrack)}.`,
571
+ );
572
+ }
573
+ const ducked = tracks.map((t, i) => (t.duck && i !== voice ? i : -1)).filter((i) => i >= 0);
574
+ if (ducked.length > 0 && voice == null) {
575
+ throw new ToolError(
576
+ "`duck` needs `voiceTrack` set to the index of the track everything else ducks under (normally the narration) — " +
577
+ "there is nothing to duck under otherwise.",
578
+ );
579
+ }
580
+
581
+ const video = requireExisting(cwd, p.input, "video");
582
+ const paths = tracks.map((t, i) => requireExisting(cwd, t.path, `tracks[${i}]`));
583
+ const output = prepareOutput(cwd, p.output, "mix_audio");
584
+
585
+ const [vInfo, ...tInfos] = await Promise.all([probe(video, signal), ...paths.map((f) => probe(f, signal))]);
586
+ if (!vInfo.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
587
+ for (const [i, info] of tInfos.entries()) {
588
+ if (!info.hasAudio) throw new ToolError(`tracks[${i}] (${tracks[i].path}) has no audio stream.`);
589
+ }
590
+
591
+ const args: string[] = ["-i", video];
592
+ for (const [i, t] of tracks.entries()) {
593
+ if (t.loop) {
594
+ if (!(vInfo.durationSec > 0)) {
595
+ throw new ToolError(`\`loop\` needs a video of known length, and ${p.input} reports none.`);
596
+ }
597
+ // Bound the loop AT THE INPUT rather than trusting -shortest to cut an endless
598
+ // stream off inside the graph: amix waits on every input, so an unbounded one
599
+ // makes the encode run until the ffmpeg timeout fires.
600
+ args.push("-stream_loop", "-1", "-t", vInfo.durationSec.toFixed(3));
601
+ }
602
+ args.push("-i", paths[i]);
603
+ }
604
+
605
+ const filters: string[] = [];
606
+ const labels: string[] = [];
607
+ if (p.keepOriginalAudio) {
608
+ if (!vInfo.hasAudio) throw new ToolError(`\`keepOriginalAudio\` was set but ${p.input} has no audio to keep.`);
609
+ filters.push(`[0:a]${AFORMAT}[aorig]`);
610
+ labels.push("aorig");
611
+ }
612
+
613
+ for (const [i, t] of tracks.entries()) {
614
+ const gain = bounded(t.gain, `tracks[${i}].gain`, 0, 4, 1);
615
+ const at = bounded(t.atSeconds, `tracks[${i}].atSeconds`, 0, 86_400, 0);
616
+ let chain = `[${i + 1}:a]${AFORMAT}`;
617
+ if (gain !== 1) chain += `,volume=${gain}`;
618
+ // adelay is what puts an effect on the frame it belongs to. `all=1` applies the
619
+ // delay to every channel — without it only the first channel moves, which sounds
620
+ // like a broken stereo image rather than a late cue.
621
+ if (at > 0) chain += `,adelay=${Math.round(at * 1000)}:all=1`;
622
+ filters.push(`${chain}[t${i}]`);
623
+ }
624
+
625
+ const mixLabel = new Map<number, string>(tracks.map((_, i) => [i, `t${i}`]));
626
+ if (ducked.length > 0) {
627
+ // The voice feeds the mix AND every compressor's sidechain, so it has to be split
628
+ // that many ways: a filter output can only be consumed once.
629
+ filters.push(`[t${voice}]asplit=${ducked.length + 1}[vmix]${ducked.map((_, k) => `[key${k}]`).join("")}`);
630
+ mixLabel.set(voice as number, "vmix");
631
+ ducked.forEach((i, k) => {
632
+ filters.push(`[t${i}][key${k}]${DUCK_FILTER}[d${i}]`);
633
+ mixLabel.set(i, `d${i}`);
634
+ });
635
+ }
636
+ labels.push(...tracks.map((_, i) => mixLabel.get(i) as string));
637
+
638
+ // normalize=0 is load-bearing: amix's default divides every input by the number of
639
+ // inputs, so the gains asked for above would silently come out at a third of
640
+ // themselves as soon as a third track joined. The limiter is what keeps the sum from
641
+ // clipping instead.
642
+ if (labels.length > 1) {
643
+ filters.push(
644
+ `${labels.map((l) => `[${l}]`).join("")}amix=inputs=${labels.length}:duration=longest:dropout_transition=0:normalize=0[amixed]`,
645
+ );
646
+ } else {
647
+ filters.push(`[${labels[0]}]anull[amixed]`);
648
+ }
649
+ filters.push("[amixed]alimiter=limit=0.95[aout]");
650
+
651
+ args.push(
652
+ "-filter_complex", filters.join(";"),
653
+ "-map", "0:v", "-map", "[aout]",
654
+ // The picture is untouched, so it is copied: a mix must never be a reason to
655
+ // re-encode video and lose a generation of quality.
656
+ "-c:v", "copy", ...AUDIO_ENCODE, "-shortest", output,
657
+ );
658
+ await ffmpeg(args, signal);
659
+
660
+ const result = await probe(output, signal);
661
+ const placed = tracks
662
+ .map((t, i) => {
663
+ const bits: string[] = [t.path];
664
+ if (t.atSeconds) bits.push(`at ${Number(t.atSeconds)}s`);
665
+ if (t.gain != null && Number(t.gain) !== 1) bits.push(`gain ${Number(t.gain)}`);
666
+ if (t.loop) bits.push("looped");
667
+ if (i === voice) bits.push("voice");
668
+ else if (ducked.includes(i)) bits.push("ducked under the voice");
669
+ return ` ${bits.join(", ")}`;
670
+ })
671
+ .join("\n");
672
+ return (
673
+ `Mixed ${tracks.length} track(s) onto ${p.input}` +
674
+ `${p.keepOriginalAudio ? " over its own audio" : ""} → ${output} (${result.durationSec.toFixed(2)}s)\n${placed}`
675
+ );
676
+ }
677
+
678
+ // Burn type into the picture: a caption, a lower third, an end card.
679
+ //
680
+ // The text is written to a TEMPORARY FILE and read back by drawtext's `textfile=`
681
+ // rather than interpolated into the filtergraph as `text=`. That is not tidiness: a
682
+ // caption is model- or user-supplied prose, and `:`, `,`, `'`, `%`, `[` and `\` all mean
683
+ // something to the filter parser. Escaping them correctly through two levels of quoting
684
+ // is exactly the kind of thing that works until someone writes a price or a ratio in a
685
+ // caption, at which point the graph either fails or silently means something else.
686
+ // `expansion=none` closes the other half of the same hole — drawtext would otherwise
687
+ // evaluate `%{...}` sequences inside the text.
688
+ async function opOverlayText(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
689
+ if (!p.input) throw new ToolError("`overlay_text` needs `input` (the video to draw on).");
690
+ const cues = p.texts ?? [];
691
+ if (cues.length === 0) {
692
+ throw new ToolError(
693
+ "`overlay_text` needs at least one entry in `texts`, each with `text` (plus optional fromSeconds, toSeconds, " +
694
+ "position, fontSize, color, box).",
695
+ );
696
+ }
697
+ const input = requireExisting(cwd, p.input, "video");
698
+ const output = prepareOutput(cwd, p.output, "overlay_text");
699
+ const info = await probe(input, signal);
700
+ if (!info.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
701
+
702
+ const font = resolveFont(cwd, p.fontFile);
703
+ const tmp = mkdtempSync(join(tmpdir(), "pv-drawtext-"));
704
+ try {
705
+ const drawtexts = cues.map((cue, i) =>
706
+ buildDrawtext({ cue: cue ?? ({} as TextCue), index: i, label: `texts[${i}]`, font, info, tmp }),
707
+ );
708
+ await drawOnVideo(input, output, drawtexts, info, signal);
709
+ const result = await probe(output, signal);
710
+ return `Burned ${cues.length} text overlay(s) into ${p.input} → ${output} (${result.durationSec.toFixed(2)}s, ${result.width}x${result.height})`;
711
+ } finally {
712
+ rmSync(tmp, { recursive: true, force: true });
713
+ }
714
+ }
715
+
716
+ // One `drawtext` filter for one cue. Shared by overlay_text and burn_subtitles so a
717
+ // caption and a subtitle are styled, bounded and (above all) ESCAPED by the same code:
718
+ // the second copy of this is the one that would forget `expansion=none`.
719
+ function buildDrawtext(args: {
720
+ cue: TextCue;
721
+ index: number;
722
+ label: string;
723
+ font: string;
724
+ info: MediaInfo;
725
+ tmp: string;
726
+ defaults?: { position?: string; box?: boolean };
727
+ }): string {
728
+ const { cue, index, label, font, info, tmp } = args;
729
+ const body = String(cue.text ?? "");
730
+ if (!body.trim()) throw new ToolError(`${label} has no \`text\`.`);
731
+ // Default type size scales with the frame: a 48px caption is right on 1080p and
732
+ // unreadable on a 4K master, and the model has no reliable idea which it holds.
733
+ const height = info.height ?? 1080;
734
+ const fontSize = Math.round(bounded(cue.fontSize, `${label}.fontSize`, 8, 400, Math.max(16, Math.round(height / 20))));
735
+ const margin = Math.round(bounded(cue.marginPx, `${label}.marginPx`, 0, 2000, Math.round(fontSize * 0.8)));
736
+ const { x, y } = positionExpr(cue.position ?? args.defaults?.position ?? "bottom", margin, label);
737
+ const opts = [
738
+ `fontfile=${filterPath(font)}`,
739
+ `textfile=${filterPath(writeCueFile(tmp, index, body))}`,
740
+ "expansion=none",
741
+ `fontsize=${fontSize}`,
742
+ `fontcolor=${colorValue(cue.color, "white", `${label}.color`)}`,
743
+ `x=${x}`,
744
+ `y=${y}`,
745
+ ];
746
+ if (cue.box ?? args.defaults?.box) {
747
+ const opacity = bounded(cue.boxOpacity, `${label}.boxOpacity`, 0, 1, 0.5);
748
+ opts.push("box=1", `boxcolor=${colorValue(cue.boxColor, "black", `${label}.boxColor`)}@${opacity}`, `boxborderw=${Math.round(fontSize / 3)}`);
749
+ } else {
750
+ // No box means the type has to hold against whatever is behind it, and generated
751
+ // footage is rarely obligingly dark. A border is cheaper than a box and reads
752
+ // as design rather than as a subtitle.
753
+ opts.push(`borderw=${Math.max(1, Math.round(fontSize / 16))}`, "bordercolor=black@0.85");
754
+ }
755
+ const from = cue.fromSeconds == null ? null : bounded(cue.fromSeconds, `${label}.fromSeconds`, 0, 86_400, 0);
756
+ const to = cue.toSeconds == null ? null : bounded(cue.toSeconds, `${label}.toSeconds`, 0, 86_400, 0);
757
+ if (from != null && to != null && to <= from) {
758
+ throw new ToolError(`${label}: toSeconds (${to}) must be after fromSeconds (${from}).`);
759
+ }
760
+ // Absent bounds mean "for the whole video", which is what omitting `enable` does.
761
+ if (from != null || to != null) {
762
+ opts.push(`enable='between(t,${from ?? 0},${to ?? Math.max(from ?? 0, info.durationSec || 86_400)})'`);
763
+ }
764
+ return `drawtext=${opts.join(":")}`;
765
+ }
766
+
767
+ // Run a chain of video filters over the picture and leave the sound alone. The picture is
768
+ // re-encoded (it has to be — we are drawing on it) and the audio is copied through
769
+ // untouched, so adding a caption never costs a generation of sound.
770
+ async function drawOnVideo(
771
+ input: string,
772
+ output: string,
773
+ filters: string[],
774
+ info: MediaInfo,
775
+ signal?: AbortSignal,
776
+ ): Promise<void> {
777
+ await ffmpeg(
778
+ [
779
+ "-i", input,
780
+ "-filter_complex", `[0:v]${filters.join(",")}[vout]`,
781
+ "-map", "[vout]",
782
+ ...(info.hasAudio ? ["-map", "0:a", "-c:a", "copy"] : []),
783
+ ...VIDEO_ENCODE,
784
+ output,
785
+ ],
786
+ signal,
787
+ );
788
+ }
789
+
790
+ function writeCueFile(dir: string, index: number, body: string): string {
791
+ const file = join(dir, `cue-${index}.txt`);
792
+ writeFileSync(file, body, "utf8");
793
+ return file;
794
+ }
795
+
796
+ const POSITIONS = ["top-left", "top", "top-right", "left", "center", "right", "bottom-left", "bottom", "bottom-right"] as const;
797
+
798
+ // The frame/element variable names a filter exposes for placement. drawtext measures the
799
+ // frame as w/h and the drawn text as text_w/text_h; overlay measures the MAIN input as
800
+ // W/H and the overlaid one as w/h. Same nine positions, two vocabularies.
801
+ const DRAWTEXT_VARS = { frameW: "w", frameH: "h", itemW: "text_w", itemH: "text_h" };
802
+ const OVERLAY_VARS = { frameW: "W", frameH: "H", itemW: "w", itemH: "h" };
803
+
804
+ // Named positions only — never a caller-supplied x/y expression. Both filters evaluate x
805
+ // and y as arithmetic over frame variables, so accepting one would hand the model (or
806
+ // anything that reached it) an expression evaluator inside the filtergraph. The nine names
807
+ // below cover every placement a title, caption or logo actually wants.
808
+ function positionExpr(
809
+ position: string,
810
+ margin: number,
811
+ label: string,
812
+ vars: typeof DRAWTEXT_VARS = DRAWTEXT_VARS,
813
+ ): { x: string; y: string } {
814
+ const name = String(position).trim().toLowerCase();
815
+ if (!(POSITIONS as readonly string[]).includes(name)) {
816
+ throw new ToolError(`${label}.position must be one of: ${POSITIONS.join(", ")} (got ${JSON.stringify(position)}).`);
817
+ }
818
+ const y = name.startsWith("top")
819
+ ? String(margin)
820
+ : name.startsWith("bottom")
821
+ ? `${vars.frameH}-${vars.itemH}-${margin}`
822
+ : `(${vars.frameH}-${vars.itemH})/2`;
823
+ const x = name.endsWith("left")
824
+ ? String(margin)
825
+ : name.endsWith("right")
826
+ ? `${vars.frameW}-${vars.itemW}-${margin}`
827
+ : `(${vars.frameW}-${vars.itemW})/2`;
828
+ return { x, y };
829
+ }
830
+
831
+ // Where to find a font, in order: the one the caller named, then the platform's usual
832
+ // suspects. drawtext with neither `fontfile` nor `font` only works on a build with
833
+ // fontconfig, and the failure ("Cannot find a valid font...") arrives after the encode
834
+ // starts — so a path is resolved here, up front, where the message can name the fix.
835
+ const FONT_CANDIDATES: Record<string, string[]> = {
836
+ darwin: [
837
+ "/System/Library/Fonts/Helvetica.ttc",
838
+ "/System/Library/Fonts/Supplemental/Arial.ttf",
839
+ "/Library/Fonts/Arial.ttf",
840
+ ],
841
+ linux: [
842
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
843
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
844
+ "/usr/share/fonts/TTF/DejaVuSans.ttf",
845
+ "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
846
+ "/usr/share/fonts/liberation-sans/LiberationSans-Bold.ttf",
847
+ ],
848
+ win32: ["C:\\Windows\\Fonts\\arialbd.ttf", "C:\\Windows\\Fonts\\arial.ttf", "C:\\Windows\\Fonts\\segoeui.ttf"],
849
+ };
850
+
851
+ function resolveFont(cwd: string, fontFile?: string): string {
852
+ if (fontFile) return requireExisting(cwd, fontFile, "fontFile");
853
+ for (const candidate of FONT_CANDIDATES[process.platform] ?? FONT_CANDIDATES.linux) {
854
+ if (existsSync(candidate)) return candidate;
855
+ }
856
+ throw new ToolError(
857
+ "no system font found to draw with — pass `fontFile` with the path to a .ttf/.otf " +
858
+ `(looked for ${(FONT_CANDIDATES[process.platform] ?? FONT_CANDIDATES.linux).join(", ")}).`,
859
+ );
860
+ }
861
+
862
+ // Composite stills onto the picture: the logo bug in the corner, an end card, a lower-third
863
+ // plate a caption then sits on. The move overlay_text can't make — type is not artwork, and
864
+ // a brand mark drawn as text is a brand mark drawn wrong.
865
+ async function opOverlayImage(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
866
+ if (!p.input) throw new ToolError("`overlay_image` needs `input` (the video to draw on).");
867
+ const raw = p.images ?? [];
868
+ if (raw.length === 0) {
869
+ throw new ToolError(
870
+ "`overlay_image` needs at least one entry in `images`, each with a `path` (plus optional fromSeconds, " +
871
+ "toSeconds, position, widthPercent, opacity, marginPx).",
872
+ );
873
+ }
874
+ // A bare string is a full-strength overlay at the default position, for the same reason
875
+ // mix_audio accepts one: `images: ["logo.png"]` is what a model writes first.
876
+ const layers: ImageLayer[] = raw.map((entry, i) => {
877
+ const layer = typeof entry === "string" ? { path: entry } : ((entry ?? {}) as ImageLayer);
878
+ if (!layer.path) throw new ToolError(`images[${i}] has no \`path\`.`);
879
+ return layer;
880
+ });
881
+
882
+ // Everything checkable without the disk is checked first, as in mix_audio: told that
883
+ // `position` isn't one of the nine, a model fixes the call; told that a file it is about
884
+ // to write doesn't exist yet, it goes looking for the wrong problem. (The numeric bounds
885
+ // below are re-applied where they are used — a second call to `bounded` with the same
886
+ // value is free, and one shared place to validate would need the probe results.)
887
+ layers.forEach((layer, i) => {
888
+ const label = `images[${i}]`;
889
+ positionExpr(layer.position ?? "top-right", 0, label, OVERLAY_VARS);
890
+ bounded(layer.opacity, `${label}.opacity`, 0, 1, 1);
891
+ bounded(layer.widthPercent, `${label}.widthPercent`, 1, 100, 100);
892
+ const from = bounded(layer.fromSeconds, `${label}.fromSeconds`, 0, 86_400, 0);
893
+ const to = layer.toSeconds == null ? null : bounded(layer.toSeconds, `${label}.toSeconds`, 0, 86_400, 0);
894
+ if (layer.fromSeconds != null && to != null && to <= from) {
895
+ throw new ToolError(`${label}: toSeconds (${to}) must be after fromSeconds (${from}).`);
896
+ }
897
+ });
898
+
899
+ const input = requireExisting(cwd, p.input, "video");
900
+ const paths = layers.map((l, i) => requireExisting(cwd, l.path, `images[${i}]`));
901
+ const output = prepareOutput(cwd, p.output, "overlay_image");
902
+ const [info, ...layerInfos] = await Promise.all([probe(input, signal), ...paths.map((f) => probe(f, signal))]);
903
+ if (!info.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
904
+ for (const [i, li] of layerInfos.entries()) {
905
+ if (!li.hasVideo) throw new ToolError(`images[${i}] (${layers[i].path}) is not an image or video ffmpeg can read.`);
906
+ }
907
+
908
+ const frameW = info.width ?? 1280;
909
+ const args: string[] = ["-i", input];
910
+ for (const [i, li] of layerInfos.entries()) {
911
+ // A still decodes to ONE frame, which composited over a 30s video appears on frame 1
912
+ // and is gone — so a still is looped for the video's length, while an animated overlay
913
+ // (a GIF, a video sting) plays at its own pace and `eof_action=pass` below lets the
914
+ // film carry on after it ends.
915
+ //
916
+ // DURATION is what distinguishes them, not frame rate: the image demuxer fabricates
917
+ // `avg_frame_rate=25/1` for a PNG, so keying off fps silently skipped the loop and
918
+ // produced a video whose logo flashed on the first frame only.
919
+ if (li.durationSec === 0 && info.durationSec > 0) args.push("-loop", "1", "-t", info.durationSec.toFixed(3));
920
+ args.push("-i", paths[i]);
921
+ }
922
+
923
+ const filters: string[] = [];
924
+ const scaledTo: (number | null)[] = [];
925
+ for (const [i, layer] of layers.entries()) {
926
+ const label = `images[${i}]`;
927
+ const pct = layer.widthPercent == null ? null : bounded(layer.widthPercent, `${label}.widthPercent`, 1, 100, 100);
928
+ const nativeW = layerInfos[i].width ?? frameW;
929
+ // No widthPercent means "as authored" — EXCEPT when the art is wider than the frame,
930
+ // where leaving it alone silently crops the overlay off the right-hand side and reads
931
+ // as a broken render rather than as a choice.
932
+ const targetW = pct != null ? Math.round((frameW * pct) / 100) : nativeW > frameW ? frameW : null;
933
+ scaledTo.push(targetW);
934
+ const chain: string[] = [];
935
+ if (targetW != null) chain.push(`scale=${Math.max(2, targetW - (targetW % 2))}:-1`);
936
+ const opacity = bounded(layer.opacity, `${label}.opacity`, 0, 1, 1);
937
+ // A watermark has to let the picture through. colorchannelmixer SCALES the alpha
938
+ // channel, so `format=rgba` first: art saved without transparency has no alpha to
939
+ // scale, and the layer would come out fully opaque with no error.
940
+ if (opacity < 1) chain.push("format=rgba", `colorchannelmixer=aa=${opacity}`);
941
+ filters.push(`[${i + 1}:v]${chain.length > 0 ? chain.join(",") : "null"}[img${i}]`);
942
+ }
943
+
944
+ let vPrev = "0:v";
945
+ layers.forEach((layer, i) => {
946
+ const label = `images[${i}]`;
947
+ const margin = Math.round(bounded(layer.marginPx, `${label}.marginPx`, 0, 2000, Math.round(frameW * 0.03)));
948
+ const { x, y } = positionExpr(layer.position ?? "top-right", margin, label, OVERLAY_VARS);
949
+ const from = layer.fromSeconds == null ? null : bounded(layer.fromSeconds, `${label}.fromSeconds`, 0, 86_400, 0);
950
+ const to = layer.toSeconds == null ? null : bounded(layer.toSeconds, `${label}.toSeconds`, 0, 86_400, 0);
951
+ if (from != null && to != null && to <= from) {
952
+ throw new ToolError(`${label}: toSeconds (${to}) must be after fromSeconds (${from}).`);
953
+ }
954
+ const opts = [`x=${x}`, `y=${y}`];
955
+ if (from != null || to != null) {
956
+ opts.push(`enable='between(t,${from ?? 0},${to ?? Math.max(from ?? 0, info.durationSec || 86_400)})'`);
957
+ }
958
+ // `eof_action=pass` keeps the MAIN video running once a layer's own frames end (a
959
+ // windowed overlay, or art shorter than the video): the default would end the output
960
+ // there, truncating the film to the length of its logo.
961
+ opts.push("eof_action=pass");
962
+ const next = i === layers.length - 1 ? "vout" : `vo${i}`;
963
+ filters.push(`[${vPrev}][img${i}]overlay=${opts.join(":")}[${next}]`);
964
+ vPrev = next;
965
+ });
966
+
967
+ await ffmpeg(
968
+ [
969
+ ...args,
970
+ "-filter_complex", filters.join(";"),
971
+ "-map", "[vout]",
972
+ ...(info.hasAudio ? ["-map", "0:a", "-c:a", "copy"] : []),
973
+ ...VIDEO_ENCODE,
974
+ output,
975
+ ],
976
+ signal,
977
+ );
978
+ const result = await probe(output, signal);
979
+ const placed = layers
980
+ .map((l, i) => {
981
+ const bits = [l.path, l.position ?? "top-right"];
982
+ if (scaledTo[i] != null) bits.push(`${scaledTo[i]}px wide`);
983
+ if (l.opacity != null && Number(l.opacity) < 1) bits.push(`opacity ${Number(l.opacity)}`);
984
+ if (l.fromSeconds != null || l.toSeconds != null) bits.push(`${l.fromSeconds ?? 0}s-${l.toSeconds ?? "end"}`);
985
+ return ` ${bits.join(", ")}`;
986
+ })
987
+ .join("\n");
988
+ return `Composited ${layers.length} image(s) onto ${p.input} → ${output} (${result.durationSec.toFixed(2)}s, ${result.width}x${result.height})\n${placed}`;
989
+ }
990
+
991
+ // A graph with a drawtext per cue is the cost of this approach; bound it rather than
992
+ // building a filter string megabytes long and letting ffmpeg fail on argv length.
993
+ const MAX_SUBTITLE_CUES = 500;
994
+ // The broadcast convention, and about what fits at frame-height/20 on a 16:9 master.
995
+ const DEFAULT_SUBTITLE_CHARS = 42;
996
+
997
+ // Burn a subtitle file into the picture.
998
+ //
999
+ // WHY WE PARSE IT OURSELVES rather than handing ffmpeg's `subtitles=` filter the file.
1000
+ // That filter needs libass, which is a build option and not present in every ffmpeg a user
1001
+ // has installed — and its styling goes through `force_style`, a comma-separated string
1002
+ // inside a filter option value, which is a second quoting layer to get wrong on exactly the
1003
+ // kind of caller-supplied text this file is careful about everywhere else. Parsing SRT/VTT
1004
+ // here and emitting the same drawtext cues overlay_text uses means one text path, already
1005
+ // escaped (textfile= + expansion=none), and no dependency beyond drawtext itself.
1006
+ //
1007
+ // What that costs, stated plainly: no italics, no per-cue positioning overrides, no
1008
+ // karaoke timing. Styling is uniform across the file, which is what burnt-in captions on a
1009
+ // marketing cut want anyway.
1010
+ async function opBurnSubtitles(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
1011
+ if (!p.input) throw new ToolError("`burn_subtitles` needs `input` (the video to draw on).");
1012
+ if (!p.subtitles) throw new ToolError("`burn_subtitles` needs `subtitles` (the path to an .srt or .vtt file).");
1013
+ const wrapAt = Math.round(bounded(p.maxCharsPerLine, "maxCharsPerLine", 10, 200, DEFAULT_SUBTITLE_CHARS));
1014
+ const input = requireExisting(cwd, p.input, "video");
1015
+ const subs = requireExisting(cwd, p.subtitles, "subtitles");
1016
+ const output = prepareOutput(cwd, p.output, "burn_subtitles");
1017
+ const info = await probe(input, signal);
1018
+ if (!info.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
1019
+
1020
+ const parsed = parseSubtitleCues(readFileSync(subs, "utf8"));
1021
+ if (parsed.length === 0) {
1022
+ throw new ToolError(
1023
+ `no cues found in ${p.subtitles} — expected SubRip (.srt) or WebVTT (.vtt), where each cue is a ` +
1024
+ "`00:00:01,000 --> 00:00:04,000` line followed by its text.",
1025
+ );
1026
+ }
1027
+ if (parsed.length > MAX_SUBTITLE_CUES) {
1028
+ throw new ToolError(
1029
+ `${p.subtitles} has ${parsed.length} cues, over the ${MAX_SUBTITLE_CUES} this can burn in one pass. ` +
1030
+ "Trim the video and its subtitles into sections and burn each, or use fewer, longer cues.",
1031
+ );
1032
+ }
1033
+
1034
+ const style = (p.style ?? {}) as TextStyle;
1035
+ const font = resolveFont(cwd, p.fontFile);
1036
+ const tmp = mkdtempSync(join(tmpdir(), "pv-subtitles-"));
1037
+ try {
1038
+ const drawtexts = parsed.map((cue, i) =>
1039
+ buildDrawtext({
1040
+ cue: { ...style, text: wrapLines(cue.text, wrapAt), fromSeconds: cue.from, toSeconds: cue.to },
1041
+ index: i,
1042
+ label: "style",
1043
+ font,
1044
+ info,
1045
+ tmp,
1046
+ // Captions live at the bottom, and against generated footage they need a plate
1047
+ // more often than a title does — so a box is the default here where overlay_text
1048
+ // defaults to an outline. `style.box: false` turns it back off.
1049
+ defaults: { position: "bottom", box: true },
1050
+ }),
1051
+ );
1052
+ await drawOnVideo(input, output, drawtexts, info, signal);
1053
+ const result = await probe(output, signal);
1054
+ const last = parsed[parsed.length - 1];
1055
+ return (
1056
+ `Burned ${parsed.length} subtitle cue(s) from ${p.subtitles} into ${p.input} → ${output} ` +
1057
+ `(${result.durationSec.toFixed(2)}s, wrapped at ${wrapAt} characters, last cue ends at ${last.to.toFixed(2)}s)` +
1058
+ (last.to > info.durationSec + 0.5
1059
+ ? `\nNote: the subtitles run ${(last.to - info.durationSec).toFixed(2)}s past the end of the video — ` +
1060
+ "those cues will never be seen."
1061
+ : "")
1062
+ );
1063
+ } finally {
1064
+ rmSync(tmp, { recursive: true, force: true });
1065
+ }
1066
+ }
1067
+
1068
+ interface ParsedCue {
1069
+ from: number;
1070
+ to: number;
1071
+ text: string;
1072
+ }
1073
+
1074
+ // "00:00:01,000", "00:00:01.000", "01:02.500" (VTT drops the hour) → seconds.
1075
+ function parseTimecode(raw: string): number | null {
1076
+ const m = /^(?:(\d{1,3}):)?(\d{1,2}):(\d{1,2})(?:[.,](\d{1,3}))?$/.exec(raw.trim());
1077
+ if (!m) return null;
1078
+ const ms = m[4] ? Number(m[4].padEnd(3, "0")) : 0;
1079
+ return Number(m[1] ?? 0) * 3600 + Number(m[2]) * 60 + Number(m[3]) + ms / 1000;
1080
+ }
1081
+
1082
+ // Inline markup we cannot render and must not print: SRT/VTT tags (<i>, <c.yellow>) and
1083
+ // ASS override blocks ({\an8}). Dropped rather than escaped — a caption reading "<i>" is
1084
+ // worse than one that lost its italics.
1085
+ function stripMarkup(s: string): string {
1086
+ return s.replace(/<[^>\n]*>/g, "").replace(/\{\\[^}\n]*\}/g, "");
1087
+ }
1088
+
1089
+ // Cue-block parser covering both formats, because they differ only in the separator inside
1090
+ // the timestamp and in a header/settings line we can ignore. Anything that isn't a cue —
1091
+ // SubRip's sequence numbers, `WEBVTT`, `NOTE` blocks, styling blocks — is skipped by
1092
+ // keying off the `-->` line rather than by counting lines, which is what makes a
1093
+ // hand-edited file with an extra blank line parse rather than derail.
1094
+ export function parseSubtitleCues(body: string): ParsedCue[] {
1095
+ const lines = body.replace(/^/, "").replace(/\r\n?/g, "\n").split("\n");
1096
+ const cues: ParsedCue[] = [];
1097
+ for (let i = 0; i < lines.length; i++) {
1098
+ const arrow = lines[i].indexOf("-->");
1099
+ if (arrow === -1) continue;
1100
+ const from = parseTimecode(lines[i].slice(0, arrow));
1101
+ // WebVTT cue settings ("align:start line:90%") follow the end time on the same line.
1102
+ const to = parseTimecode(lines[i].slice(arrow + 3).trim().split(/\s+/)[0] ?? "");
1103
+ if (from == null || to == null || to <= from) continue;
1104
+ const text: string[] = [];
1105
+ let j = i + 1;
1106
+ for (; j < lines.length && lines[j].trim() !== ""; j++) text.push(lines[j]);
1107
+ i = j;
1108
+ const clean = stripMarkup(text.join("\n"))
1109
+ .split("\n")
1110
+ .map((l) => l.trim())
1111
+ .filter(Boolean)
1112
+ .join("\n");
1113
+ if (clean) cues.push({ from, to, text: clean });
1114
+ }
1115
+ return cues;
1116
+ }
1117
+
1118
+ // Wrap to a character count, per authored line. drawtext does not wrap, so a cue written as
1119
+ // one long line would run off both edges of the frame. Existing line breaks are kept — a
1120
+ // subtitler who split a line meant it — and only over-long lines are broken further.
1121
+ export function wrapLines(text: string, max: number): string {
1122
+ return text
1123
+ .split("\n")
1124
+ .map((line) => {
1125
+ const out: string[] = [];
1126
+ let current = "";
1127
+ for (const word of line.split(/\s+/).filter(Boolean)) {
1128
+ if (current === "") current = word;
1129
+ else if (current.length + 1 + word.length <= max) current += ` ${word}`;
1130
+ else {
1131
+ out.push(current);
1132
+ current = word;
1133
+ }
1134
+ }
1135
+ if (current) out.push(current);
1136
+ return out.join("\n");
1137
+ })
1138
+ .join("\n");
1139
+ }
1140
+
425
1141
  async function opTrim(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
426
1142
  if (!p.input) throw new ToolError("`trim` needs `input`.");
427
1143
  const input = requireExisting(cwd, p.input, "input");
@@ -511,7 +1227,28 @@ export const videoComposeToolDefinition = {
511
1227
  " slideshow — inputs (images), output, optional secondsPerImage, crossfadeSeconds, size, fps: " +
512
1228
  "turn stills into a video. Output is silent; add sound with mux_audio.\n" +
513
1229
  " mux_audio — input (video), audio, output, optional loopAudio, volume, keepOriginalAudio: put " +
514
- "narration or a score onto a video, trimmed to the video's length.\n" +
1230
+ "ONE track onto a video at one level, trimmed to the video's length.\n" +
1231
+ " mix_audio — input (video), tracks, output, optional voiceTrack, keepOriginalAudio: put SEVERAL " +
1232
+ "tracks on at once, each placed and levelled. Every track takes `path` plus optional `atSeconds` (when " +
1233
+ "it starts — this is how a sound effect lands on the frame it belongs to), `gain` (0-4), `loop` (repeat " +
1234
+ "a short bed to the end) and `duck` (press this track down while the narration speaks). Set " +
1235
+ "`voiceTrack` to the index of the narration for `duck` to key off. Use this rather than several " +
1236
+ "mux_audio passes: each pass re-encodes the sound and mixes blind to what comes next.\n" +
1237
+ " overlay_text — input (video), texts, output, optional fontFile: burn type into the picture. Each " +
1238
+ "entry takes `text` plus optional `fromSeconds`/`toSeconds` (when it shows), `position` (one of " +
1239
+ "top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right), `fontSize` " +
1240
+ "(defaults to the frame height / 20), `color`, `box`, `boxColor`, `boxOpacity`, `marginPx`. Audio is " +
1241
+ "copied through untouched.\n" +
1242
+ " overlay_image — input (video), images, output: composite stills onto the picture — a logo bug, an " +
1243
+ "end card, a lower-third plate. Each entry takes `path` plus optional `fromSeconds`/`toSeconds`, " +
1244
+ "`position` (same nine names, defaults to top-right), `widthPercent` (of the frame width — the way to " +
1245
+ "size a logo, since the art's own pixel size means nothing), `opacity` (0-1, for a watermark) and " +
1246
+ "`marginPx`. Audio is copied through untouched.\n" +
1247
+ " burn_subtitles — input (video), subtitles (.srt or .vtt), output, optional style, maxCharsPerLine, " +
1248
+ "fontFile: burn a whole subtitle file in, timed from the file itself. Lines are wrapped to fit (42 " +
1249
+ "characters by default) and drawn at the bottom on a plate; `style` takes the same fields as an " +
1250
+ "overlay_text entry and applies to every cue. Needs no libass — italics and per-cue positioning in the " +
1251
+ "file are dropped, everything else is honoured.\n" +
515
1252
  " trim — input, output, start, optional duration: cut a section out, frame-accurate.\n" +
516
1253
  " extract_frame — input, output, optional at (seconds, or \"last\"): pull one still. Extract the " +
517
1254
  "last frame of a clip and pass it to generate_video as `firstFrame` to keep consecutive clips " +
@@ -520,7 +1257,8 @@ export const videoComposeToolDefinition = {
520
1257
  "Needs ffmpeg installed; every operation says so plainly if it isn't.",
521
1258
  parameters: Type.Object({
522
1259
  operation: Type.String({
523
- description: 'One of: "probe", "concat", "slideshow", "mux_audio", "trim", "extract_frame", "gif".',
1260
+ description:
1261
+ 'One of: "probe", "concat", "slideshow", "mux_audio", "mix_audio", "overlay_text", "overlay_image", "burn_subtitles", "trim", "extract_frame", "gif".',
524
1262
  }),
525
1263
  inputs: Type.Optional(Type.Array(Type.String(), { description: "Input paths, in order. Used by concat, slideshow, and probe." })),
526
1264
  input: Type.Optional(Type.String({ description: "A single input path. Used by probe, mux_audio (the video), trim, extract_frame, gif." })),
@@ -535,7 +1273,83 @@ export const videoComposeToolDefinition = {
535
1273
  at: Type.Optional(Type.Union([Type.Number(), Type.String()], { description: 'Timestamp for extract_frame: seconds, or "last" for the final frame. Defaults to 0.' })),
536
1274
  volume: Type.Optional(Type.Number({ description: "Volume multiplier for the added track, 0-4. mux_audio only; defaults to 1." })),
537
1275
  loopAudio: Type.Optional(Type.Boolean({ description: "Repeat a short audio track until the video ends. mux_audio only." })),
538
- keepOriginalAudio: Type.Optional(Type.Boolean({ description: "Mix under the video's existing audio instead of replacing it. mux_audio only." })),
1276
+ keepOriginalAudio: Type.Optional(Type.Boolean({ description: "Mix under the video's existing audio instead of replacing it. mux_audio and mix_audio." })),
1277
+ tracks: Type.Optional(
1278
+ Type.Array(
1279
+ Type.Object({
1280
+ path: Type.String({ description: "The audio file." }),
1281
+ atSeconds: Type.Optional(Type.Number({ description: "When it starts, in seconds from the beginning of the video. Defaults to 0." })),
1282
+ gain: Type.Optional(Type.Number({ description: "Volume multiplier, 0-4. Defaults to 1. A bed under narration usually wants 0.2-0.4." })),
1283
+ loop: Type.Optional(Type.Boolean({ description: "Repeat this track until the video ends — for a short music bed." })),
1284
+ duck: Type.Optional(Type.Boolean({ description: "Press this track down whenever the `voiceTrack` is speaking, and let it back up between lines." })),
1285
+ }),
1286
+ { description: "The tracks to place, in any order. mix_audio only." },
1287
+ ),
1288
+ ),
1289
+ voiceTrack: Type.Optional(
1290
+ Type.Number({
1291
+ description:
1292
+ "Index into `tracks` of the narration everything else ducks under (0 = the first track). " +
1293
+ "Required if any track sets `duck`. mix_audio only.",
1294
+ }),
1295
+ ),
1296
+ texts: Type.Optional(
1297
+ Type.Array(
1298
+ Type.Object({
1299
+ text: Type.String({ description: "The words to draw. Punctuation, colons and percent signs are all safe — the text never reaches the filter as syntax." }),
1300
+ fromSeconds: Type.Optional(Type.Number({ description: "When it appears. Omit for the whole video." })),
1301
+ toSeconds: Type.Optional(Type.Number({ description: "When it disappears. Omit to hold to the end." })),
1302
+ position: Type.Optional(Type.String({ description: "top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right. Defaults to bottom." })),
1303
+ fontSize: Type.Optional(Type.Number({ description: "In pixels, 8-400. Defaults to the frame height / 20." })),
1304
+ color: Type.Optional(Type.String({ description: "A colour name or #rrggbb. Defaults to white." })),
1305
+ box: Type.Optional(Type.Boolean({ description: "Draw a filled box behind the type. Without it the type gets a dark outline instead." })),
1306
+ boxColor: Type.Optional(Type.String({ description: "Box colour name or #rrggbb. Defaults to black." })),
1307
+ boxOpacity: Type.Optional(Type.Number({ description: "Box opacity, 0-1. Defaults to 0.5." })),
1308
+ marginPx: Type.Optional(Type.Number({ description: "Distance from the frame edge. Defaults to about 0.8 of the font size." })),
1309
+ }),
1310
+ { description: "The text to burn in. overlay_text only." },
1311
+ ),
1312
+ ),
1313
+ fontFile: Type.Optional(
1314
+ Type.String({
1315
+ description:
1316
+ "Path to a .ttf/.otf to draw with. Omit to use a system font — pass one if the video needs the brand's typeface. overlay_text and burn_subtitles.",
1317
+ }),
1318
+ ),
1319
+ images: Type.Optional(
1320
+ Type.Array(
1321
+ Type.Object({
1322
+ path: Type.String({ description: "The image (or short video) to composite. PNG transparency is preserved." }),
1323
+ fromSeconds: Type.Optional(Type.Number({ description: "When it appears. Omit for the whole video." })),
1324
+ toSeconds: Type.Optional(Type.Number({ description: "When it disappears. Omit to hold to the end." })),
1325
+ position: Type.Optional(Type.String({ description: "top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right. Defaults to top-right." })),
1326
+ widthPercent: Type.Optional(Type.Number({ description: "Width as a percentage of the video's width, 1-100. The right way to size a logo. Omit to use the art at its own size (shrunk to fit if it is wider than the frame)." })),
1327
+ opacity: Type.Optional(Type.Number({ description: "0-1. Defaults to 1 (opaque); 0.3-0.6 is the range a watermark wants." })),
1328
+ marginPx: Type.Optional(Type.Number({ description: "Distance from the frame edge. Defaults to about 3% of the frame width." })),
1329
+ }),
1330
+ { description: "The images to composite, drawn in order (later entries sit on top). overlay_image only." },
1331
+ ),
1332
+ ),
1333
+ subtitles: Type.Optional(
1334
+ Type.String({ description: "Path to a SubRip (.srt) or WebVTT (.vtt) file. Its own timings drive the cues. burn_subtitles only." }),
1335
+ ),
1336
+ style: Type.Optional(
1337
+ Type.Object({
1338
+ position: Type.Optional(Type.String({ description: "Where every cue sits. Defaults to bottom." })),
1339
+ fontSize: Type.Optional(Type.Number({ description: "In pixels, 8-400. Defaults to the frame height / 20." })),
1340
+ color: Type.Optional(Type.String({ description: "A colour name or #rrggbb. Defaults to white." })),
1341
+ box: Type.Optional(Type.Boolean({ description: "Draw a plate behind the type. Defaults to TRUE for subtitles; set false for an outline instead." })),
1342
+ boxColor: Type.Optional(Type.String({ description: "Plate colour name or #rrggbb. Defaults to black." })),
1343
+ boxOpacity: Type.Optional(Type.Number({ description: "Plate opacity, 0-1. Defaults to 0.5." })),
1344
+ marginPx: Type.Optional(Type.Number({ description: "Distance from the frame edge. Defaults to about 0.8 of the font size." })),
1345
+ }, { description: "How every subtitle cue is drawn. burn_subtitles only." }),
1346
+ ),
1347
+ maxCharsPerLine: Type.Optional(
1348
+ Type.Number({
1349
+ description:
1350
+ "Wrap subtitle lines longer than this, 10-200. Defaults to 42 (the broadcast convention). burn_subtitles only.",
1351
+ }),
1352
+ ),
539
1353
  }),
540
1354
  async execute(
541
1355
  _toolCallId: string,
@@ -555,6 +1369,10 @@ export const videoComposeToolDefinition = {
555
1369
  case "concat": return text(await opConcat(cwd, params, signal));
556
1370
  case "slideshow": return text(await opSlideshow(cwd, params, signal));
557
1371
  case "mux_audio": return text(await opMuxAudio(cwd, params, signal));
1372
+ case "mix_audio": return text(await opMixAudio(cwd, params, signal));
1373
+ case "overlay_text": return text(await opOverlayText(cwd, params, signal));
1374
+ case "overlay_image": return text(await opOverlayImage(cwd, params, signal));
1375
+ case "burn_subtitles": return text(await opBurnSubtitles(cwd, params, signal));
558
1376
  case "trim": return text(await opTrim(cwd, params, signal));
559
1377
  case "extract_frame": return text(await opExtractFrame(cwd, params, signal));
560
1378
  case "gif": return text(await opGif(cwd, params, signal));