privateer-agent 0.12.13 → 0.12.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/privateer-subagent.mjs +10 -2
- package/extensions/privateer-gate.ts +7 -0
- package/extensions/privateer-media.ts +17 -4
- package/extensions/privateer-privacy.ts +13 -33
- package/extensions/privateer-tools.ts +8 -3
- package/package.json +1 -1
- package/patches/@earendil-works+pi-coding-agent+0.84.1.patch +102 -5
- package/src/acp/run.ts +29 -2
- package/src/channels/run.ts +9 -0
- package/src/cli/chat.ts +47 -0
- package/src/config/moat.ts +6 -26
- package/src/config/moatManifest.ts +5 -0
- package/src/config/privacyPolicy.ts +97 -0
- package/src/engine/errors.ts +97 -1
- package/src/ext/permissionGate.ts +6 -0
- package/src/harbor/index.ts +65 -11
- package/src/outbox/cloudOutbox.ts +72 -4
- package/src/permissions/childSpend.ts +105 -0
- package/src/permissions/classify.ts +61 -3
- package/src/permissions/modeGate.ts +39 -0
- package/src/providers/account.ts +8 -1
- package/src/providers/phala/measurements.ts +170 -0
- package/src/providers/phala/pin.ts +98 -0
- package/src/providers/phalaSeal.ts +131 -9
- package/src/providers/sealedShim.ts +6 -1
- package/src/remote/liveTaskSession.ts +8 -1
- package/src/remote/relayClient.ts +21 -0
- package/src/remote/remoteBridge.ts +9 -0
- package/src/routines/store.ts +11 -0
- package/src/tools/media.ts +516 -11
- package/src/tools/routineResult.ts +146 -0
- package/src/tools/videoCompose.ts +825 -7
|
@@ -15,13 +15,22 @@
|
|
|
15
15
|
// something finished. ffmpeg is spawned WITHOUT a shell and every argument is built
|
|
16
16
|
// from validated parameters, so nothing a prompt injects can become a command.
|
|
17
17
|
//
|
|
18
|
+
// FILTERGRAPHS ARE THE SECOND INJECTION SURFACE, and a shell-free spawn does nothing
|
|
19
|
+
// about it: `-filter_complex` takes one string that ffmpeg itself parses, where `:`,
|
|
20
|
+
// `,`, `;`, `'`, `[` and `\` are all syntax. So no caller-supplied text is ever
|
|
21
|
+
// interpolated into a graph raw. Numbers go through `bounded()`, colours through a
|
|
22
|
+
// closed pattern, positions through a fixed list of nine names (never an x/y
|
|
23
|
+
// expression), paths through `filterPath()`, and prose — a caption — is written to a
|
|
24
|
+
// temp file that drawtext READS, so its content is never parsed as a filter at all.
|
|
25
|
+
//
|
|
18
26
|
// ffmpeg is not a dependency we ship. When it's missing every operation says so once,
|
|
19
27
|
// clearly, with the install line for the platform — rather than surfacing ENOENT.
|
|
20
28
|
|
|
21
29
|
import { Type } from "typebox";
|
|
22
30
|
import { spawn } from "node:child_process";
|
|
23
|
-
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
24
|
-
import {
|
|
31
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
32
|
+
import { tmpdir } from "node:os";
|
|
33
|
+
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
25
34
|
|
|
26
35
|
/** Tool names this module registers, for allow-list construction. */
|
|
27
36
|
export const COMPOSE_TOOL_NAMES = ["video_compose"] as const;
|
|
@@ -32,7 +41,19 @@ const FFMPEG_TIMEOUT_MS = Number(process.env.PRIVATEER_FFMPEG_TIMEOUT_MS) || 10
|
|
|
32
41
|
// would eat the model's context.
|
|
33
42
|
const STDERR_TAIL_CHARS = 1200;
|
|
34
43
|
|
|
35
|
-
const OPERATIONS = [
|
|
44
|
+
const OPERATIONS = [
|
|
45
|
+
"probe",
|
|
46
|
+
"concat",
|
|
47
|
+
"slideshow",
|
|
48
|
+
"mux_audio",
|
|
49
|
+
"mix_audio",
|
|
50
|
+
"overlay_text",
|
|
51
|
+
"overlay_image",
|
|
52
|
+
"burn_subtitles",
|
|
53
|
+
"trim",
|
|
54
|
+
"extract_frame",
|
|
55
|
+
"gif",
|
|
56
|
+
] as const;
|
|
36
57
|
type Operation = (typeof OPERATIONS)[number];
|
|
37
58
|
|
|
38
59
|
function text(t: string) {
|
|
@@ -218,12 +239,87 @@ function normalizeVideo(index: number, label: string, w: number, h: number, fps:
|
|
|
218
239
|
);
|
|
219
240
|
}
|
|
220
241
|
|
|
242
|
+
const AFORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo";
|
|
243
|
+
|
|
221
244
|
function normalizeAudio(index: number, label: string): string {
|
|
222
|
-
return `[${index}:a]
|
|
245
|
+
return `[${index}:a]${AFORMAT},aresample=async=1[${label}]`;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// A number from the model, bounded and named. Every numeric parameter that reaches a
|
|
249
|
+
// filter string goes through here: it is both the input validation the model gets a
|
|
250
|
+
// useful message from, and the guarantee that nothing but a finite number is ever
|
|
251
|
+
// interpolated into a filtergraph.
|
|
252
|
+
function bounded(v: unknown, label: string, min: number, max: number, dflt: number): number {
|
|
253
|
+
if (v == null) return dflt;
|
|
254
|
+
const n = Number(v);
|
|
255
|
+
if (!Number.isFinite(n) || n < min || n > max) {
|
|
256
|
+
throw new ToolError(`\`${label}\` must be a number between ${min} and ${max} (got ${JSON.stringify(v)}).`);
|
|
257
|
+
}
|
|
258
|
+
return n;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// Colours reach ffmpeg as filter option values, so they are matched against a closed
|
|
262
|
+
// shape rather than passed through: a name, or #rrggbb(aa). Opacity is a separate
|
|
263
|
+
// numeric parameter, which is why `black@0.5` is not accepted here — it would be a
|
|
264
|
+
// second, unvalidated way to write a filter option.
|
|
265
|
+
const COLOR_RE = /^(?:#[0-9a-fA-F]{6}(?:[0-9a-fA-F]{2})?|[a-zA-Z]{3,20})$/;
|
|
266
|
+
|
|
267
|
+
function colorValue(v: string | undefined, dflt: string, label: string): string {
|
|
268
|
+
const c = String(v ?? dflt).trim();
|
|
269
|
+
if (!COLOR_RE.test(c)) {
|
|
270
|
+
throw new ToolError(`\`${label}\` must be a colour name (e.g. white) or #rrggbb (got ${JSON.stringify(v)}).`);
|
|
271
|
+
}
|
|
272
|
+
return c;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// Quote a path for use as a filter OPTION VALUE. Single quotes protect the separators
|
|
276
|
+
// the filtergraph parser would otherwise act on, and `\` and `:` are escaped inside them
|
|
277
|
+
// so a Windows path (C:\Users\…) survives — verified against ffmpeg 7 with a path
|
|
278
|
+
// containing a colon.
|
|
279
|
+
function filterPath(p: string): string {
|
|
280
|
+
return `'${p.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/:/g, "\\:")}'`;
|
|
223
281
|
}
|
|
224
282
|
|
|
225
283
|
// ── Operations ───────────────────────────────────────────────────────────────
|
|
226
284
|
|
|
285
|
+
/** One audio track placed on the timeline by `mix_audio`. */
|
|
286
|
+
interface MixTrack {
|
|
287
|
+
path: string;
|
|
288
|
+
atSeconds?: number;
|
|
289
|
+
gain?: number;
|
|
290
|
+
loop?: boolean;
|
|
291
|
+
duck?: boolean;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** How type is drawn — shared by an `overlay_text` cue and a whole subtitle file. */
|
|
295
|
+
interface TextStyle {
|
|
296
|
+
position?: string;
|
|
297
|
+
fontSize?: number;
|
|
298
|
+
color?: string;
|
|
299
|
+
box?: boolean;
|
|
300
|
+
boxColor?: string;
|
|
301
|
+
boxOpacity?: number;
|
|
302
|
+
marginPx?: number;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** One line of burnt-in type placed by `overlay_text`. */
|
|
306
|
+
interface TextCue extends TextStyle {
|
|
307
|
+
text: string;
|
|
308
|
+
fromSeconds?: number;
|
|
309
|
+
toSeconds?: number;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** One still composited onto the picture by `overlay_image`. */
|
|
313
|
+
interface ImageLayer {
|
|
314
|
+
path: string;
|
|
315
|
+
fromSeconds?: number;
|
|
316
|
+
toSeconds?: number;
|
|
317
|
+
position?: string;
|
|
318
|
+
widthPercent?: number;
|
|
319
|
+
opacity?: number;
|
|
320
|
+
marginPx?: number;
|
|
321
|
+
}
|
|
322
|
+
|
|
227
323
|
interface Params {
|
|
228
324
|
operation: Operation;
|
|
229
325
|
inputs?: string[];
|
|
@@ -240,6 +336,14 @@ interface Params {
|
|
|
240
336
|
volume?: number;
|
|
241
337
|
loopAudio?: boolean;
|
|
242
338
|
keepOriginalAudio?: boolean;
|
|
339
|
+
tracks?: (MixTrack | string)[];
|
|
340
|
+
voiceTrack?: number;
|
|
341
|
+
texts?: TextCue[];
|
|
342
|
+
fontFile?: string;
|
|
343
|
+
images?: (ImageLayer | string)[];
|
|
344
|
+
subtitles?: string;
|
|
345
|
+
style?: TextStyle;
|
|
346
|
+
maxCharsPerLine?: number;
|
|
243
347
|
}
|
|
244
348
|
|
|
245
349
|
async function opProbe(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
|
|
@@ -422,6 +526,618 @@ async function opMuxAudio(cwd: string, p: Params, signal?: AbortSignal): Promise
|
|
|
422
526
|
return `Muxed ${p.audio} onto ${p.input}${p.loopAudio ? " (looped)" : ""}${p.keepOriginalAudio ? " (mixed with the original audio)" : ""} → ${output} (${result.durationSec.toFixed(2)}s)`;
|
|
423
527
|
}
|
|
424
528
|
|
|
529
|
+
// Ducking: press the bed down while the narration is speaking, let it back up between
|
|
530
|
+
// lines. A compressor keyed off the voice does this on its own, which is why the level
|
|
531
|
+
// isn't a parameter — the alternative is the model authoring a volume envelope per line
|
|
532
|
+
// and re-authoring it every time a word changes.
|
|
533
|
+
//
|
|
534
|
+
// `ratio=12` with a low threshold is a firm duck rather than a gentle one (a marketing
|
|
535
|
+
// bed should get out of the way, not merely dip), and `release=350` is long enough that
|
|
536
|
+
// the bed doesn't pump between words. attack=15ms keeps the first syllable clear.
|
|
537
|
+
const DUCK_FILTER = "sidechaincompress=threshold=0.03:ratio=12:attack=15:release=350:detection=rms";
|
|
538
|
+
|
|
539
|
+
// Several audio tracks, each placed at a moment and at a level, mixed onto one video.
|
|
540
|
+
//
|
|
541
|
+
// WHY THIS EXISTS ALONGSIDE mux_audio. mux_audio lays ONE track over the whole video at
|
|
542
|
+
// one volume, which is the right tool for "put this narration on" and useless for a cut
|
|
543
|
+
// that has narration AND a bed AND effects that land on specific frames. Without
|
|
544
|
+
// placement an effect can only start at zero, and without ducking a bed loud enough to
|
|
545
|
+
// hear is loud enough to bury the voice — those two gaps are most of the distance
|
|
546
|
+
// between "clips with audio" and something that sounds finished.
|
|
547
|
+
async function opMixAudio(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
|
|
548
|
+
if (!p.input) throw new ToolError("`mix_audio` needs `input` (the video the tracks go onto).");
|
|
549
|
+
const raw = p.tracks ?? [];
|
|
550
|
+
if (raw.length === 0) {
|
|
551
|
+
throw new ToolError(
|
|
552
|
+
"`mix_audio` needs at least one entry in `tracks`, each with a `path` (plus optional atSeconds, gain, loop, duck). " +
|
|
553
|
+
"For one track at one level over the whole video, mux_audio is simpler.",
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
// A bare string is accepted as a track at 0s and full level: models write
|
|
557
|
+
// `tracks: ["vo.mp3"]` often enough that refusing it teaches nothing.
|
|
558
|
+
const tracks: MixTrack[] = raw.map((t, i) => {
|
|
559
|
+
const track = typeof t === "string" ? { path: t } : ((t ?? {}) as MixTrack);
|
|
560
|
+
if (!track.path) throw new ToolError(`tracks[${i}] has no \`path\`.`);
|
|
561
|
+
return track;
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
// Argument validation runs BEFORE anything touches the disk: told that `duck` needs a
|
|
565
|
+
// `voiceTrack`, a model fixes the call; told that a file it hasn't written yet is
|
|
566
|
+
// missing, it goes looking for the wrong problem.
|
|
567
|
+
const voice = p.voiceTrack;
|
|
568
|
+
if (voice != null && (!Number.isInteger(Number(voice)) || Number(voice) < 0 || Number(voice) >= tracks.length)) {
|
|
569
|
+
throw new ToolError(
|
|
570
|
+
`\`voiceTrack\` must be the index of one of the ${tracks.length} track(s) — 0 to ${tracks.length - 1} — got ${JSON.stringify(p.voiceTrack)}.`,
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
const ducked = tracks.map((t, i) => (t.duck && i !== voice ? i : -1)).filter((i) => i >= 0);
|
|
574
|
+
if (ducked.length > 0 && voice == null) {
|
|
575
|
+
throw new ToolError(
|
|
576
|
+
"`duck` needs `voiceTrack` set to the index of the track everything else ducks under (normally the narration) — " +
|
|
577
|
+
"there is nothing to duck under otherwise.",
|
|
578
|
+
);
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
const video = requireExisting(cwd, p.input, "video");
|
|
582
|
+
const paths = tracks.map((t, i) => requireExisting(cwd, t.path, `tracks[${i}]`));
|
|
583
|
+
const output = prepareOutput(cwd, p.output, "mix_audio");
|
|
584
|
+
|
|
585
|
+
const [vInfo, ...tInfos] = await Promise.all([probe(video, signal), ...paths.map((f) => probe(f, signal))]);
|
|
586
|
+
if (!vInfo.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
|
|
587
|
+
for (const [i, info] of tInfos.entries()) {
|
|
588
|
+
if (!info.hasAudio) throw new ToolError(`tracks[${i}] (${tracks[i].path}) has no audio stream.`);
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
const args: string[] = ["-i", video];
|
|
592
|
+
for (const [i, t] of tracks.entries()) {
|
|
593
|
+
if (t.loop) {
|
|
594
|
+
if (!(vInfo.durationSec > 0)) {
|
|
595
|
+
throw new ToolError(`\`loop\` needs a video of known length, and ${p.input} reports none.`);
|
|
596
|
+
}
|
|
597
|
+
// Bound the loop AT THE INPUT rather than trusting -shortest to cut an endless
|
|
598
|
+
// stream off inside the graph: amix waits on every input, so an unbounded one
|
|
599
|
+
// makes the encode run until the ffmpeg timeout fires.
|
|
600
|
+
args.push("-stream_loop", "-1", "-t", vInfo.durationSec.toFixed(3));
|
|
601
|
+
}
|
|
602
|
+
args.push("-i", paths[i]);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
const filters: string[] = [];
|
|
606
|
+
const labels: string[] = [];
|
|
607
|
+
if (p.keepOriginalAudio) {
|
|
608
|
+
if (!vInfo.hasAudio) throw new ToolError(`\`keepOriginalAudio\` was set but ${p.input} has no audio to keep.`);
|
|
609
|
+
filters.push(`[0:a]${AFORMAT}[aorig]`);
|
|
610
|
+
labels.push("aorig");
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
for (const [i, t] of tracks.entries()) {
|
|
614
|
+
const gain = bounded(t.gain, `tracks[${i}].gain`, 0, 4, 1);
|
|
615
|
+
const at = bounded(t.atSeconds, `tracks[${i}].atSeconds`, 0, 86_400, 0);
|
|
616
|
+
let chain = `[${i + 1}:a]${AFORMAT}`;
|
|
617
|
+
if (gain !== 1) chain += `,volume=${gain}`;
|
|
618
|
+
// adelay is what puts an effect on the frame it belongs to. `all=1` applies the
|
|
619
|
+
// delay to every channel — without it only the first channel moves, which sounds
|
|
620
|
+
// like a broken stereo image rather than a late cue.
|
|
621
|
+
if (at > 0) chain += `,adelay=${Math.round(at * 1000)}:all=1`;
|
|
622
|
+
filters.push(`${chain}[t${i}]`);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
const mixLabel = new Map<number, string>(tracks.map((_, i) => [i, `t${i}`]));
|
|
626
|
+
if (ducked.length > 0) {
|
|
627
|
+
// The voice feeds the mix AND every compressor's sidechain, so it has to be split
|
|
628
|
+
// that many ways: a filter output can only be consumed once.
|
|
629
|
+
filters.push(`[t${voice}]asplit=${ducked.length + 1}[vmix]${ducked.map((_, k) => `[key${k}]`).join("")}`);
|
|
630
|
+
mixLabel.set(voice as number, "vmix");
|
|
631
|
+
ducked.forEach((i, k) => {
|
|
632
|
+
filters.push(`[t${i}][key${k}]${DUCK_FILTER}[d${i}]`);
|
|
633
|
+
mixLabel.set(i, `d${i}`);
|
|
634
|
+
});
|
|
635
|
+
}
|
|
636
|
+
labels.push(...tracks.map((_, i) => mixLabel.get(i) as string));
|
|
637
|
+
|
|
638
|
+
// normalize=0 is load-bearing: amix's default divides every input by the number of
|
|
639
|
+
// inputs, so the gains asked for above would silently come out at a third of
|
|
640
|
+
// themselves as soon as a third track joined. The limiter is what keeps the sum from
|
|
641
|
+
// clipping instead.
|
|
642
|
+
if (labels.length > 1) {
|
|
643
|
+
filters.push(
|
|
644
|
+
`${labels.map((l) => `[${l}]`).join("")}amix=inputs=${labels.length}:duration=longest:dropout_transition=0:normalize=0[amixed]`,
|
|
645
|
+
);
|
|
646
|
+
} else {
|
|
647
|
+
filters.push(`[${labels[0]}]anull[amixed]`);
|
|
648
|
+
}
|
|
649
|
+
filters.push("[amixed]alimiter=limit=0.95[aout]");
|
|
650
|
+
|
|
651
|
+
args.push(
|
|
652
|
+
"-filter_complex", filters.join(";"),
|
|
653
|
+
"-map", "0:v", "-map", "[aout]",
|
|
654
|
+
// The picture is untouched, so it is copied: a mix must never be a reason to
|
|
655
|
+
// re-encode video and lose a generation of quality.
|
|
656
|
+
"-c:v", "copy", ...AUDIO_ENCODE, "-shortest", output,
|
|
657
|
+
);
|
|
658
|
+
await ffmpeg(args, signal);
|
|
659
|
+
|
|
660
|
+
const result = await probe(output, signal);
|
|
661
|
+
const placed = tracks
|
|
662
|
+
.map((t, i) => {
|
|
663
|
+
const bits: string[] = [t.path];
|
|
664
|
+
if (t.atSeconds) bits.push(`at ${Number(t.atSeconds)}s`);
|
|
665
|
+
if (t.gain != null && Number(t.gain) !== 1) bits.push(`gain ${Number(t.gain)}`);
|
|
666
|
+
if (t.loop) bits.push("looped");
|
|
667
|
+
if (i === voice) bits.push("voice");
|
|
668
|
+
else if (ducked.includes(i)) bits.push("ducked under the voice");
|
|
669
|
+
return ` ${bits.join(", ")}`;
|
|
670
|
+
})
|
|
671
|
+
.join("\n");
|
|
672
|
+
return (
|
|
673
|
+
`Mixed ${tracks.length} track(s) onto ${p.input}` +
|
|
674
|
+
`${p.keepOriginalAudio ? " over its own audio" : ""} → ${output} (${result.durationSec.toFixed(2)}s)\n${placed}`
|
|
675
|
+
);
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
// Burn type into the picture: a caption, a lower third, an end card.
|
|
679
|
+
//
|
|
680
|
+
// The text is written to a TEMPORARY FILE and read back by drawtext's `textfile=`
|
|
681
|
+
// rather than interpolated into the filtergraph as `text=`. That is not tidiness: a
|
|
682
|
+
// caption is model- or user-supplied prose, and `:`, `,`, `'`, `%`, `[` and `\` all mean
|
|
683
|
+
// something to the filter parser. Escaping them correctly through two levels of quoting
|
|
684
|
+
// is exactly the kind of thing that works until someone writes a price or a ratio in a
|
|
685
|
+
// caption, at which point the graph either fails or silently means something else.
|
|
686
|
+
// `expansion=none` closes the other half of the same hole — drawtext would otherwise
|
|
687
|
+
// evaluate `%{...}` sequences inside the text.
|
|
688
|
+
async function opOverlayText(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
|
|
689
|
+
if (!p.input) throw new ToolError("`overlay_text` needs `input` (the video to draw on).");
|
|
690
|
+
const cues = p.texts ?? [];
|
|
691
|
+
if (cues.length === 0) {
|
|
692
|
+
throw new ToolError(
|
|
693
|
+
"`overlay_text` needs at least one entry in `texts`, each with `text` (plus optional fromSeconds, toSeconds, " +
|
|
694
|
+
"position, fontSize, color, box).",
|
|
695
|
+
);
|
|
696
|
+
}
|
|
697
|
+
const input = requireExisting(cwd, p.input, "video");
|
|
698
|
+
const output = prepareOutput(cwd, p.output, "overlay_text");
|
|
699
|
+
const info = await probe(input, signal);
|
|
700
|
+
if (!info.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
|
|
701
|
+
|
|
702
|
+
const font = resolveFont(cwd, p.fontFile);
|
|
703
|
+
const tmp = mkdtempSync(join(tmpdir(), "pv-drawtext-"));
|
|
704
|
+
try {
|
|
705
|
+
const drawtexts = cues.map((cue, i) =>
|
|
706
|
+
buildDrawtext({ cue: cue ?? ({} as TextCue), index: i, label: `texts[${i}]`, font, info, tmp }),
|
|
707
|
+
);
|
|
708
|
+
await drawOnVideo(input, output, drawtexts, info, signal);
|
|
709
|
+
const result = await probe(output, signal);
|
|
710
|
+
return `Burned ${cues.length} text overlay(s) into ${p.input} → ${output} (${result.durationSec.toFixed(2)}s, ${result.width}x${result.height})`;
|
|
711
|
+
} finally {
|
|
712
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// One `drawtext` filter for one cue. Shared by overlay_text and burn_subtitles so a
|
|
717
|
+
// caption and a subtitle are styled, bounded and (above all) ESCAPED by the same code:
|
|
718
|
+
// the second copy of this is the one that would forget `expansion=none`.
|
|
719
|
+
function buildDrawtext(args: {
|
|
720
|
+
cue: TextCue;
|
|
721
|
+
index: number;
|
|
722
|
+
label: string;
|
|
723
|
+
font: string;
|
|
724
|
+
info: MediaInfo;
|
|
725
|
+
tmp: string;
|
|
726
|
+
defaults?: { position?: string; box?: boolean };
|
|
727
|
+
}): string {
|
|
728
|
+
const { cue, index, label, font, info, tmp } = args;
|
|
729
|
+
const body = String(cue.text ?? "");
|
|
730
|
+
if (!body.trim()) throw new ToolError(`${label} has no \`text\`.`);
|
|
731
|
+
// Default type size scales with the frame: a 48px caption is right on 1080p and
|
|
732
|
+
// unreadable on a 4K master, and the model has no reliable idea which it holds.
|
|
733
|
+
const height = info.height ?? 1080;
|
|
734
|
+
const fontSize = Math.round(bounded(cue.fontSize, `${label}.fontSize`, 8, 400, Math.max(16, Math.round(height / 20))));
|
|
735
|
+
const margin = Math.round(bounded(cue.marginPx, `${label}.marginPx`, 0, 2000, Math.round(fontSize * 0.8)));
|
|
736
|
+
const { x, y } = positionExpr(cue.position ?? args.defaults?.position ?? "bottom", margin, label);
|
|
737
|
+
const opts = [
|
|
738
|
+
`fontfile=${filterPath(font)}`,
|
|
739
|
+
`textfile=${filterPath(writeCueFile(tmp, index, body))}`,
|
|
740
|
+
"expansion=none",
|
|
741
|
+
`fontsize=${fontSize}`,
|
|
742
|
+
`fontcolor=${colorValue(cue.color, "white", `${label}.color`)}`,
|
|
743
|
+
`x=${x}`,
|
|
744
|
+
`y=${y}`,
|
|
745
|
+
];
|
|
746
|
+
if (cue.box ?? args.defaults?.box) {
|
|
747
|
+
const opacity = bounded(cue.boxOpacity, `${label}.boxOpacity`, 0, 1, 0.5);
|
|
748
|
+
opts.push("box=1", `boxcolor=${colorValue(cue.boxColor, "black", `${label}.boxColor`)}@${opacity}`, `boxborderw=${Math.round(fontSize / 3)}`);
|
|
749
|
+
} else {
|
|
750
|
+
// No box means the type has to hold against whatever is behind it, and generated
|
|
751
|
+
// footage is rarely obligingly dark. A border is cheaper than a box and reads
|
|
752
|
+
// as design rather than as a subtitle.
|
|
753
|
+
opts.push(`borderw=${Math.max(1, Math.round(fontSize / 16))}`, "bordercolor=black@0.85");
|
|
754
|
+
}
|
|
755
|
+
const from = cue.fromSeconds == null ? null : bounded(cue.fromSeconds, `${label}.fromSeconds`, 0, 86_400, 0);
|
|
756
|
+
const to = cue.toSeconds == null ? null : bounded(cue.toSeconds, `${label}.toSeconds`, 0, 86_400, 0);
|
|
757
|
+
if (from != null && to != null && to <= from) {
|
|
758
|
+
throw new ToolError(`${label}: toSeconds (${to}) must be after fromSeconds (${from}).`);
|
|
759
|
+
}
|
|
760
|
+
// Absent bounds mean "for the whole video", which is what omitting `enable` does.
|
|
761
|
+
if (from != null || to != null) {
|
|
762
|
+
opts.push(`enable='between(t,${from ?? 0},${to ?? Math.max(from ?? 0, info.durationSec || 86_400)})'`);
|
|
763
|
+
}
|
|
764
|
+
return `drawtext=${opts.join(":")}`;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
// Run a chain of video filters over the picture and leave the sound alone. The picture is
|
|
768
|
+
// re-encoded (it has to be — we are drawing on it) and the audio is copied through
|
|
769
|
+
// untouched, so adding a caption never costs a generation of sound.
|
|
770
|
+
async function drawOnVideo(
|
|
771
|
+
input: string,
|
|
772
|
+
output: string,
|
|
773
|
+
filters: string[],
|
|
774
|
+
info: MediaInfo,
|
|
775
|
+
signal?: AbortSignal,
|
|
776
|
+
): Promise<void> {
|
|
777
|
+
await ffmpeg(
|
|
778
|
+
[
|
|
779
|
+
"-i", input,
|
|
780
|
+
"-filter_complex", `[0:v]${filters.join(",")}[vout]`,
|
|
781
|
+
"-map", "[vout]",
|
|
782
|
+
...(info.hasAudio ? ["-map", "0:a", "-c:a", "copy"] : []),
|
|
783
|
+
...VIDEO_ENCODE,
|
|
784
|
+
output,
|
|
785
|
+
],
|
|
786
|
+
signal,
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
function writeCueFile(dir: string, index: number, body: string): string {
|
|
791
|
+
const file = join(dir, `cue-${index}.txt`);
|
|
792
|
+
writeFileSync(file, body, "utf8");
|
|
793
|
+
return file;
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
const POSITIONS = ["top-left", "top", "top-right", "left", "center", "right", "bottom-left", "bottom", "bottom-right"] as const;
|
|
797
|
+
|
|
798
|
+
// The frame/element variable names a filter exposes for placement. drawtext measures the
|
|
799
|
+
// frame as w/h and the drawn text as text_w/text_h; overlay measures the MAIN input as
|
|
800
|
+
// W/H and the overlaid one as w/h. Same nine positions, two vocabularies.
|
|
801
|
+
const DRAWTEXT_VARS = { frameW: "w", frameH: "h", itemW: "text_w", itemH: "text_h" };
|
|
802
|
+
const OVERLAY_VARS = { frameW: "W", frameH: "H", itemW: "w", itemH: "h" };
|
|
803
|
+
|
|
804
|
+
// Named positions only — never a caller-supplied x/y expression. Both filters evaluate x
|
|
805
|
+
// and y as arithmetic over frame variables, so accepting one would hand the model (or
|
|
806
|
+
// anything that reached it) an expression evaluator inside the filtergraph. The nine names
|
|
807
|
+
// below cover every placement a title, caption or logo actually wants.
|
|
808
|
+
function positionExpr(
|
|
809
|
+
position: string,
|
|
810
|
+
margin: number,
|
|
811
|
+
label: string,
|
|
812
|
+
vars: typeof DRAWTEXT_VARS = DRAWTEXT_VARS,
|
|
813
|
+
): { x: string; y: string } {
|
|
814
|
+
const name = String(position).trim().toLowerCase();
|
|
815
|
+
if (!(POSITIONS as readonly string[]).includes(name)) {
|
|
816
|
+
throw new ToolError(`${label}.position must be one of: ${POSITIONS.join(", ")} (got ${JSON.stringify(position)}).`);
|
|
817
|
+
}
|
|
818
|
+
const y = name.startsWith("top")
|
|
819
|
+
? String(margin)
|
|
820
|
+
: name.startsWith("bottom")
|
|
821
|
+
? `${vars.frameH}-${vars.itemH}-${margin}`
|
|
822
|
+
: `(${vars.frameH}-${vars.itemH})/2`;
|
|
823
|
+
const x = name.endsWith("left")
|
|
824
|
+
? String(margin)
|
|
825
|
+
: name.endsWith("right")
|
|
826
|
+
? `${vars.frameW}-${vars.itemW}-${margin}`
|
|
827
|
+
: `(${vars.frameW}-${vars.itemW})/2`;
|
|
828
|
+
return { x, y };
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// Where to find a font, in order: the one the caller named, then the platform's usual
|
|
832
|
+
// suspects. drawtext with neither `fontfile` nor `font` only works on a build with
|
|
833
|
+
// fontconfig, and the failure ("Cannot find a valid font...") arrives after the encode
|
|
834
|
+
// starts — so a path is resolved here, up front, where the message can name the fix.
|
|
835
|
+
const FONT_CANDIDATES: Record<string, string[]> = {
|
|
836
|
+
darwin: [
|
|
837
|
+
"/System/Library/Fonts/Helvetica.ttc",
|
|
838
|
+
"/System/Library/Fonts/Supplemental/Arial.ttf",
|
|
839
|
+
"/Library/Fonts/Arial.ttf",
|
|
840
|
+
],
|
|
841
|
+
linux: [
|
|
842
|
+
"/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
|
|
843
|
+
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
|
|
844
|
+
"/usr/share/fonts/TTF/DejaVuSans.ttf",
|
|
845
|
+
"/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
|
|
846
|
+
"/usr/share/fonts/liberation-sans/LiberationSans-Bold.ttf",
|
|
847
|
+
],
|
|
848
|
+
win32: ["C:\\Windows\\Fonts\\arialbd.ttf", "C:\\Windows\\Fonts\\arial.ttf", "C:\\Windows\\Fonts\\segoeui.ttf"],
|
|
849
|
+
};
|
|
850
|
+
|
|
851
|
+
function resolveFont(cwd: string, fontFile?: string): string {
|
|
852
|
+
if (fontFile) return requireExisting(cwd, fontFile, "fontFile");
|
|
853
|
+
for (const candidate of FONT_CANDIDATES[process.platform] ?? FONT_CANDIDATES.linux) {
|
|
854
|
+
if (existsSync(candidate)) return candidate;
|
|
855
|
+
}
|
|
856
|
+
throw new ToolError(
|
|
857
|
+
"no system font found to draw with — pass `fontFile` with the path to a .ttf/.otf " +
|
|
858
|
+
`(looked for ${(FONT_CANDIDATES[process.platform] ?? FONT_CANDIDATES.linux).join(", ")}).`,
|
|
859
|
+
);
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
// Composite stills onto the picture: the logo bug in the corner, an end card, a lower-third
|
|
863
|
+
// plate a caption then sits on. The move overlay_text can't make — type is not artwork, and
|
|
864
|
+
// a brand mark drawn as text is a brand mark drawn wrong.
|
|
865
|
+
async function opOverlayImage(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
|
|
866
|
+
if (!p.input) throw new ToolError("`overlay_image` needs `input` (the video to draw on).");
|
|
867
|
+
const raw = p.images ?? [];
|
|
868
|
+
if (raw.length === 0) {
|
|
869
|
+
throw new ToolError(
|
|
870
|
+
"`overlay_image` needs at least one entry in `images`, each with a `path` (plus optional fromSeconds, " +
|
|
871
|
+
"toSeconds, position, widthPercent, opacity, marginPx).",
|
|
872
|
+
);
|
|
873
|
+
}
|
|
874
|
+
// A bare string is a full-strength overlay at the default position, for the same reason
|
|
875
|
+
// mix_audio accepts one: `images: ["logo.png"]` is what a model writes first.
|
|
876
|
+
const layers: ImageLayer[] = raw.map((entry, i) => {
|
|
877
|
+
const layer = typeof entry === "string" ? { path: entry } : ((entry ?? {}) as ImageLayer);
|
|
878
|
+
if (!layer.path) throw new ToolError(`images[${i}] has no \`path\`.`);
|
|
879
|
+
return layer;
|
|
880
|
+
});
|
|
881
|
+
|
|
882
|
+
// Everything checkable without the disk is checked first, as in mix_audio: told that
|
|
883
|
+
// `position` isn't one of the nine, a model fixes the call; told that a file it is about
|
|
884
|
+
// to write doesn't exist yet, it goes looking for the wrong problem. (The numeric bounds
|
|
885
|
+
// below are re-applied where they are used — a second call to `bounded` with the same
|
|
886
|
+
// value is free, and one shared place to validate would need the probe results.)
|
|
887
|
+
layers.forEach((layer, i) => {
|
|
888
|
+
const label = `images[${i}]`;
|
|
889
|
+
positionExpr(layer.position ?? "top-right", 0, label, OVERLAY_VARS);
|
|
890
|
+
bounded(layer.opacity, `${label}.opacity`, 0, 1, 1);
|
|
891
|
+
bounded(layer.widthPercent, `${label}.widthPercent`, 1, 100, 100);
|
|
892
|
+
const from = bounded(layer.fromSeconds, `${label}.fromSeconds`, 0, 86_400, 0);
|
|
893
|
+
const to = layer.toSeconds == null ? null : bounded(layer.toSeconds, `${label}.toSeconds`, 0, 86_400, 0);
|
|
894
|
+
if (layer.fromSeconds != null && to != null && to <= from) {
|
|
895
|
+
throw new ToolError(`${label}: toSeconds (${to}) must be after fromSeconds (${from}).`);
|
|
896
|
+
}
|
|
897
|
+
});
|
|
898
|
+
|
|
899
|
+
const input = requireExisting(cwd, p.input, "video");
|
|
900
|
+
const paths = layers.map((l, i) => requireExisting(cwd, l.path, `images[${i}]`));
|
|
901
|
+
const output = prepareOutput(cwd, p.output, "overlay_image");
|
|
902
|
+
const [info, ...layerInfos] = await Promise.all([probe(input, signal), ...paths.map((f) => probe(f, signal))]);
|
|
903
|
+
if (!info.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
|
|
904
|
+
for (const [i, li] of layerInfos.entries()) {
|
|
905
|
+
if (!li.hasVideo) throw new ToolError(`images[${i}] (${layers[i].path}) is not an image or video ffmpeg can read.`);
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
const frameW = info.width ?? 1280;
|
|
909
|
+
const args: string[] = ["-i", input];
|
|
910
|
+
for (const [i, li] of layerInfos.entries()) {
|
|
911
|
+
// A still decodes to ONE frame, which composited over a 30s video appears on frame 1
|
|
912
|
+
// and is gone — so a still is looped for the video's length, while an animated overlay
|
|
913
|
+
// (a GIF, a video sting) plays at its own pace and `eof_action=pass` below lets the
|
|
914
|
+
// film carry on after it ends.
|
|
915
|
+
//
|
|
916
|
+
// DURATION is what distinguishes them, not frame rate: the image demuxer fabricates
|
|
917
|
+
// `avg_frame_rate=25/1` for a PNG, so keying off fps silently skipped the loop and
|
|
918
|
+
// produced a video whose logo flashed on the first frame only.
|
|
919
|
+
if (li.durationSec === 0 && info.durationSec > 0) args.push("-loop", "1", "-t", info.durationSec.toFixed(3));
|
|
920
|
+
args.push("-i", paths[i]);
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
const filters: string[] = [];
|
|
924
|
+
const scaledTo: (number | null)[] = [];
|
|
925
|
+
for (const [i, layer] of layers.entries()) {
|
|
926
|
+
const label = `images[${i}]`;
|
|
927
|
+
const pct = layer.widthPercent == null ? null : bounded(layer.widthPercent, `${label}.widthPercent`, 1, 100, 100);
|
|
928
|
+
const nativeW = layerInfos[i].width ?? frameW;
|
|
929
|
+
// No widthPercent means "as authored" — EXCEPT when the art is wider than the frame,
|
|
930
|
+
// where leaving it alone silently crops the overlay off the right-hand side and reads
|
|
931
|
+
// as a broken render rather than as a choice.
|
|
932
|
+
const targetW = pct != null ? Math.round((frameW * pct) / 100) : nativeW > frameW ? frameW : null;
|
|
933
|
+
scaledTo.push(targetW);
|
|
934
|
+
const chain: string[] = [];
|
|
935
|
+
if (targetW != null) chain.push(`scale=${Math.max(2, targetW - (targetW % 2))}:-1`);
|
|
936
|
+
const opacity = bounded(layer.opacity, `${label}.opacity`, 0, 1, 1);
|
|
937
|
+
// A watermark has to let the picture through. colorchannelmixer SCALES the alpha
|
|
938
|
+
// channel, so `format=rgba` first: art saved without transparency has no alpha to
|
|
939
|
+
// scale, and the layer would come out fully opaque with no error.
|
|
940
|
+
if (opacity < 1) chain.push("format=rgba", `colorchannelmixer=aa=${opacity}`);
|
|
941
|
+
filters.push(`[${i + 1}:v]${chain.length > 0 ? chain.join(",") : "null"}[img${i}]`);
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
let vPrev = "0:v";
|
|
945
|
+
layers.forEach((layer, i) => {
|
|
946
|
+
const label = `images[${i}]`;
|
|
947
|
+
const margin = Math.round(bounded(layer.marginPx, `${label}.marginPx`, 0, 2000, Math.round(frameW * 0.03)));
|
|
948
|
+
const { x, y } = positionExpr(layer.position ?? "top-right", margin, label, OVERLAY_VARS);
|
|
949
|
+
const from = layer.fromSeconds == null ? null : bounded(layer.fromSeconds, `${label}.fromSeconds`, 0, 86_400, 0);
|
|
950
|
+
const to = layer.toSeconds == null ? null : bounded(layer.toSeconds, `${label}.toSeconds`, 0, 86_400, 0);
|
|
951
|
+
if (from != null && to != null && to <= from) {
|
|
952
|
+
throw new ToolError(`${label}: toSeconds (${to}) must be after fromSeconds (${from}).`);
|
|
953
|
+
}
|
|
954
|
+
const opts = [`x=${x}`, `y=${y}`];
|
|
955
|
+
if (from != null || to != null) {
|
|
956
|
+
opts.push(`enable='between(t,${from ?? 0},${to ?? Math.max(from ?? 0, info.durationSec || 86_400)})'`);
|
|
957
|
+
}
|
|
958
|
+
// `eof_action=pass` keeps the MAIN video running once a layer's own frames end (a
|
|
959
|
+
// windowed overlay, or art shorter than the video): the default would end the output
|
|
960
|
+
// there, truncating the film to the length of its logo.
|
|
961
|
+
opts.push("eof_action=pass");
|
|
962
|
+
const next = i === layers.length - 1 ? "vout" : `vo${i}`;
|
|
963
|
+
filters.push(`[${vPrev}][img${i}]overlay=${opts.join(":")}[${next}]`);
|
|
964
|
+
vPrev = next;
|
|
965
|
+
});
|
|
966
|
+
|
|
967
|
+
await ffmpeg(
|
|
968
|
+
[
|
|
969
|
+
...args,
|
|
970
|
+
"-filter_complex", filters.join(";"),
|
|
971
|
+
"-map", "[vout]",
|
|
972
|
+
...(info.hasAudio ? ["-map", "0:a", "-c:a", "copy"] : []),
|
|
973
|
+
...VIDEO_ENCODE,
|
|
974
|
+
output,
|
|
975
|
+
],
|
|
976
|
+
signal,
|
|
977
|
+
);
|
|
978
|
+
const result = await probe(output, signal);
|
|
979
|
+
const placed = layers
|
|
980
|
+
.map((l, i) => {
|
|
981
|
+
const bits = [l.path, l.position ?? "top-right"];
|
|
982
|
+
if (scaledTo[i] != null) bits.push(`${scaledTo[i]}px wide`);
|
|
983
|
+
if (l.opacity != null && Number(l.opacity) < 1) bits.push(`opacity ${Number(l.opacity)}`);
|
|
984
|
+
if (l.fromSeconds != null || l.toSeconds != null) bits.push(`${l.fromSeconds ?? 0}s-${l.toSeconds ?? "end"}`);
|
|
985
|
+
return ` ${bits.join(", ")}`;
|
|
986
|
+
})
|
|
987
|
+
.join("\n");
|
|
988
|
+
return `Composited ${layers.length} image(s) onto ${p.input} → ${output} (${result.durationSec.toFixed(2)}s, ${result.width}x${result.height})\n${placed}`;
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
// A graph with a drawtext per cue is the cost of this approach; bound it rather than
|
|
992
|
+
// building a filter string megabytes long and letting ffmpeg fail on argv length.
|
|
993
|
+
const MAX_SUBTITLE_CUES = 500;
|
|
994
|
+
// The broadcast convention, and about what fits at frame-height/20 on a 16:9 master.
|
|
995
|
+
const DEFAULT_SUBTITLE_CHARS = 42;
|
|
996
|
+
|
|
997
|
+
// Burn a subtitle file into the picture.
|
|
998
|
+
//
|
|
999
|
+
// WHY WE PARSE IT OURSELVES rather than handing ffmpeg's `subtitles=` filter the file.
|
|
1000
|
+
// That filter needs libass, which is a build option and not present in every ffmpeg a user
|
|
1001
|
+
// has installed — and its styling goes through `force_style`, a comma-separated string
|
|
1002
|
+
// inside a filter option value, which is a second quoting layer to get wrong on exactly the
|
|
1003
|
+
// kind of caller-supplied text this file is careful about everywhere else. Parsing SRT/VTT
|
|
1004
|
+
// here and emitting the same drawtext cues overlay_text uses means one text path, already
|
|
1005
|
+
// escaped (textfile= + expansion=none), and no dependency beyond drawtext itself.
|
|
1006
|
+
//
|
|
1007
|
+
// What that costs, stated plainly: no italics, no per-cue positioning overrides, no
|
|
1008
|
+
// karaoke timing. Styling is uniform across the file, which is what burnt-in captions on a
|
|
1009
|
+
// marketing cut want anyway.
|
|
1010
|
+
async function opBurnSubtitles(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
|
|
1011
|
+
if (!p.input) throw new ToolError("`burn_subtitles` needs `input` (the video to draw on).");
|
|
1012
|
+
if (!p.subtitles) throw new ToolError("`burn_subtitles` needs `subtitles` (the path to an .srt or .vtt file).");
|
|
1013
|
+
const wrapAt = Math.round(bounded(p.maxCharsPerLine, "maxCharsPerLine", 10, 200, DEFAULT_SUBTITLE_CHARS));
|
|
1014
|
+
const input = requireExisting(cwd, p.input, "video");
|
|
1015
|
+
const subs = requireExisting(cwd, p.subtitles, "subtitles");
|
|
1016
|
+
const output = prepareOutput(cwd, p.output, "burn_subtitles");
|
|
1017
|
+
const info = await probe(input, signal);
|
|
1018
|
+
if (!info.hasVideo) throw new ToolError(`${p.input} has no video stream.`);
|
|
1019
|
+
|
|
1020
|
+
const parsed = parseSubtitleCues(readFileSync(subs, "utf8"));
|
|
1021
|
+
if (parsed.length === 0) {
|
|
1022
|
+
throw new ToolError(
|
|
1023
|
+
`no cues found in ${p.subtitles} — expected SubRip (.srt) or WebVTT (.vtt), where each cue is a ` +
|
|
1024
|
+
"`00:00:01,000 --> 00:00:04,000` line followed by its text.",
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
if (parsed.length > MAX_SUBTITLE_CUES) {
|
|
1028
|
+
throw new ToolError(
|
|
1029
|
+
`${p.subtitles} has ${parsed.length} cues, over the ${MAX_SUBTITLE_CUES} this can burn in one pass. ` +
|
|
1030
|
+
"Trim the video and its subtitles into sections and burn each, or use fewer, longer cues.",
|
|
1031
|
+
);
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
const style = (p.style ?? {}) as TextStyle;
|
|
1035
|
+
const font = resolveFont(cwd, p.fontFile);
|
|
1036
|
+
const tmp = mkdtempSync(join(tmpdir(), "pv-subtitles-"));
|
|
1037
|
+
try {
|
|
1038
|
+
const drawtexts = parsed.map((cue, i) =>
|
|
1039
|
+
buildDrawtext({
|
|
1040
|
+
cue: { ...style, text: wrapLines(cue.text, wrapAt), fromSeconds: cue.from, toSeconds: cue.to },
|
|
1041
|
+
index: i,
|
|
1042
|
+
label: "style",
|
|
1043
|
+
font,
|
|
1044
|
+
info,
|
|
1045
|
+
tmp,
|
|
1046
|
+
// Captions live at the bottom, and against generated footage they need a plate
|
|
1047
|
+
// more often than a title does — so a box is the default here where overlay_text
|
|
1048
|
+
// defaults to an outline. `style.box: false` turns it back off.
|
|
1049
|
+
defaults: { position: "bottom", box: true },
|
|
1050
|
+
}),
|
|
1051
|
+
);
|
|
1052
|
+
await drawOnVideo(input, output, drawtexts, info, signal);
|
|
1053
|
+
const result = await probe(output, signal);
|
|
1054
|
+
const last = parsed[parsed.length - 1];
|
|
1055
|
+
return (
|
|
1056
|
+
`Burned ${parsed.length} subtitle cue(s) from ${p.subtitles} into ${p.input} → ${output} ` +
|
|
1057
|
+
`(${result.durationSec.toFixed(2)}s, wrapped at ${wrapAt} characters, last cue ends at ${last.to.toFixed(2)}s)` +
|
|
1058
|
+
(last.to > info.durationSec + 0.5
|
|
1059
|
+
? `\nNote: the subtitles run ${(last.to - info.durationSec).toFixed(2)}s past the end of the video — ` +
|
|
1060
|
+
"those cues will never be seen."
|
|
1061
|
+
: "")
|
|
1062
|
+
);
|
|
1063
|
+
} finally {
|
|
1064
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
interface ParsedCue {
|
|
1069
|
+
from: number;
|
|
1070
|
+
to: number;
|
|
1071
|
+
text: string;
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
// "00:00:01,000", "00:00:01.000", "01:02.500" (VTT drops the hour) → seconds.
|
|
1075
|
+
function parseTimecode(raw: string): number | null {
|
|
1076
|
+
const m = /^(?:(\d{1,3}):)?(\d{1,2}):(\d{1,2})(?:[.,](\d{1,3}))?$/.exec(raw.trim());
|
|
1077
|
+
if (!m) return null;
|
|
1078
|
+
const ms = m[4] ? Number(m[4].padEnd(3, "0")) : 0;
|
|
1079
|
+
return Number(m[1] ?? 0) * 3600 + Number(m[2]) * 60 + Number(m[3]) + ms / 1000;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
// Inline markup we cannot render and must not print: SRT/VTT tags (<i>, <c.yellow>) and
|
|
1083
|
+
// ASS override blocks ({\an8}). Dropped rather than escaped — a caption reading "<i>" is
|
|
1084
|
+
// worse than one that lost its italics.
|
|
1085
|
+
function stripMarkup(s: string): string {
|
|
1086
|
+
return s.replace(/<[^>\n]*>/g, "").replace(/\{\\[^}\n]*\}/g, "");
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// Cue-block parser covering both formats, because they differ only in the separator inside
|
|
1090
|
+
// the timestamp and in a header/settings line we can ignore. Anything that isn't a cue —
|
|
1091
|
+
// SubRip's sequence numbers, `WEBVTT`, `NOTE` blocks, styling blocks — is skipped by
|
|
1092
|
+
// keying off the `-->` line rather than by counting lines, which is what makes a
|
|
1093
|
+
// hand-edited file with an extra blank line parse rather than derail.
|
|
1094
|
+
export function parseSubtitleCues(body: string): ParsedCue[] {
|
|
1095
|
+
const lines = body.replace(/^/, "").replace(/\r\n?/g, "\n").split("\n");
|
|
1096
|
+
const cues: ParsedCue[] = [];
|
|
1097
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1098
|
+
const arrow = lines[i].indexOf("-->");
|
|
1099
|
+
if (arrow === -1) continue;
|
|
1100
|
+
const from = parseTimecode(lines[i].slice(0, arrow));
|
|
1101
|
+
// WebVTT cue settings ("align:start line:90%") follow the end time on the same line.
|
|
1102
|
+
const to = parseTimecode(lines[i].slice(arrow + 3).trim().split(/\s+/)[0] ?? "");
|
|
1103
|
+
if (from == null || to == null || to <= from) continue;
|
|
1104
|
+
const text: string[] = [];
|
|
1105
|
+
let j = i + 1;
|
|
1106
|
+
for (; j < lines.length && lines[j].trim() !== ""; j++) text.push(lines[j]);
|
|
1107
|
+
i = j;
|
|
1108
|
+
const clean = stripMarkup(text.join("\n"))
|
|
1109
|
+
.split("\n")
|
|
1110
|
+
.map((l) => l.trim())
|
|
1111
|
+
.filter(Boolean)
|
|
1112
|
+
.join("\n");
|
|
1113
|
+
if (clean) cues.push({ from, to, text: clean });
|
|
1114
|
+
}
|
|
1115
|
+
return cues;
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
// Wrap to a character count, per authored line. drawtext does not wrap, so a cue written as
|
|
1119
|
+
// one long line would run off both edges of the frame. Existing line breaks are kept — a
|
|
1120
|
+
// subtitler who split a line meant it — and only over-long lines are broken further.
|
|
1121
|
+
export function wrapLines(text: string, max: number): string {
|
|
1122
|
+
return text
|
|
1123
|
+
.split("\n")
|
|
1124
|
+
.map((line) => {
|
|
1125
|
+
const out: string[] = [];
|
|
1126
|
+
let current = "";
|
|
1127
|
+
for (const word of line.split(/\s+/).filter(Boolean)) {
|
|
1128
|
+
if (current === "") current = word;
|
|
1129
|
+
else if (current.length + 1 + word.length <= max) current += ` ${word}`;
|
|
1130
|
+
else {
|
|
1131
|
+
out.push(current);
|
|
1132
|
+
current = word;
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
if (current) out.push(current);
|
|
1136
|
+
return out.join("\n");
|
|
1137
|
+
})
|
|
1138
|
+
.join("\n");
|
|
1139
|
+
}
|
|
1140
|
+
|
|
425
1141
|
async function opTrim(cwd: string, p: Params, signal?: AbortSignal): Promise<string> {
|
|
426
1142
|
if (!p.input) throw new ToolError("`trim` needs `input`.");
|
|
427
1143
|
const input = requireExisting(cwd, p.input, "input");
|
|
@@ -511,7 +1227,28 @@ export const videoComposeToolDefinition = {
|
|
|
511
1227
|
" slideshow — inputs (images), output, optional secondsPerImage, crossfadeSeconds, size, fps: " +
|
|
512
1228
|
"turn stills into a video. Output is silent; add sound with mux_audio.\n" +
|
|
513
1229
|
" mux_audio — input (video), audio, output, optional loopAudio, volume, keepOriginalAudio: put " +
|
|
514
|
-
"
|
|
1230
|
+
"ONE track onto a video at one level, trimmed to the video's length.\n" +
|
|
1231
|
+
" mix_audio — input (video), tracks, output, optional voiceTrack, keepOriginalAudio: put SEVERAL " +
|
|
1232
|
+
"tracks on at once, each placed and levelled. Every track takes `path` plus optional `atSeconds` (when " +
|
|
1233
|
+
"it starts — this is how a sound effect lands on the frame it belongs to), `gain` (0-4), `loop` (repeat " +
|
|
1234
|
+
"a short bed to the end) and `duck` (press this track down while the narration speaks). Set " +
|
|
1235
|
+
"`voiceTrack` to the index of the narration for `duck` to key off. Use this rather than several " +
|
|
1236
|
+
"mux_audio passes: each pass re-encodes the sound and mixes blind to what comes next.\n" +
|
|
1237
|
+
" overlay_text — input (video), texts, output, optional fontFile: burn type into the picture. Each " +
|
|
1238
|
+
"entry takes `text` plus optional `fromSeconds`/`toSeconds` (when it shows), `position` (one of " +
|
|
1239
|
+
"top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right), `fontSize` " +
|
|
1240
|
+
"(defaults to the frame height / 20), `color`, `box`, `boxColor`, `boxOpacity`, `marginPx`. Audio is " +
|
|
1241
|
+
"copied through untouched.\n" +
|
|
1242
|
+
" overlay_image — input (video), images, output: composite stills onto the picture — a logo bug, an " +
|
|
1243
|
+
"end card, a lower-third plate. Each entry takes `path` plus optional `fromSeconds`/`toSeconds`, " +
|
|
1244
|
+
"`position` (same nine names, defaults to top-right), `widthPercent` (of the frame width — the way to " +
|
|
1245
|
+
"size a logo, since the art's own pixel size means nothing), `opacity` (0-1, for a watermark) and " +
|
|
1246
|
+
"`marginPx`. Audio is copied through untouched.\n" +
|
|
1247
|
+
" burn_subtitles — input (video), subtitles (.srt or .vtt), output, optional style, maxCharsPerLine, " +
|
|
1248
|
+
"fontFile: burn a whole subtitle file in, timed from the file itself. Lines are wrapped to fit (42 " +
|
|
1249
|
+
"characters by default) and drawn at the bottom on a plate; `style` takes the same fields as an " +
|
|
1250
|
+
"overlay_text entry and applies to every cue. Needs no libass — italics and per-cue positioning in the " +
|
|
1251
|
+
"file are dropped, everything else is honoured.\n" +
|
|
515
1252
|
" trim — input, output, start, optional duration: cut a section out, frame-accurate.\n" +
|
|
516
1253
|
" extract_frame — input, output, optional at (seconds, or \"last\"): pull one still. Extract the " +
|
|
517
1254
|
"last frame of a clip and pass it to generate_video as `firstFrame` to keep consecutive clips " +
|
|
@@ -520,7 +1257,8 @@ export const videoComposeToolDefinition = {
|
|
|
520
1257
|
"Needs ffmpeg installed; every operation says so plainly if it isn't.",
|
|
521
1258
|
parameters: Type.Object({
|
|
522
1259
|
operation: Type.String({
|
|
523
|
-
description:
|
|
1260
|
+
description:
|
|
1261
|
+
'One of: "probe", "concat", "slideshow", "mux_audio", "mix_audio", "overlay_text", "overlay_image", "burn_subtitles", "trim", "extract_frame", "gif".',
|
|
524
1262
|
}),
|
|
525
1263
|
inputs: Type.Optional(Type.Array(Type.String(), { description: "Input paths, in order. Used by concat, slideshow, and probe." })),
|
|
526
1264
|
input: Type.Optional(Type.String({ description: "A single input path. Used by probe, mux_audio (the video), trim, extract_frame, gif." })),
|
|
@@ -535,7 +1273,83 @@ export const videoComposeToolDefinition = {
|
|
|
535
1273
|
at: Type.Optional(Type.Union([Type.Number(), Type.String()], { description: 'Timestamp for extract_frame: seconds, or "last" for the final frame. Defaults to 0.' })),
|
|
536
1274
|
volume: Type.Optional(Type.Number({ description: "Volume multiplier for the added track, 0-4. mux_audio only; defaults to 1." })),
|
|
537
1275
|
loopAudio: Type.Optional(Type.Boolean({ description: "Repeat a short audio track until the video ends. mux_audio only." })),
|
|
538
|
-
keepOriginalAudio: Type.Optional(Type.Boolean({ description: "Mix under the video's existing audio instead of replacing it. mux_audio
|
|
1276
|
+
keepOriginalAudio: Type.Optional(Type.Boolean({ description: "Mix under the video's existing audio instead of replacing it. mux_audio and mix_audio." })),
|
|
1277
|
+
tracks: Type.Optional(
|
|
1278
|
+
Type.Array(
|
|
1279
|
+
Type.Object({
|
|
1280
|
+
path: Type.String({ description: "The audio file." }),
|
|
1281
|
+
atSeconds: Type.Optional(Type.Number({ description: "When it starts, in seconds from the beginning of the video. Defaults to 0." })),
|
|
1282
|
+
gain: Type.Optional(Type.Number({ description: "Volume multiplier, 0-4. Defaults to 1. A bed under narration usually wants 0.2-0.4." })),
|
|
1283
|
+
loop: Type.Optional(Type.Boolean({ description: "Repeat this track until the video ends — for a short music bed." })),
|
|
1284
|
+
duck: Type.Optional(Type.Boolean({ description: "Press this track down whenever the `voiceTrack` is speaking, and let it back up between lines." })),
|
|
1285
|
+
}),
|
|
1286
|
+
{ description: "The tracks to place, in any order. mix_audio only." },
|
|
1287
|
+
),
|
|
1288
|
+
),
|
|
1289
|
+
voiceTrack: Type.Optional(
|
|
1290
|
+
Type.Number({
|
|
1291
|
+
description:
|
|
1292
|
+
"Index into `tracks` of the narration everything else ducks under (0 = the first track). " +
|
|
1293
|
+
"Required if any track sets `duck`. mix_audio only.",
|
|
1294
|
+
}),
|
|
1295
|
+
),
|
|
1296
|
+
texts: Type.Optional(
|
|
1297
|
+
Type.Array(
|
|
1298
|
+
Type.Object({
|
|
1299
|
+
text: Type.String({ description: "The words to draw. Punctuation, colons and percent signs are all safe — the text never reaches the filter as syntax." }),
|
|
1300
|
+
fromSeconds: Type.Optional(Type.Number({ description: "When it appears. Omit for the whole video." })),
|
|
1301
|
+
toSeconds: Type.Optional(Type.Number({ description: "When it disappears. Omit to hold to the end." })),
|
|
1302
|
+
position: Type.Optional(Type.String({ description: "top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right. Defaults to bottom." })),
|
|
1303
|
+
fontSize: Type.Optional(Type.Number({ description: "In pixels, 8-400. Defaults to the frame height / 20." })),
|
|
1304
|
+
color: Type.Optional(Type.String({ description: "A colour name or #rrggbb. Defaults to white." })),
|
|
1305
|
+
box: Type.Optional(Type.Boolean({ description: "Draw a filled box behind the type. Without it the type gets a dark outline instead." })),
|
|
1306
|
+
boxColor: Type.Optional(Type.String({ description: "Box colour name or #rrggbb. Defaults to black." })),
|
|
1307
|
+
boxOpacity: Type.Optional(Type.Number({ description: "Box opacity, 0-1. Defaults to 0.5." })),
|
|
1308
|
+
marginPx: Type.Optional(Type.Number({ description: "Distance from the frame edge. Defaults to about 0.8 of the font size." })),
|
|
1309
|
+
}),
|
|
1310
|
+
{ description: "The text to burn in. overlay_text only." },
|
|
1311
|
+
),
|
|
1312
|
+
),
|
|
1313
|
+
fontFile: Type.Optional(
|
|
1314
|
+
Type.String({
|
|
1315
|
+
description:
|
|
1316
|
+
"Path to a .ttf/.otf to draw with. Omit to use a system font — pass one if the video needs the brand's typeface. overlay_text and burn_subtitles.",
|
|
1317
|
+
}),
|
|
1318
|
+
),
|
|
1319
|
+
images: Type.Optional(
|
|
1320
|
+
Type.Array(
|
|
1321
|
+
Type.Object({
|
|
1322
|
+
path: Type.String({ description: "The image (or short video) to composite. PNG transparency is preserved." }),
|
|
1323
|
+
fromSeconds: Type.Optional(Type.Number({ description: "When it appears. Omit for the whole video." })),
|
|
1324
|
+
toSeconds: Type.Optional(Type.Number({ description: "When it disappears. Omit to hold to the end." })),
|
|
1325
|
+
position: Type.Optional(Type.String({ description: "top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right. Defaults to top-right." })),
|
|
1326
|
+
widthPercent: Type.Optional(Type.Number({ description: "Width as a percentage of the video's width, 1-100. The right way to size a logo. Omit to use the art at its own size (shrunk to fit if it is wider than the frame)." })),
|
|
1327
|
+
opacity: Type.Optional(Type.Number({ description: "0-1. Defaults to 1 (opaque); 0.3-0.6 is the range a watermark wants." })),
|
|
1328
|
+
marginPx: Type.Optional(Type.Number({ description: "Distance from the frame edge. Defaults to about 3% of the frame width." })),
|
|
1329
|
+
}),
|
|
1330
|
+
{ description: "The images to composite, drawn in order (later entries sit on top). overlay_image only." },
|
|
1331
|
+
),
|
|
1332
|
+
),
|
|
1333
|
+
subtitles: Type.Optional(
|
|
1334
|
+
Type.String({ description: "Path to a SubRip (.srt) or WebVTT (.vtt) file. Its own timings drive the cues. burn_subtitles only." }),
|
|
1335
|
+
),
|
|
1336
|
+
style: Type.Optional(
|
|
1337
|
+
Type.Object({
|
|
1338
|
+
position: Type.Optional(Type.String({ description: "Where every cue sits. Defaults to bottom." })),
|
|
1339
|
+
fontSize: Type.Optional(Type.Number({ description: "In pixels, 8-400. Defaults to the frame height / 20." })),
|
|
1340
|
+
color: Type.Optional(Type.String({ description: "A colour name or #rrggbb. Defaults to white." })),
|
|
1341
|
+
box: Type.Optional(Type.Boolean({ description: "Draw a plate behind the type. Defaults to TRUE for subtitles; set false for an outline instead." })),
|
|
1342
|
+
boxColor: Type.Optional(Type.String({ description: "Plate colour name or #rrggbb. Defaults to black." })),
|
|
1343
|
+
boxOpacity: Type.Optional(Type.Number({ description: "Plate opacity, 0-1. Defaults to 0.5." })),
|
|
1344
|
+
marginPx: Type.Optional(Type.Number({ description: "Distance from the frame edge. Defaults to about 0.8 of the font size." })),
|
|
1345
|
+
}, { description: "How every subtitle cue is drawn. burn_subtitles only." }),
|
|
1346
|
+
),
|
|
1347
|
+
maxCharsPerLine: Type.Optional(
|
|
1348
|
+
Type.Number({
|
|
1349
|
+
description:
|
|
1350
|
+
"Wrap subtitle lines longer than this, 10-200. Defaults to 42 (the broadcast convention). burn_subtitles only.",
|
|
1351
|
+
}),
|
|
1352
|
+
),
|
|
539
1353
|
}),
|
|
540
1354
|
async execute(
|
|
541
1355
|
_toolCallId: string,
|
|
@@ -555,6 +1369,10 @@ export const videoComposeToolDefinition = {
|
|
|
555
1369
|
case "concat": return text(await opConcat(cwd, params, signal));
|
|
556
1370
|
case "slideshow": return text(await opSlideshow(cwd, params, signal));
|
|
557
1371
|
case "mux_audio": return text(await opMuxAudio(cwd, params, signal));
|
|
1372
|
+
case "mix_audio": return text(await opMixAudio(cwd, params, signal));
|
|
1373
|
+
case "overlay_text": return text(await opOverlayText(cwd, params, signal));
|
|
1374
|
+
case "overlay_image": return text(await opOverlayImage(cwd, params, signal));
|
|
1375
|
+
case "burn_subtitles": return text(await opBurnSubtitles(cwd, params, signal));
|
|
558
1376
|
case "trim": return text(await opTrim(cwd, params, signal));
|
|
559
1377
|
case "extract_frame": return text(await opExtractFrame(cwd, params, signal));
|
|
560
1378
|
case "gif": return text(await opGif(cwd, params, signal));
|