reframe-video 0.6.6 → 0.6.8

package/dist/index.js

@@ -357,7 +357,7 @@ var PROPS_BY_TYPE = {
   line: ["x1", "y1", "x2", "y2", "stroke", "strokeWidth", "opacity", "progress", ...FX_PROPS],
   text: [...COMMON_PROPS, "content", "contentDecimals", "contentThousands", "fontFamily", "fontSize", "fontWeight", "fill", "letterSpacing"],
   image: [...COMMON_PROPS, "src", "width", "height", "fit"],
-  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart"],
+  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart", "volume"],
   path: [...COMMON_PROPS, "d", "fill", "stroke", "strokeWidth", "progress", "originX", "originY"],
   group: COMMON_PROPS
 };
@@ -1020,6 +1020,8 @@ function dropShadow(color, blur = 24, x = 0, y = 12) {
 }
 
 // ../core/src/montage.ts
+var VIDEO_EXT = /\.(mp4|mov|webm|m4v|mkv)$/i;
+var isVideoSrc = (src) => VIDEO_EXT.test(src);
 function makeRng(seed) {
   let a = seed >>> 0 || 2654435769;
   return () => {
@@ -1043,10 +1045,13 @@ function photoMontage(images, opts = {}) {
   const cy = H / 2;
   const nodes = [];
   const shots = [];
+  let clock = 0;
   slides.forEach((slide, i) => {
     const nid = `${id}-${i}`;
     const slideHold = Math.max(0.5, slide.hold ?? hold);
     const transition = Math.min(opts.transition ?? 0.6, slideHold * 0.9);
+    const shotStart = clock;
+    clock += slideHold;
     const kind = slide.ken ?? ["in", "out", "pan"][Math.floor(rand2() * 3)] ?? "in";
     const angle = rand2() * Math.PI * 2;
     const panFrac = 0.4 + rand2() * 0.35;
@@ -1070,19 +1075,9 @@ function photoMontage(images, opts = {}) {
       yA = cy + dy * (kA - 1) * (H / 2) * panFrac;
       yB = cy + dy * (kB - 1) * (H / 2) * panFrac;
     }
+    const box = { id: nid, src: slide.src, x: xA, y: yA, width: W, height: H, anchor: "center", fit: "cover", scale: kA, opacity: i === 0 ? 1 : 0 };
     nodes.push(
-      image({
-        id: nid,
-        src: slide.src,
-        x: xA,
-        y: yA,
-        width: W,
-        height: H,
-        anchor: "center",
-        fit: "cover",
-        scale: kA,
-        opacity: i === 0 ? 1 : 0
-      })
+      isVideoSrc(slide.src) ? video({ ...box, start: shotStart, volume: slide.volume ?? 0 }) : image(box)
     );
     const ken = tween(
       nid,
@@ -1134,6 +1129,7 @@ function photoMontage(images, opts = {}) {
   }
   return { nodes, timeline: beat("montage", { nodes: nodes.map((n3) => n3.id) }, [seq(...shots)]) };
 }
+var videoMontage = photoMontage;
 
 // ../core/src/presets.ts
 var PRESET_NAMES = [
@@ -2628,11 +2624,34 @@ var SFX_DURATION = {
   thud: 0.25
 };
 var FILE_CUE_DURATION = 0.4;
+function collectClipAudio(ir, duration, warnings) {
+  const out = [];
+  const walk = (nodes) => {
+    for (const node of nodes) {
+      if (node.type === "video") {
+        const gain = node.props.volume ?? 1;
+        const start = node.props.start ?? 0;
+        if (gain <= 0) continue;
+        if (start >= duration) {
+          warnings.push(`video "${node.id}": start ${start.toFixed(2)}s past the scene end \u2014 audio dropped`);
+          continue;
+        }
+        out.push({ nodeId: node.id, src: node.props.src, start, rate: node.props.rate ?? 1, clipStart: node.props.clipStart ?? 0, gain });
+      }
+      if (node.type === "group") walk(node.children);
+    }
+  };
+  walk(ir.nodes);
+  return out;
+}
 function resolveAudioPlan(compiled) {
   const audio = compiled.ir.audio;
-  if (!audio || !audio.bgm && (audio.cues ?? []).length === 0) return null;
   const warnings = [];
   const duration = compiled.duration;
+  const clipAudio = collectClipAudio(compiled.ir, duration, warnings);
+  if (!audio || !audio.bgm && (audio.cues ?? []).length === 0) {
+    return clipAudio.length === 0 ? null : { duration, bgm: null, cues: [], duckWindows: [], clipAudio, warnings };
+  }
   const cues = [];
   for (const [index, cue] of (audio.cues ?? []).entries()) {
     let anchor;
@@ -2668,6 +2687,7 @@ function resolveAudioPlan(compiled) {
     bgm: resolveBgm(audio.bgm),
     cues,
     duckWindows: mergeDuckWindows(cues, duration),
+    clipAudio,
     warnings
   };
 }
@@ -2701,6 +2721,7 @@ function resolveCompositionAudioPlan(comp) {
   const duration = comp.duration;
   const warnings = [];
   const cues = [];
+  const clipAudio = [];
   for (const placement of comp.scenes) {
     const plan = resolveAudioPlan(placement.compiled);
     if (!plan) continue;
@@ -2713,6 +2734,11 @@ function resolveCompositionAudioPlan(comp) {
       if (t >= duration) continue;
       cues.push({ ...cue, t });
     }
+    for (const clip of plan.clipAudio) {
+      const start = clip.start + placement.start;
+      if (start >= duration) continue;
+      clipAudio.push({ ...clip, start });
+    }
   }
   for (const [index, cue] of (audio?.cues ?? []).entries()) {
     if (typeof cue.at !== "number") {
@@ -2732,13 +2758,14 @@ function resolveCompositionAudioPlan(comp) {
       source: cue.sfx ? { kind: "sfx", name: cue.sfx, params: cue.params ?? {} } : { kind: "file", path: cue.file }
     });
   }
-  if (!audio?.bgm && cues.length === 0) return null;
+  if (!audio?.bgm && cues.length === 0 && clipAudio.length === 0) return null;
   cues.sort((a, b) => a.t - b.t);
   return {
     duration,
     bgm: resolveBgm(audio?.bgm),
     cues,
     duckWindows: mergeDuckWindows(cues, duration),
+    clipAudio,
     warnings
   };
 }
@@ -3457,6 +3484,7 @@ export {
   validateComposition,
   validateScene,
   video,
+  videoMontage,
   wait,
   wiggle
 };

package/dist/labels.js

@@ -341,7 +341,7 @@ var PROPS_BY_TYPE = {
   line: ["x1", "y1", "x2", "y2", "stroke", "strokeWidth", "opacity", "progress", ...FX_PROPS],
   text: [...COMMON_PROPS, "content", "contentDecimals", "contentThousands", "fontFamily", "fontSize", "fontWeight", "fill", "letterSpacing"],
   image: [...COMMON_PROPS, "src", "width", "height", "fit"],
-  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart"],
+  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart", "volume"],
   path: [...COMMON_PROPS, "d", "fill", "stroke", "strokeWidth", "progress", "originX", "originY"],
   group: COMMON_PROPS
 };

package/dist/trace-cli.js

@@ -14,7 +14,7 @@ var PROPS_BY_TYPE = {
   line: ["x1", "y1", "x2", "y2", "stroke", "strokeWidth", "opacity", "progress", ...FX_PROPS],
   text: [...COMMON_PROPS, "content", "contentDecimals", "contentThousands", "fontFamily", "fontSize", "fontWeight", "fill", "letterSpacing"],
   image: [...COMMON_PROPS, "src", "width", "height", "fit"],
-  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart"],
+  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart", "volume"],
   path: [...COMMON_PROPS, "d", "fill", "stroke", "strokeWidth", "progress", "originX", "originY"],
   group: COMMON_PROPS
 };

package/dist/types/audio.d.ts

@@ -24,6 +24,19 @@ export interface ResolvedCue {
         path: string;
     };
 }
+/** A video node's own audio track, placed on the scene clock. */
+export interface ClipAudio {
+    nodeId: string;
+    src: string;
+    /** Scene-time (s) the clip's audio begins. */
+    start: number;
+    /** Playback speed (atempo). */
+    rate: number;
+    /** Source in-point (s) — audio is trimmed to begin here. */
+    clipStart: number;
+    /** Linear gain. */
+    gain: number;
+}
 export interface AudioPlan {
     duration: number;
     bgm: {
@@ -49,6 +62,8 @@ export interface AudioPlan {
         t0: number;
         t1: number;
     }[];
+    /** Video clip soundtracks to mux in (a clip with no audio stream is skipped at render). */
+    clipAudio: ClipAudio[];
     warnings: string[];
 }
 export declare function resolveAudioPlan(compiled: CompiledScene): AudioPlan | null;

package/dist/types/index.d.ts

@@ -8,7 +8,7 @@ export { pathPoint, pathTangentAngle, type Pt } from "./path.js";
 export { cameraTo, cameraMatrix, CAMERA_ID, CAMERA_PROPS } from "./camera.js";
 export { linearGradient, radialGradient, conicGradient, isGradient } from "./gradient.js";
 export { glow, dropShadow } from "./effects.js";
-export { photoMontage, type MontageImage, type MontageOpts, type MontageResult, type KenBurns } from "./montage.js";
+export { photoMontage, videoMontage, type MontageImage, type MontageOpts, type MontageResult, type KenBurns } from "./montage.js";
 export { motionPreset, PRESET_NAMES, type PresetName, type PresetRig, type PresetOpts } from "./presets.js";
 export { devicePreset, deviceScreen, deviceScreenCenter, deviceBounds, deviceScreenPoint, DEVICE_PRESET_NAMES, type DevicePresetName, type DevicePresetOpts } from "./devicePreset.js";
 export { cursor, cursorTo, cursorPath, cursorClick, cursorDouble, type CursorStyle, type CursorOpts, type CursorToOpts, type CursorPathOpts, type CursorClickOpts } from "./cursor.js";
@@ -17,7 +17,7 @@ export { characterPreset, CHARACTER_PRESET_NAMES, type CharacterPresetName, type
 export { figure, type FigureStyle, type FigureOpts, type FigurePalette } from "./figure.js";
 export { splitText, textIn, textLoop, textOut, textTypeCues, type SplitOpts, type Glyph, type TextBlock, type FontWeight, type TextInName, type TextLoopName, type TextOutName, type TextLoopOpts, type TextOutOpts, type TypeCueOpts, } from "./textFx.js";
 export { motionOp, motionOpLabel, MOTION_OPS, type MotionOpName, type MotionOpOpts, type MotionOpResult } from "./motionOps.js";
-export { resolveAudioPlan, resolveCompositionAudioPlan, SFX_DURATION, type AudioPlan, type ResolvedCue, } from "./audio.js";
+export { resolveAudioPlan, resolveCompositionAudioPlan, SFX_DURATION, type AudioPlan, type ResolvedCue, type ClipAudio, } from "./audio.js";
 export { evaluate, sampleProp, nodeParentMatrix, type DisplayList, type DisplayOp, type Mat2D, type ClipRegion, type TextAlign, type TextBaseline, } from "./evaluate.js";
 export { resolveEase, lerpValue, isColor, EASE_NAMES } from "./interpolate.js";
 export { sampleBehavior } from "./behaviors.js";

package/dist/types/ir.d.ts

@@ -208,6 +208,11 @@ export interface VideoProps extends BaseProps {
     rate?: number;
     /** Source in-point (seconds) shown at `start`. Default 0. */
     clipStart?: number;
+    /**
+     * Linear gain for the clip's own audio track, muxed into the output at `start`
+     * (trimmed from `clipStart`, sped by `rate`). Default 1; `0` mutes the clip.
+     */
+    volume?: number;
 }
 export type NodeIR = {
     type: "rect";

package/dist/types/montage.d.ts

@@ -1,9 +1,10 @@
 /**
- * Photo montage — a SEEDED GENERATOR that turns a list of images into a polished
- * slideshow: layered image nodes + a retimable `beat` that crossfades between
- * slides and pans/zooms each (Ken Burns), with an optional cinematic grade
- * (vignette + bottom scrim) built from gradients + blend modes. The photo analog
- * of `motionPreset` / `splitText`.
+ * Photo/video montage — a SEEDED GENERATOR that turns a list of shots (images AND
+ * video clips, mixed freely) into a polished slideshow: layered image/video nodes +
+ * a retimable `beat` that crossfades between shots and pans/zooms each (Ken Burns),
+ * with an optional cinematic grade (vignette + bottom scrim) built from gradients +
+ * blend modes. A video src plays as a clip for its `hold`. The photo analog of
+ * `motionPreset` / `splitText`. (`videoMontage` is the same generator, by intent.)
  *
  * Pure and deterministic: the per-slide Ken Burns direction / framing is chosen by
  * a seeded PRNG (mulberry32) — same (images, opts) → identical IR; a different
@@ -16,11 +17,16 @@
  */
 import type { NodeIR, TimelineIR } from "./ir.js";
 export type KenBurns = "in" | "out" | "pan";
-/** One slide: a bare src, or a src with per-slide overrides. */
+/**
+ * One shot: a bare src, or a src with per-shot overrides. A video src plays as a
+ * clip for its `hold`; `volume` (video shots only) is the clip-audio gain — default
+ * 0 (muted) in a montage to avoid stacking soundtracks; set it per shot to include.
+ */
 export type MontageImage = string | {
     src: string;
     hold?: number;
     ken?: KenBurns;
+    volume?: number;
 };
 export interface MontageOpts {
     /** Node-id prefix → stable regen addresses `${id}-${i}`. Default "shot". */
@@ -54,3 +60,10 @@ export interface MontageResult {
  *   scene({ size, nodes: [...m.nodes, ...titles], timeline: seq(m.timeline) });
  */
 export declare function photoMontage(images: MontageImage[], opts?: MontageOpts): MontageResult;
+/**
+ * Same as `photoMontage`, named for clip-driven montages — shots may be images or
+ * video clips (mixed freely; a video src plays for its `hold`, muted by default).
+ *
+ *   videoMontage(["intro.jpg", "shot-a.mp4", { src: "shot-b.mp4", volume: 1 }], { seed: 3 })
+ */
+export declare const videoMontage: typeof photoMontage;

package/guides/edsl-guide.md

@@ -291,14 +291,16 @@ const T = splitText("MOTION IS DATA", { id: "t", x: 960, y: 470, fontSize: 130 }
 Every effect is seeded (same `seed` → identical) and pure keyframes. To time a
 `textLoop` window, add up the `textIn` beat length (≈ `(n-1)·stagger + glyphDur`).
 
-## Photo montage (`photoMontage`)
+## Photo / video montage (`photoMontage` / `videoMontage`)
 
-Turn a list of images into a polished slideshow — crossfades + seeded Ken Burns
+Turn a list of shots into a polished slideshow — crossfades + seeded Ken Burns
 (pan/zoom) + an optional cinematic grade (vignette + bottom scrim via gradients +
-blend) — without hand-wiring each move. The photo analog of `motionPreset`.
+blend) — without hand-wiring each move. Shots may be images AND video clips, mixed
+freely (a video src, by extension, plays as a clip for its `hold`). `videoMontage`
+is the same generator, named for clip-driven cuts. The photo analog of `motionPreset`.
 
 ```ts
-const m = photoMontage(["a.jpg", "b.jpg", "c.jpg"], {
+const m = videoMontage(["a.jpg", { src: "b.mp4", volume: 1 }, "c.jpg"], {
   id: "shot", size: { width: 1920, height: 1080 },
   hold: 3.4, transition: 0.7, zoom: 1.16, seed: 7,
 });
@@ -306,16 +308,20 @@ scene({ size, nodes: [...m.nodes, ...titles], timeline: par(m.timeline, titleTra
 ```
 
 - Returns `{ nodes, timeline }` (like `splitText` owns its glyph nodes). `nodes` are
-  the stacked image layers (+ `${id}-vignette` / `${id}-scrim` grade overlays);
+  the stacked image/video layers (+ `${id}-vignette` / `${id}-scrim` grade overlays);
   `timeline` is a retimable `beat("montage", …)`. Stable addresses: `${id}-${i}`,
   labels `shot-${i}` / `cross-${i}`.
-- **Any-aspect photos work** — each layer uses `fit: "cover"`, so the renderer
-  crops to fill the frame at the image's aspect (no pre-cropping, no distortion).
+- **Any-aspect media works** — each layer uses `fit: "cover"`, so the renderer
+  crops to fill the frame at the source's aspect (no pre-cropping, no distortion).
   The Ken Burns keeps `scale ≥ 1` with the pan bounded to its slack, so an edge is
   never revealed.
-- Per-slide overrides: `{ src, hold?, ken? }` where `ken` is `"in" | "out" | "pan"`.
-- Seeded + pure (same `(images, opts)` → identical IR). Note: image-node sources do
-  not render in `reframe player` / artifacts — montage ships as mp4.
+- Per-shot overrides: `{ src, hold?, ken?, volume? }` where `ken` is `"in" | "out" |
+  "pan"`. A **video** shot plays as a clip from its slot's start; its audio is **muted
+  by default** in a montage — set `volume` (per shot) to include it, or add a `scene.audio`
+  bed.
+- Seeded + pure (same `(shots, opts)` → identical IR). Note: image/video sources do
+  not render in `reframe player` / artifacts — montage ships as mp4. See
+  `examples/scenes/video-montage.ts`.
 
 ## Video clips (`video`)
 
@@ -324,19 +330,23 @@ shows the source frame at `clipStart + max(0, t - start) * rate`.
 
 ```ts
 video({ id: "clip", src: "shot.mp4", x: 960, y: 540, width: 1920, height: 1080,
-        anchor: "center", fit: "cover", start: 0, rate: 1, clipStart: 0 })
+        anchor: "center", fit: "cover", start: 0, rate: 1, clipStart: 0, volume: 1 })
 tween("clip", { scale: 1.08 }, { duration: 5 })  // transform composes with playback (Ken Burns)
 ```
 
 - Props: `src` (mp4 / mov / webm / m4v / mkv, absolute or scene-relative), `width`/`height`,
   `fit` (`"cover"` like the image node), `start` (scene-time playback begins), `rate`
-  (speed), `clipStart` (source in-point s). Transform/opacity/effects compose as usual.
+  (speed), `clipStart` (source in-point s), `volume` (clip-audio gain, default 1; `0` mutes).
+  Transform/opacity/effects compose as usual.
 - **Deterministic by frame extraction**: render-cli runs `ffmpeg -vf fps=<sceneFps>` to pull
   the clip's frames, and the renderer draws frame `round(t·fps)` — no live `<video>` seek, so
   it stays byte-identical (same machine).
-- **v1 limitations**: visual-only (the clip's own audio is not muxed — use `scene.audio`);
-  all frames are pre-decoded so keep clips short (≤~5s); like images, not rendered in
-  `reframe player` / artifacts (mp4 only). See `examples/scenes/video-demo.ts`.
+- **Clip audio**: the clip's own audio track is muxed into the output, placed at `start`
+  (trimmed from `clipStart`, sped by `rate`, scaled by `volume`), mixed with `scene.audio`.
+  A clip with no audio stream is skipped; set `volume: 0` to drop a clip's sound.
+- **Limitations**: all frames are pre-decoded so keep clips short (≤~5s); like images, video
+  sources are not rendered in `reframe player` / artifacts (mp4 only). See
+  `examples/scenes/video-demo.ts`.
 
 ## Cursor (UI demos)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reframe-video",
-  "version": "0.6.6",
+  "version": "0.6.8",
   "description": "Declarative motion graphics that AI can write and humans can tweak — human edits survive AI regeneration. Deterministic mp4 renders from a plain-data scene format.",
   "keywords": [
     "motion-graphics",