reframe-video 0.6.5 → 0.6.7

package/dist/index.js

@@ -357,6 +357,7 @@ var PROPS_BY_TYPE = {
   line: ["x1", "y1", "x2", "y2", "stroke", "strokeWidth", "opacity", "progress", ...FX_PROPS],
   text: [...COMMON_PROPS, "content", "contentDecimals", "contentThousands", "fontFamily", "fontSize", "fontWeight", "fill", "letterSpacing"],
   image: [...COMMON_PROPS, "src", "width", "height", "fit"],
+  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart", "volume"],
   path: [...COMMON_PROPS, "d", "fill", "stroke", "strokeWidth", "progress", "originX", "originY"],
   group: COMMON_PROPS
 };
@@ -654,6 +655,10 @@ function image(props) {
   const { id, ...rest } = props;
   return { type: "image", id, props: rest };
 }
+function video(props) {
+  const { id, ...rest } = props;
+  return { type: "video", id, props: rest };
+}
 function path(props) {
   const { id, ...rest } = props;
   return { type: "path", id, props: rest };
@@ -2623,11 +2628,34 @@ var SFX_DURATION = {
   thud: 0.25
 };
 var FILE_CUE_DURATION = 0.4;
+function collectClipAudio(ir, duration, warnings) {
+  const out = [];
+  const walk = (nodes) => {
+    for (const node of nodes) {
+      if (node.type === "video") {
+        const gain = node.props.volume ?? 1;
+        const start = node.props.start ?? 0;
+        if (gain <= 0) continue;
+        if (start >= duration) {
+          warnings.push(`video "${node.id}": start ${start.toFixed(2)}s past the scene end \u2014 audio dropped`);
+          continue;
+        }
+        out.push({ nodeId: node.id, src: node.props.src, start, rate: node.props.rate ?? 1, clipStart: node.props.clipStart ?? 0, gain });
+      }
+      if (node.type === "group") walk(node.children);
+    }
+  };
+  walk(ir.nodes);
+  return out;
+}
 function resolveAudioPlan(compiled) {
   const audio = compiled.ir.audio;
-  if (!audio || !audio.bgm && (audio.cues ?? []).length === 0) return null;
   const warnings = [];
   const duration = compiled.duration;
+  const clipAudio = collectClipAudio(compiled.ir, duration, warnings);
+  if (!audio || !audio.bgm && (audio.cues ?? []).length === 0) {
+    return clipAudio.length === 0 ? null : { duration, bgm: null, cues: [], duckWindows: [], clipAudio, warnings };
+  }
   const cues = [];
   for (const [index, cue] of (audio.cues ?? []).entries()) {
     let anchor;
@@ -2663,6 +2691,7 @@ function resolveAudioPlan(compiled) {
     bgm: resolveBgm(audio.bgm),
     cues,
     duckWindows: mergeDuckWindows(cues, duration),
+    clipAudio,
     warnings
   };
 }
@@ -2696,6 +2725,7 @@ function resolveCompositionAudioPlan(comp) {
   const duration = comp.duration;
   const warnings = [];
   const cues = [];
+  const clipAudio = [];
   for (const placement of comp.scenes) {
     const plan = resolveAudioPlan(placement.compiled);
     if (!plan) continue;
@@ -2708,6 +2738,11 @@ function resolveCompositionAudioPlan(comp) {
       if (t >= duration) continue;
       cues.push({ ...cue, t });
     }
+    for (const clip of plan.clipAudio) {
+      const start = clip.start + placement.start;
+      if (start >= duration) continue;
+      clipAudio.push({ ...clip, start });
+    }
   }
   for (const [index, cue] of (audio?.cues ?? []).entries()) {
     if (typeof cue.at !== "number") {
@@ -2727,13 +2762,14 @@ function resolveCompositionAudioPlan(comp) {
       source: cue.sfx ? { kind: "sfx", name: cue.sfx, params: cue.params ?? {} } : { kind: "file", path: cue.file }
     });
   }
-  if (!audio?.bgm && cues.length === 0) return null;
+  if (!audio?.bgm && cues.length === 0 && clipAudio.length === 0) return null;
   cues.sort((a, b) => a.t - b.t);
   return {
     duration,
     bgm: resolveBgm(audio?.bgm),
     cues,
     duckWindows: mergeDuckWindows(cues, duration),
+    clipAudio,
     warnings
   };
 }
@@ -3187,6 +3223,33 @@ function evaluate(compiled, t) {
         });
         return;
       }
+      case "video": {
+        const width = num(id, "width", node.props.width);
+        const height = num(id, "height", node.props.height);
+        const [ax, ay] = ANCHOR_FACTORS[node.props.anchor ?? "top-left"];
+        const fps = compiled.ir.fps ?? 30;
+        const start = node.props.start ?? 0;
+        const rate = node.props.rate ?? 1;
+        const clipStart = node.props.clipStart ?? 0;
+        const srcT = clipStart + Math.max(0, t - start) * rate;
+        const frame = Math.max(0, Math.round(srcT * fps));
+        ops.push({
+          type: "video",
+          id,
+          transform: matrix,
+          opacity,
+          src: str(id, "src", node.props.src),
+          width,
+          height,
+          offsetX: -width * ax,
+          offsetY: -height * ay,
+          frame,
+          ...node.props.fit && node.props.fit !== "fill" ? { fit: node.props.fit } : {},
+          ...fx,
+          ...clipSpread
+        });
+        return;
+      }
       case "path": {
         const ox = num(id, "originX", node.props.originX ?? 0);
         const oy = num(id, "originY", node.props.originY ?? 0);
@@ -3255,13 +3318,13 @@ function evaluate(compiled, t) {
 }
 
 // ../core/src/assets.ts
-function collectImageSrcs(ir) {
+function collectSrcs(ir, type) {
   const srcs = /* @__PURE__ */ new Set();
-  const imageIds = /* @__PURE__ */ new Set();
+  const ids = /* @__PURE__ */ new Set();
   const walkNodes = (nodes) => {
     for (const node of nodes) {
-      if (node.type === "image") {
-        imageIds.add(node.id);
+      if (node.type === type) {
+        ids.add(node.id);
         srcs.add(node.props.src);
       }
       if (node.type === "group") walkNodes(node.children);
@@ -3270,14 +3333,14 @@ function collectImageSrcs(ir) {
   walkNodes(ir.nodes);
   for (const overrides of Object.values(ir.states ?? {})) {
     for (const [nodeId, props] of Object.entries(overrides)) {
-      if (imageIds.has(nodeId) && typeof props.src === "string") srcs.add(props.src);
+      if (ids.has(nodeId) && typeof props.src === "string") srcs.add(props.src);
     }
   }
   const walkTimeline = (step) => {
     if (!step) return;
     if (step.kind === "seq" || step.kind === "par" || step.kind === "stagger") {
       for (const child of step.children) walkTimeline(child);
-    } else if (step.kind === "tween" && imageIds.has(step.target)) {
+    } else if (step.kind === "tween" && ids.has(step.target)) {
       const src = step.props.src;
       if (typeof src === "string") srcs.add(src);
     }
@@ -3285,6 +3348,12 @@ function collectImageSrcs(ir) {
   walkTimeline(ir.timeline);
   return [...srcs];
 }
+function collectImageSrcs(ir) {
+  return collectSrcs(ir, "image");
+}
+function collectVideoSrcs(ir) {
+  return collectSrcs(ir, "video");
+}
 
 // ../core/src/motion.ts
 var EASE_BY_CLASS = {
@@ -3351,6 +3420,7 @@ export {
   cameraTo,
   characterPreset,
   collectImageSrcs,
+  collectVideoSrcs,
   compileComposition,
   compileScene,
   composeScene,
@@ -3417,6 +3487,7 @@ export {
   tween,
   validateComposition,
   validateScene,
+  video,
   wait,
   wiggle
 };

package/dist/labels.js

@@ -341,6 +341,7 @@ var PROPS_BY_TYPE = {
   line: ["x1", "y1", "x2", "y2", "stroke", "strokeWidth", "opacity", "progress", ...FX_PROPS],
   text: [...COMMON_PROPS, "content", "contentDecimals", "contentThousands", "fontFamily", "fontSize", "fontWeight", "fill", "letterSpacing"],
   image: [...COMMON_PROPS, "src", "width", "height", "fit"],
+  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart", "volume"],
   path: [...COMMON_PROPS, "d", "fill", "stroke", "strokeWidth", "progress", "originX", "originY"],
   group: COMMON_PROPS
 };

package/dist/renderer-canvas.js

@@ -25,7 +25,7 @@ function resolvePaint(ctx, paint, box) {
   for (const s of paint.stops) g.addColorStop(Math.max(0, Math.min(1, s.offset)), s.color);
   return g;
 }
-function renderFrame(ctx, compiled, t, images) {
+function renderFrame(ctx, compiled, t, images, videos) {
   const { size, background } = compiled.ir;
   ctx.setTransform(1, 0, 0, 1, 0, 0);
   ctx.clearRect(0, 0, size.width, size.height);
@@ -33,9 +33,9 @@ function renderFrame(ctx, compiled, t, images) {
     ctx.fillStyle = background;
     ctx.fillRect(0, 0, size.width, size.height);
   }
-  drawDisplayList(ctx, evaluate(compiled, t), images);
+  drawDisplayList(ctx, evaluate(compiled, t), images, videos);
 }
-function drawDisplayList(ctx, ops, images) {
+function drawDisplayList(ctx, ops, images, videos) {
   for (const op of ops) {
     ctx.save();
     if (op.clips) {
@@ -117,28 +117,11 @@ function drawDisplayList(ctx, ops, images) {
         break;
       }
       case "image": {
-        const img = images?.get(op.src);
-        if (img) {
-          if (op.fit === "cover") {
-            const [iw, ih] = intrinsicSize(img);
-            const { sx, sy, sw, sh } = coverRect(iw, ih, op.width, op.height);
-            ctx.drawImage(img, sx, sy, sw, sh, op.offsetX, op.offsetY, op.width, op.height);
-          } else {
-            ctx.drawImage(img, op.offsetX, op.offsetY, op.width, op.height);
-          }
-        } else {
-          ctx.fillStyle = "#2A2A30";
-          ctx.fillRect(op.offsetX, op.offsetY, op.width, op.height);
-          ctx.strokeStyle = "#FF00FF";
-          ctx.lineWidth = 2;
-          ctx.strokeRect(op.offsetX, op.offsetY, op.width, op.height);
-          ctx.beginPath();
-          ctx.moveTo(op.offsetX, op.offsetY);
-          ctx.lineTo(op.offsetX + op.width, op.offsetY + op.height);
-          ctx.moveTo(op.offsetX + op.width, op.offsetY);
-          ctx.lineTo(op.offsetX, op.offsetY + op.height);
-          ctx.stroke();
-        }
+        drawRaster(ctx, images?.get(op.src), op);
+        break;
+      }
+      case "video": {
+        drawRaster(ctx, videos?.frame(op.src, op.frame), op);
         break;
       }
       case "path": {
@@ -193,6 +176,29 @@ function intrinsicSize(img) {
   const a = img;
   return [a.naturalWidth || a.width || 0, a.naturalHeight || a.height || 0];
 }
+function drawRaster(ctx, img, op) {
+  if (img) {
+    if (op.fit === "cover") {
+      const [iw, ih] = intrinsicSize(img);
+      const { sx, sy, sw, sh } = coverRect(iw, ih, op.width, op.height);
+      ctx.drawImage(img, sx, sy, sw, sh, op.offsetX, op.offsetY, op.width, op.height);
+    } else {
+      ctx.drawImage(img, op.offsetX, op.offsetY, op.width, op.height);
+    }
+    return;
+  }
+  ctx.fillStyle = "#2A2A30";
+  ctx.fillRect(op.offsetX, op.offsetY, op.width, op.height);
+  ctx.strokeStyle = "#FF00FF";
+  ctx.lineWidth = 2;
+  ctx.strokeRect(op.offsetX, op.offsetY, op.width, op.height);
+  ctx.beginPath();
+  ctx.moveTo(op.offsetX, op.offsetY);
+  ctx.lineTo(op.offsetX + op.width, op.offsetY + op.height);
+  ctx.moveTo(op.offsetX + op.width, op.offsetY);
+  ctx.lineTo(op.offsetX, op.offsetY + op.height);
+  ctx.stroke();
+}
 function quoteFamily(family) {
   return family.includes(" ") && !family.includes('"') ? `"${family}"` : family;
 }

package/dist/trace-cli.js

@@ -14,6 +14,7 @@ var PROPS_BY_TYPE = {
   line: ["x1", "y1", "x2", "y2", "stroke", "strokeWidth", "opacity", "progress", ...FX_PROPS],
   text: [...COMMON_PROPS, "content", "contentDecimals", "contentThousands", "fontFamily", "fontSize", "fontWeight", "fill", "letterSpacing"],
   image: [...COMMON_PROPS, "src", "width", "height", "fit"],
+  video: [...COMMON_PROPS, "src", "width", "height", "fit", "start", "rate", "clipStart", "volume"],
   path: [...COMMON_PROPS, "d", "fill", "stroke", "strokeWidth", "progress", "originX", "originY"],
   group: COMMON_PROPS
 };

package/dist/types/assets.d.ts

@@ -1,5 +1,5 @@
 /**
- * Asset discovery shared by every consumer that must preload images before
+ * Asset discovery shared by every consumer that must preload media before
  * rendering (the capture page and the preview). One walker means the two
  * sides can never disagree about which srcs a scene uses — including srcs
  * introduced only mid-scene by a state override or a tween.
@@ -7,3 +7,5 @@
 import type { SceneIR } from "./ir.js";
 /** All image srcs a scene can ever display, deduped, in discovery order. */
 export declare function collectImageSrcs(ir: SceneIR): string[];
+/** All video srcs a scene can ever display, deduped, in discovery order. */
+export declare function collectVideoSrcs(ir: SceneIR): string[];

package/dist/types/audio.d.ts

@@ -24,6 +24,19 @@ export interface ResolvedCue {
         path: string;
     };
 }
+/** A video node's own audio track, placed on the scene clock. */
+export interface ClipAudio {
+    nodeId: string;
+    src: string;
+    /** Scene-time (s) the clip's audio begins. */
+    start: number;
+    /** Playback speed (atempo). */
+    rate: number;
+    /** Source in-point (s) — audio is trimmed to begin here. */
+    clipStart: number;
+    /** Linear gain. */
+    gain: number;
+}
 export interface AudioPlan {
     duration: number;
     bgm: {
@@ -49,6 +62,8 @@ export interface AudioPlan {
         t0: number;
         t1: number;
     }[];
+    /** Video clip soundtracks to mux in (a clip with no audio stream is skipped at render). */
+    clipAudio: ClipAudio[];
     warnings: string[];
 }
 export declare function resolveAudioPlan(compiled: CompiledScene): AudioPlan | null;

package/dist/types/dsl.d.ts

@@ -2,7 +2,7 @@
  * The eDSL surface: thin factories that build plain IR objects.
  * `scene()` validates and returns the IR — the return value is the document.
  */
-import type { AudioIR, BehaviorIR, CameraIR, CompositionIR, CompositionSceneEntry, Ease, EllipseProps, GroupProps, ImageProps, LineProps, NodeIR, PathProps, PropValue, RectProps, SceneIR, Size, StateOverride, TextProps, TimelineIR } from "./ir.js";
+import type { AudioIR, BehaviorIR, CameraIR, CompositionIR, CompositionSceneEntry, Ease, EllipseProps, GroupProps, ImageProps, LineProps, NodeIR, PathProps, PropValue, RectProps, SceneIR, Size, StateOverride, TextProps, TimelineIR, VideoProps } from "./ir.js";
 export interface SceneInput {
     id: string;
     size: Size;
@@ -42,6 +42,9 @@ export declare function text(props: {
 export declare function image(props: {
     id: string;
 } & ImageProps): NodeIR;
+export declare function video(props: {
+    id: string;
+} & VideoProps): NodeIR;
 export declare function path(props: {
     id: string;
 } & PathProps): NodeIR;

package/dist/types/evaluate.d.ts

@@ -80,6 +80,18 @@ export type DisplayOp = (OpBase & {
     offsetY: number;
     /** Box-fit; present only when authored and not "fill". */
     fit?: ImageFit;
+}) | (OpBase & {
+    type: "video";
+    /** Raw src string as authored in the IR — consumers resolve it. */
+    src: string;
+    width: number;
+    height: number;
+    offsetX: number;
+    offsetY: number;
+    /** Source frame index at scene-time t (renderer clamps to the extracted count). */
+    frame: number;
+    /** Box-fit; present only when authored and not "fill". */
+    fit?: ImageFit;
 }) | (OpBase & {
     type: "path";
     /** SVG path data, drawn via Path2D. */

package/dist/types/index.d.ts

@@ -17,9 +17,9 @@ export { characterPreset, CHARACTER_PRESET_NAMES, type CharacterPresetName, type
 export { figure, type FigureStyle, type FigureOpts, type FigurePalette } from "./figure.js";
 export { splitText, textIn, textLoop, textOut, textTypeCues, type SplitOpts, type Glyph, type TextBlock, type FontWeight, type TextInName, type TextLoopName, type TextOutName, type TextLoopOpts, type TextOutOpts, type TypeCueOpts, } from "./textFx.js";
 export { motionOp, motionOpLabel, MOTION_OPS, type MotionOpName, type MotionOpOpts, type MotionOpResult } from "./motionOps.js";
-export { resolveAudioPlan, resolveCompositionAudioPlan, SFX_DURATION, type AudioPlan, type ResolvedCue, } from "./audio.js";
+export { resolveAudioPlan, resolveCompositionAudioPlan, SFX_DURATION, type AudioPlan, type ResolvedCue, type ClipAudio, } from "./audio.js";
 export { evaluate, sampleProp, nodeParentMatrix, type DisplayList, type DisplayOp, type Mat2D, type ClipRegion, type TextAlign, type TextBaseline, } from "./evaluate.js";
 export { resolveEase, lerpValue, isColor, EASE_NAMES } from "./interpolate.js";
 export { sampleBehavior } from "./behaviors.js";
-export { collectImageSrcs } from "./assets.js";
+export { collectImageSrcs, collectVideoSrcs } from "./assets.js";
 export { sketchToTimeline, type MotionSketch, type MotionEvent, type MotionEventKind, type MotionRegion, } from "./motion.js";

package/dist/types/ir.d.ts

@@ -192,6 +192,28 @@ export interface ImageProps extends BaseProps {
 }
 /** Image box-fit mode. `cover` = crop-to-fill at the image's aspect (centered). */
 export type ImageFit = "fill" | "cover";
+export interface VideoProps extends BaseProps {
+    /** Video file path (absolute, or relative to the scene file). */
+    src: string;
+    width: number;
+    height: number;
+    /** Box-fit into width×height, like the image node. `"fill"` (default) | `"cover"`. */
+    fit?: ImageFit;
+    /**
+     * Scene-time (seconds) at which playback begins. Before it, frame 0 (clipStart) shows;
+     * the node's visibility is still controlled by opacity/timeline. Default 0.
+     */
+    start?: number;
+    /** Playback speed multiplier (2 = double speed). Default 1. */
+    rate?: number;
+    /** Source in-point (seconds) shown at `start`. Default 0. */
+    clipStart?: number;
+    /**
+     * Linear gain for the clip's own audio track, muxed into the output at `start`
+     * (trimmed from `clipStart`, sped by `rate`). Default 1; `0` mutes the clip.
+     */
+    volume?: number;
+}
 export type NodeIR = {
     type: "rect";
     id: string;
@@ -212,6 +234,10 @@ export type NodeIR = {
     type: "image";
     id: string;
     props: ImageProps;
+} | {
+    type: "video";
+    id: string;
+    props: VideoProps;
 } | {
     type: "path";
     id: string;

package/guides/edsl-guide.md

@@ -317,6 +317,31 @@ scene({ size, nodes: [...m.nodes, ...titles], timeline: par(m.timeline, titleTra
 - Seeded + pure (same `(images, opts)` → identical IR). Note: image-node sources do
   not render in `reframe player` / artifacts — montage ships as mp4.
 
+## Video clips (`video`)
+
+Draw a video clip as a layer. It plays on the scene clock — at scene-time `t` it
+shows the source frame at `clipStart + max(0, t - start) * rate`.
+
+```ts
+video({ id: "clip", src: "shot.mp4", x: 960, y: 540, width: 1920, height: 1080,
+        anchor: "center", fit: "cover", start: 0, rate: 1, clipStart: 0, volume: 1 })
+tween("clip", { scale: 1.08 }, { duration: 5 })  // transform composes with playback (Ken Burns)
+```
+
+- Props: `src` (mp4 / mov / webm / m4v / mkv, absolute or scene-relative), `width`/`height`,
+  `fit` (`"cover"` like the image node), `start` (scene-time playback begins), `rate`
+  (speed), `clipStart` (source in-point s), `volume` (clip-audio gain, default 1; `0` mutes).
+  Transform/opacity/effects compose as usual.
+- **Deterministic by frame extraction**: render-cli runs `ffmpeg -vf fps=<sceneFps>` to pull
+  the clip's frames, and the renderer draws frame `round(t·fps)` — no live `<video>` seek, so
+  it stays byte-identical (same machine).
+- **Clip audio**: the clip's own audio track is muxed into the output, placed at `start`
+  (trimmed from `clipStart`, sped by `rate`, scaled by `volume`), mixed with `scene.audio`.
+  A clip with no audio stream is skipped; set `volume: 0` to drop a clip's sound.
+- **Limitations**: all frames are pre-decoded so keep clips short (≤~5s); like images, video
+  sources are not rendered in `reframe player` / artifacts (mp4 only). See
+  `examples/scenes/video-demo.ts`.
+
 ## Cursor (UI demos)
 
 A vector mouse pointer that glides across the scene and clicks things — for app

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reframe-video",
-  "version": "0.6.5",
+  "version": "0.6.7",
   "description": "Declarative motion graphics that AI can write and humans can tweak — human edits survive AI regeneration. Deterministic mp4 renders from a plain-data scene format.",
   "keywords": [
     "motion-graphics",

package/preview/src/main.ts

@@ -551,7 +551,8 @@ function opCorners(op: DisplayOp): [number, number][] {
   switch (op.type) {
     case "rect":
     case "ellipse":
-    case "image": {
+    case "image":
+    case "video": {
       const { offsetX: x, offsetY: y, width: w, height: h } = op;
       return [[x, y], [x + w, y], [x + w, y + h], [x, y + h]].map(([px, py]) =>
         applyMat(op.transform, px!, py!),