takomi 2.1.45 → 2.5.4

package/assets/.agent/skills/remotion/rules/audio-visualization.md

@@ -0,0 +1,198 @@
+---
+name: audio-visualization
+description: Audio visualization patterns - spectrum bars, waveforms, bass-reactive effects
+metadata:
+  tags: audio, visualization, spectrum, waveform, bass, music, audiogram, frequency
+---
+
+# Audio Visualization in Remotion
+
+## Prerequisites
+
+```bash
+npx remotion add @remotion/media-utils
+```
+
+## Loading Audio Data
+
+Use `useWindowedAudioData()` (https://www.remotion.dev/docs/use-windowed-audio-data) to load audio data:
+
+```tsx
+import { useWindowedAudioData } from "@remotion/media-utils";
+import { staticFile, useCurrentFrame, useVideoConfig } from "remotion";
+
+const frame = useCurrentFrame();
+const { fps } = useVideoConfig();
+
+const { audioData, dataOffsetInSeconds } = useWindowedAudioData({
+  src: staticFile("podcast.wav"),
+  frame,
+  fps,
+  windowInSeconds: 30,
+});
+```
+
+## Spectrum Bar Visualization
+
+Use `visualizeAudio()` (https://www.remotion.dev/docs/visualize-audio) to get frequency data for bar charts:
+
+```tsx
+import { useWindowedAudioData, visualizeAudio } from "@remotion/media-utils";
+import { staticFile, useCurrentFrame, useVideoConfig } from "remotion";
+
+const frame = useCurrentFrame();
+const { fps } = useVideoConfig();
+
+const { audioData, dataOffsetInSeconds } = useWindowedAudioData({
+  src: staticFile("music.mp3"),
+  frame,
+  fps,
+  windowInSeconds: 30,
+});
+
+if (!audioData) {
+  return null;
+}
+
+const frequencies = visualizeAudio({
+  fps,
+  frame,
+  audioData,
+  numberOfSamples: 256,
+  optimizeFor: "speed",
+  dataOffsetInSeconds,
+});
+
+return (
+  <div style={{ display: "flex", alignItems: "flex-end", height: 200 }}>
+    {frequencies.map((v, i) => (
+      <div
+        key={i}
+        style={{
+          flex: 1,
+          height: `${v * 100}%`,
+          backgroundColor: "#0b84f3",
+          margin: "0 1px",
+        }}
+      />
+    ))}
+  </div>
+);
+```
+
+- `numberOfSamples` must be power of 2 (32, 64, 128, 256, 512, 1024)
+- Values range 0-1; left of array = bass, right = highs
+- Use `optimizeFor: "speed"` for Lambda or high sample counts
+
+**Important:** When passing `audioData` to child components, also pass the `frame` from the parent. Do not call `useCurrentFrame()` in each child - this causes discontinuous visualization when children are inside `<Sequence>` with offsets.
+
+## Waveform Visualization
+
+Use `visualizeAudioWaveform()` (https://www.remotion.dev/docs/media-utils/visualize-audio-waveform) with `createSmoothSvgPath()` (https://www.remotion.dev/docs/media-utils/create-smooth-svg-path) for oscilloscope-style displays:
+
+```tsx
+import {
+  createSmoothSvgPath,
+  useWindowedAudioData,
+  visualizeAudioWaveform,
+} from "@remotion/media-utils";
+import { staticFile, useCurrentFrame, useVideoConfig } from "remotion";
+
+const frame = useCurrentFrame();
+const { width, fps } = useVideoConfig();
+const HEIGHT = 200;
+
+const { audioData, dataOffsetInSeconds } = useWindowedAudioData({
+  src: staticFile("voice.wav"),
+  frame,
+  fps,
+  windowInSeconds: 30,
+});
+
+if (!audioData) {
+  return null;
+}
+
+const waveform = visualizeAudioWaveform({
+  fps,
+  frame,
+  audioData,
+  numberOfSamples: 256,
+  windowInSeconds: 0.5,
+  dataOffsetInSeconds,
+});
+
+const path = createSmoothSvgPath({
+  points: waveform.map((y, i) => ({
+    x: (i / (waveform.length - 1)) * width,
+    y: HEIGHT / 2 + (y * HEIGHT) / 2,
+  })),
+});
+
+return (
+  <svg width={width} height={HEIGHT}>
+    <path d={path} fill="none" stroke="#0b84f3" strokeWidth={2} />
+  </svg>
+);
+```
+
+## Bass-Reactive Effects
+
+Extract low frequencies for beat-reactive animations:
+
+```tsx
+const frequencies = visualizeAudio({
+  fps,
+  frame,
+  audioData,
+  numberOfSamples: 128,
+  optimizeFor: "speed",
+  dataOffsetInSeconds,
+});
+
+const lowFrequencies = frequencies.slice(0, 32);
+const bassIntensity =
+  lowFrequencies.reduce((sum, v) => sum + v, 0) / lowFrequencies.length;
+
+const scale = 1 + bassIntensity * 0.5;
+const opacity = Math.min(0.6, bassIntensity * 0.8);
+```
+
+## Volume-Based Waveform
+
+Use `getWaveformPortion()` (https://www.remotion.dev/docs/get-waveform-portion) when you need simplified volume data instead of frequency spectrum:
+
+```tsx
+import { getWaveformPortion } from "@remotion/media-utils";
+import { useCurrentFrame, useVideoConfig } from "remotion";
+
+const frame = useCurrentFrame();
+const { fps } = useVideoConfig();
+const currentTimeInSeconds = frame / fps;
+
+const waveform = getWaveformPortion({
+  audioData,
+  startTimeInSeconds: currentTimeInSeconds,
+  durationInSeconds: 5,
+  numberOfSamples: 50,
+});
+
+// Returns array of { index, amplitude } objects (amplitude: 0-1)
+waveform.map((bar) => (
+  <div key={bar.index} style={{ height: bar.amplitude * 100 }} />
+));
+```
+
+## Postprocessing
+
+Low frequencies naturally dominate. Apply logarithmic scaling for visual balance:
+
+```tsx
+const minDb = -100;
+const maxDb = -30;
+
+const scaled = frequencies.map((value) => {
+  const db = 20 * Math.log10(value);
+  return (db - minDb) / (maxDb - minDb);
+});
+```

package/assets/.agent/skills/remotion/rules/audio.md

@@ -13,10 +13,7 @@ First, the @remotion/media package needs to be installed.
 If it is not installed, use the following command:
 
 ```bash
-npx remotion add @remotion/media # If project uses npm
-bunx remotion add @remotion/media # If project uses bun
-yarn remotion add @remotion/media # If project uses yarn
-pnpm exec remotion add @remotion/media # If project uses pnpm
+npx remotion add @remotion/media
 ```
 
 ## Importing Audio

package/assets/.agent/skills/remotion/rules/calculate-metadata.md

@@ -10,19 +10,30 @@ metadata:
 Use `calculateMetadata` on a `<Composition>` to dynamically set duration, dimensions, and transform props before rendering.
 
 ```tsx
-<Composition id="MyComp" component={MyComponent} durationInFrames={300} fps={30} width={1920} height={1080} defaultProps={{videoSrc: 'https://remotion.media/video.mp4'}} calculateMetadata={calculateMetadata} />
+<Composition
+  id="MyComp"
+  component={MyComponent}
+  durationInFrames={300}
+  fps={30}
+  width={1920}
+  height={1080}
+  defaultProps={{ videoSrc: "https://remotion.media/video.mp4" }}
+  calculateMetadata={calculateMetadata}
+/>
 ```
 
 ## Setting duration based on a video
 
-Use the `getMediaMetadata()` function from the mediabunny/metadata skill to get the video duration:
+Use the [`getVideoDuration`](./get-video-duration.md) and [`getVideoDimensions`](./get-video-dimensions.md) skills to get the video duration and dimensions:
 
 ```tsx
-import {CalculateMetadataFunction} from 'remotion';
-import {getMediaMetadata} from '../get-media-metadata';
+import { CalculateMetadataFunction } from "remotion";
+import { getVideoDuration } from "./get-video-duration";
 
-const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
-  const {durationInSeconds} = await getMediaMetadata(props.videoSrc);
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  const durationInSeconds = await getVideoDuration(props.videoSrc);
 
   return {
     durationInFrames: Math.ceil(durationInSeconds * 30),
@@ -32,14 +43,21 @@ const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
 
 ## Matching dimensions of a video
 
+Use the [`getVideoDimensions`](./get-video-dimensions.md) skill to get the video dimensions:
+
 ```tsx
-const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
-  const {durationInSeconds, dimensions} = await getMediaMetadata(props.videoSrc);
+import { CalculateMetadataFunction } from "remotion";
+import { getVideoDuration } from "./get-video-duration";
+import { getVideoDimensions } from "./get-video-dimensions";
+
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  const dimensions = await getVideoDimensions(props.videoSrc);
 
   return {
-    durationInFrames: Math.ceil(durationInSeconds * 30),
-    width: dimensions?.width ?? 1920,
-    height: dimensions?.height ?? 1080,
+    width: dimensions.width,
+    height: dimensions.height,
   };
 };
 ```
@@ -47,11 +65,18 @@ const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
 ## Setting duration based on multiple videos
 
 ```tsx
-const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
-  const metadataPromises = props.videos.map((video) => getMediaMetadata(video.src));
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  const metadataPromises = props.videos.map((video) =>
+    getVideoDuration(video.src),
+  );
   const allMetadata = await Promise.all(metadataPromises);
 
-  const totalDuration = allMetadata.reduce((sum, meta) => sum + meta.durationInSeconds, 0);
+  const totalDuration = allMetadata.reduce(
+    (sum, durationInSeconds) => sum + durationInSeconds,
+    0,
+  );
 
   return {
     durationInFrames: Math.ceil(totalDuration * 30),
@@ -64,9 +89,11 @@ const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
 Set the default output filename based on props:
 
 ```tsx
-const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
   return {
-    defaultOutName: `video-${props.id}.mp4`,
+    defaultOutName: `video-${props.id}`, // .mp4 is added automatically
   };
 };
 ```
@@ -76,8 +103,11 @@ const calculateMetadata: CalculateMetadataFunction<Props> = async ({props}) => {
 Fetch data or transform props before rendering:
 
 ```tsx
-const calculateMetadata: CalculateMetadataFunction<Props> = async ({props, abortSignal}) => {
-  const response = await fetch(props.dataUrl, {signal: abortSignal});
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+  abortSignal,
+}) => {
+  const response = await fetch(props.dataUrl, { signal: abortSignal });
   const data = await response.json();
 
   return {

package/assets/.agent/skills/remotion/rules/compositions.md

@@ -7,25 +7,14 @@ metadata:
 
 A `<Composition>` defines the component, width, height, fps and duration of a renderable video.
 
-It normally is placed in the `src/Root.tsx` file.
-
-```tsx
-import {Composition} from 'remotion';
-import {MyComposition} from './MyComposition';
-
-export const RemotionRoot = () => {
-  return <Composition id="MyComposition" component={MyComposition} durationInFrames={100} fps={30} width={1080} height={1080} />;
-};
-```
-
 ## Default Props
 
 Pass `defaultProps` to provide initial values for your component.  
 Values must be JSON-serializable (`Date`, `Map`, `Set`, and `staticFile()` are supported).
 
 ```tsx
-import {Composition} from 'remotion';
-import {MyComposition, MyCompositionProps} from './MyComposition';
+import { Composition } from "remotion";
+import { MyComposition, MyCompositionProps } from "./MyComposition";
 
 export const RemotionRoot = () => {
   return (
@@ -38,8 +27,8 @@ export const RemotionRoot = () => {
       height={1080}
       defaultProps={
         {
-          title: 'Hello World',
-          color: '#ff0000',
+          title: "Hello World",
+          color: "#ff0000",
         } satisfies MyCompositionProps
       }
     />
@@ -55,7 +44,7 @@ Use `<Folder>` to organize compositions in the sidebar.
 Folder names can only contain letters, numbers, and hyphens.
 
 ```tsx
-import {Composition, Folder} from 'remotion';
+import { Composition, Folder } from "remotion";
 
 export const RemotionRoot = () => {
   return (
@@ -80,11 +69,13 @@ export const RemotionRoot = () => {
 Use `<Still>` for single-frame images. It does not require `durationInFrames` or `fps`.
 
 ```tsx
-import {Still} from 'remotion';
-import {Thumbnail} from './Thumbnail';
+import { Still } from "remotion";
+import { Thumbnail } from "./Thumbnail";
 
 export const RemotionRoot = () => {
-  return <Still id="Thumbnail" component={Thumbnail} width={1280} height={720} />;
+  return (
+    <Still id="Thumbnail" component={Thumbnail} width={1280} height={720} />
+  );
 };
 ```
 
@@ -93,10 +84,12 @@ export const RemotionRoot = () => {
 Use `calculateMetadata` to make dimensions, duration, or props dynamic based on data.
 
 ```tsx
-import {Composition, CalculateMetadataFunction} from 'remotion';
-import {MyComposition, MyCompositionProps} from './MyComposition';
+import { Composition, CalculateMetadataFunction } from "remotion";
+import { MyComposition, MyCompositionProps } from "./MyComposition";
 
-const calculateMetadata: CalculateMetadataFunction<MyCompositionProps> = async ({props, abortSignal}) => {
+const calculateMetadata: CalculateMetadataFunction<
+  MyCompositionProps
+> = async ({ props, abortSignal }) => {
   const data = await fetch(`https://api.example.com/video/${props.videoId}`, {
     signal: abortSignal,
   }).then((res) => res.json());
@@ -115,11 +108,10 @@ export const RemotionRoot = () => {
     <Composition
       id="MyComposition"
       component={MyComposition}
-      durationInFrames={100} // Placeholder, will be overridden
       fps={30}
       width={1080}
       height={1080}
-      defaultProps={{videoId: 'abc123'}}
+      defaultProps={{ videoId: "abc123" }}
       calculateMetadata={calculateMetadata}
     />
   );

package/assets/.agent/skills/remotion/rules/display-captions.md

@@ -7,18 +7,55 @@ metadata:
 
 # Displaying captions in Remotion
 
-This guide explains how to display captions in Remotion, assuming you already have captions in the `Caption` format.
+This guide explains how to display captions in Remotion, assuming you already have captions in the [`Caption`](https://www.remotion.dev/docs/captions/caption) format.
 
 ## Prerequisites
 
-First, the @remotion/captions package needs to be installed.
+Read [Transcribing audio](transcribe-captions.md) for how to generate captions.
+
+First, the [`@remotion/captions`](https://www.remotion.dev/docs/captions) package needs to be installed.
 If it is not installed, use the following command:
 
 ```bash
-npx remotion add @remotion/captions # If project uses npm
-bunx remotion add @remotion/captions # If project uses bun
-yarn remotion add @remotion/captions # If project uses yarn
-pnpm exec remotion add @remotion/captions # If project uses pnpm
+npx remotion add @remotion/captions
+```
+
+## Fetching captions
+
+First, fetch your captions JSON file. Use [`useDelayRender()`](https://www.remotion.dev/docs/use-delay-render) to hold the render until the captions are loaded:
+
+```tsx
+import { useState, useEffect, useCallback } from "react";
+import { AbsoluteFill, staticFile, useDelayRender } from "remotion";
+import type { Caption } from "@remotion/captions";
+
+export const MyComponent: React.FC = () => {
+  const [captions, setCaptions] = useState<Caption[] | null>(null);
+  const { delayRender, continueRender, cancelRender } = useDelayRender();
+  const [handle] = useState(() => delayRender());
+
+  const fetchCaptions = useCallback(async () => {
+    try {
+      // Assuming captions.json is in the public/ folder.
+      const response = await fetch(staticFile("captions123.json"));
+      const data = await response.json();
+      setCaptions(data);
+      continueRender(handle);
+    } catch (e) {
+      cancelRender(e);
+    }
+  }, [continueRender, cancelRender, handle]);
+
+  useEffect(() => {
+    fetchCaptions();
+  }, [fetchCaptions]);
+
+  if (!captions) {
+    return null;
+  }
+
+  return <AbsoluteFill>{/* Render captions here */}</AbsoluteFill>;
+};
 ```
 
 ## Creating pages
@@ -26,16 +63,16 @@ pnpm exec remotion add @remotion/captions # If project uses pnpm
 Use `createTikTokStyleCaptions()` to group captions into pages. The `combineTokensWithinMilliseconds` option controls how many words appear at once:
 
 ```tsx
-import {useMemo} from 'react';
-import {createTikTokStyleCaptions} from '@remotion/captions';
-import type {Caption} from '@remotion/captions';
+import { useMemo } from "react";
+import { createTikTokStyleCaptions } from "@remotion/captions";
+import type { Caption } from "@remotion/captions";
 
 // How often captions should switch (in milliseconds)
 // Higher values = more words per page
 // Lower values = fewer words (more word-by-word)
 const SWITCH_CAPTIONS_EVERY_MS = 1200;
 
-const {pages} = useMemo(() => {
+const { pages } = useMemo(() => {
   return createTikTokStyleCaptions({
     captions,
     combineTokensWithinMilliseconds: SWITCH_CAPTIONS_EVERY_MS,
@@ -48,11 +85,11 @@ const {pages} = useMemo(() => {
 Map over the pages and render each one in a `<Sequence>`. Calculate the start frame and duration from the page timing:
 
 ```tsx
-import {Sequence, useVideoConfig, AbsoluteFill} from 'remotion';
-import type {TikTokPage} from '@remotion/captions';
+import { Sequence, useVideoConfig, AbsoluteFill } from "remotion";
+import type { TikTokPage } from "@remotion/captions";
 
 const CaptionedContent: React.FC = () => {
-  const {fps} = useVideoConfig();
+  const { fps } = useVideoConfig();
 
   return (
     <AbsoluteFill>
@@ -84,19 +121,28 @@ const CaptionedContent: React.FC = () => {
 };
 ```
 
+## White-space preservation
+
+The captions are whitespace sensitive. You should include spaces in the `text` field before each word. Use `whiteSpace: "pre"` to preserve the whitespace in the captions.
+
+## Separate component for captions
+
+Put captioning logic in a separate component.  
+Make a new file for it.
+
 ## Word highlighting
 
 A caption page contains `tokens` which you can use to highlight the currently spoken word:
 
 ```tsx
-import {AbsoluteFill, useCurrentFrame, useVideoConfig} from 'remotion';
-import type {TikTokPage} from '@remotion/captions';
+import { AbsoluteFill, useCurrentFrame, useVideoConfig } from "remotion";
+import type { TikTokPage } from "@remotion/captions";
 
-const HIGHLIGHT_COLOR = '#39E508';
+const HIGHLIGHT_COLOR = "#39E508";
 
-const CaptionPage: React.FC<{page: TikTokPage}> = ({page}) => {
+const CaptionPage: React.FC<{ page: TikTokPage }> = ({ page }) => {
   const frame = useCurrentFrame();
-  const {fps} = useVideoConfig();
+  const { fps } = useVideoConfig();
 
   // Current time relative to the start of the sequence
   const currentTimeMs = (frame / fps) * 1000;
@@ -104,8 +150,8 @@ const CaptionPage: React.FC<{page: TikTokPage}> = ({page}) => {
   const absoluteTimeMs = page.startMs + currentTimeMs;
 
   return (
-    <AbsoluteFill style={{justifyContent: 'center', alignItems: 'center'}}>
-      <div style={{fontSize: 80, fontWeight: 'bold', whiteSpace: 'pre'}}>
+    <AbsoluteFill style={{ justifyContent: "center", alignItems: "center" }}>
+      <div style={{ fontSize: 80, fontWeight: "bold", whiteSpace: "pre" }}>
         {page.tokens.map((token) => {
           const isActive =
             token.fromMs <= absoluteTimeMs && token.toMs > absoluteTimeMs;
@@ -113,7 +159,7 @@ const CaptionPage: React.FC<{page: TikTokPage}> = ({page}) => {
           return (
             <span
               key={token.fromMs}
-              style={{color: isActive ? HIGHLIGHT_COLOR : 'white'}}
+              style={{ color: isActive ? HIGHLIGHT_COLOR : "white" }}
             >
               {token.text}
             </span>
@@ -124,3 +170,15 @@ const CaptionPage: React.FC<{page: TikTokPage}> = ({page}) => {
   );
 };
 ```
+
+## Display captions alongside video content
+
+By default, put the captions alongside the video content, so the captions are in sync.  
+For each video, make a new captions JSON file.
+
+```tsx
+<AbsoluteFill>
+  <Video src={staticFile("video.mp4")} />
+  <CaptionPage page={page} />
+</AbsoluteFill>
+```