webcodecs-node 0.5.1 → 0.7.0

package/docs/api.md

@@ -9,7 +9,9 @@ This document provides detailed API documentation for webcodecs-node.
 - [AudioEncoder](#audioencoder)
 - [AudioDecoder](#audiodecoder)
 - [ImageDecoder](#imagedecoder)
+- [ImageEncoder](#imageencoder)
 - [VideoFrame](#videoframe)
+- [VideoColorSpace](#videocolorspace)
 - [AudioData](#audiodata)
 - [EncodedVideoChunk](#encodedvideochunk)
 - [EncodedAudioChunk](#encodedaudiochunk)
@@ -64,6 +66,7 @@ Configure the encoder. Must be called before encoding.
 | `alpha` | 'discard' \| 'keep' | No | Alpha channel handling |
 | `latencyMode` | 'quality' \| 'realtime' | No | Latency vs quality tradeoff |
 | `hardwareAcceleration` | 'no-preference' \| 'prefer-hardware' \| 'prefer-software' | No | Hardware acceleration preference |
+| `maxQueueSize` | number | No | Max pending frames before QuotaExceededError. Auto-calculated from resolution if not set (~300MB target memory). |
 
 #### `encode(frame: VideoFrame, options?: VideoEncoderEncodeOptions): void`
 
@@ -122,6 +125,7 @@ Check if a configuration is supported.
 | `codedWidth` | number | No | Coded frame width |
 | `codedHeight` | number | No | Coded frame height |
 | `description` | BufferSource | No | Codec-specific data (e.g., SPS/PPS for H.264) |
+| `maxQueueSize` | number | No | Max pending chunks before QuotaExceededError. Auto-calculated from resolution if dimensions provided (~300MB target memory). |
 
 #### `decode(chunk: EncodedVideoChunk): void`
 
@@ -289,6 +293,63 @@ Close the decoder.
 
 ---
 
+## ImageEncoder
+
+Encodes VideoFrames to image formats (PNG, JPEG, WebP). This is a utility class (not part of WebCodecs spec) that mirrors ImageDecoder.
+
+### Static Methods
+
+#### `isTypeSupported(type: string): boolean`
+
+Check if an output format is supported.
+
+```typescript
+ImageEncoder.isTypeSupported('image/webp'); // true
+ImageEncoder.isTypeSupported('image/gif');  // false
+```
+
+#### `encode(frame: VideoFrame, options?: ImageEncoderOptions): Promise<ImageEncoderResult>`
+
+Encode a VideoFrame to an image format asynchronously.
+
+```typescript
+const result = await ImageEncoder.encode(frame, {
+  type: 'image/jpeg',
+  quality: 0.85,
+});
+fs.writeFileSync('output.jpg', Buffer.from(result.data));
+```
+
+#### `encodeSync(frame: VideoFrame, options?: ImageEncoderOptions): ImageEncoderResult`
+
+Encode a VideoFrame synchronously.
+
+```typescript
+const result = ImageEncoder.encodeSync(frame, { type: 'image/png' });
+```
+
+#### `encodeBatch(frames: VideoFrame[], options?: ImageEncoderOptions): Promise<ImageEncoderResult[]>`
+
+Encode multiple frames in parallel.
+
+```typescript
+const results = await ImageEncoder.encodeBatch(frames, { type: 'image/webp' });
+```
+
+### Types
+
+**ImageEncoderOptions:**
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `type` | `'image/png' \| 'image/jpeg' \| 'image/webp'` | `'image/png'` | Output format |
+| `quality` | number | 0.92 (JPEG), 0.8 (WebP) | Quality for lossy formats (0-1) |
+
+**ImageEncoderResult:**
+- `data: ArrayBuffer` - Encoded image data
+- `type: string` - MIME type of the encoded image
+
+---
+
 ## VideoFrame
 
 Represents a single video frame with raw pixel data.
@@ -311,6 +372,8 @@ new VideoFrame(data: BufferSource, init: VideoFrameBufferInit)
 | `displayHeight` | number | No | Display height |
 
 **Supported VideoPixelFormat values:**
+
+*8-bit formats:*
 - `'I420'` - YUV 4:2:0 planar
 - `'I420A'` - YUV 4:2:0 planar with alpha
 - `'I422'` - YUV 4:2:2 planar
@@ -321,6 +384,12 @@ new VideoFrame(data: BufferSource, init: VideoFrameBufferInit)
 - `'BGRA'` - 8-bit BGRA
 - `'BGRX'` - 8-bit BGR (alpha ignored)
 
+*10-bit formats (HDR):*
+- `'I420P10'` - YUV 4:2:0 planar, 10-bit (16-bit container)
+- `'I422P10'` - YUV 4:2:2 planar, 10-bit (16-bit container)
+- `'I444P10'` - YUV 4:4:4 planar, 10-bit (16-bit container)
+- `'P010'` - YUV 4:2:0 semi-planar, 10-bit (16-bit container)
+
 ### Instance Methods
 
 #### `allocationSize(options?: VideoFrameCopyToOptions): number`
@@ -348,6 +417,92 @@ Release frame resources. Always call this when done.
 - `displayHeight: number`
 - `timestamp: number`
 - `duration: number | null`
+- `colorSpace: VideoColorSpace` - Color space information
+
+---
+
+## VideoColorSpace
+
+Describes the color space of a video frame, including HDR metadata.
+
+### Constructor
+
+```typescript
+new VideoColorSpace(init?: VideoColorSpaceInit)
+```
+
+**VideoColorSpaceInit:**
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `primaries` | string | null | Color primaries ('bt709', 'bt2020', 'smpte432') |
+| `transfer` | string | null | Transfer function ('bt709', 'pq', 'hlg', 'srgb') |
+| `matrix` | string | null | Matrix coefficients ('bt709', 'bt2020-ncl') |
+| `fullRange` | boolean | null | Full vs limited range |
+| `hdrMetadata` | HdrMetadata | null | HDR mastering metadata |
+
+### Properties
+
+- `primaries: string | null` - Color primaries
+- `transfer: string | null` - Transfer function
+- `matrix: string | null` - Matrix coefficients
+- `fullRange: boolean | null` - Full range flag
+- `hdrMetadata: HdrMetadata | null` - HDR metadata (if present)
+- `isHdr: boolean` - True if PQ or HLG transfer
+- `hasHdrMetadata: boolean` - True if HDR metadata is set
+
+### Instance Methods
+
+#### `toJSON(): VideoColorSpaceInit`
+
+Serialize to JSON-compatible object.
+
+### HDR Metadata Types
+
+**HdrMetadata:**
+```typescript
+interface HdrMetadata {
+  smpteSt2086?: SmpteSt2086Metadata;   // Mastering display metadata
+  contentLightLevel?: ContentLightLevelInfo;  // Content light levels
+}
+```
+
+**SmpteSt2086Metadata (Mastering Display):**
+| Property | Type | Description |
+|----------|------|-------------|
+| `primaryRChromaticityX/Y` | number | Red primary chromaticity |
+| `primaryGChromaticityX/Y` | number | Green primary chromaticity |
+| `primaryBChromaticityX/Y` | number | Blue primary chromaticity |
+| `whitePointChromaticityX/Y` | number | White point chromaticity |
+| `maxLuminance` | number | Maximum luminance (nits) |
+| `minLuminance` | number | Minimum luminance (nits) |
+
+**ContentLightLevelInfo:**
+| Property | Type | Description |
+|----------|------|-------------|
+| `maxCLL` | number | Maximum Content Light Level (nits) |
+| `maxFALL` | number | Maximum Frame Average Light Level (nits) |
+
+### Helper Functions
+
+#### `createHdr10MasteringMetadata(maxLuminance, minLuminance?): SmpteSt2086Metadata`
+
+Create HDR10 mastering metadata with BT.2020 primaries.
+
+```typescript
+const metadata = createHdr10MasteringMetadata(1000, 0.0001);
+```
+
+#### `createContentLightLevel(maxCLL, maxFALL): ContentLightLevelInfo`
+
+Create content light level info.
+
+```typescript
+const cll = createContentLightLevel(800, 400);
+```
+
+#### `HDR10_DISPLAY_PRIMARIES`
+
+Constant with standard BT.2020 display primaries.
 
 ---
 
@@ -496,7 +651,7 @@ const info = await mediaCapabilities.encodingInfo({
 
 ### Capability Profiles
 
-You can generate a hardware-specific capability profile to make `mediaCapabilities` match your actual FFmpeg install. Use the provided CLI:
+You can generate a hardware-specific capability profile to make `mediaCapabilities` match your actual system capabilities. Use the provided CLI:
 
 ```bash
 npm run capabilities:generate -- ./webcodecs-capabilities.json

package/docs/configuration.md

@@ -5,6 +5,7 @@ This guide covers all configuration options for video and audio encoding in webc
 ## Table of Contents
 
 - [Bitrate Mode](#bitrate-mode)
+- [Non-standard Quality Overrides](#non-standard-quality-overrides)
 - [Alpha Channel Handling](#alpha-channel-handling)
 - [Latency Mode](#latency-mode)
 - [Hardware Acceleration](#hardware-acceleration)
@@ -93,16 +94,31 @@ encoder.configure({
 - Unpredictable file size
 - Not suitable for streaming
 
-### Codec-Specific Implementation
+---
+
+## Non-standard Quality Overrides
+
+`crf` and `preset` are **extensions** (not part of the WebCodecs spec). They are loaded globally from a JS file and passed to FFmpeg when supported.
 
-Different codecs use different parameters for each bitrate mode:
+Edit `ffmpeg-quality.js` in the project root:
+
+```javascript
+export default {
+  // Global overrides:
+  // crf: 28,
+  // preset: 'veryfast',
+
+  // Per-codec overrides:
+  // perCodec: {
+  //   h264: { crf: 30, preset: 'veryfast' },
+  //   hevc: { crf: 28, preset: 'medium' },
+  // },
+};
+```
 
-| Codec | CBR | VBR | Quantizer |
-|-------|-----|-----|-----------|
-| H.264 | `-b:v` + `maxrate`/`bufsize`, CBR mode | `-b:v` only | `-crf 23` |
-| H.265 | `-b:v` + `maxrate`/`bufsize`, CBR mode | `-b:v` only | `-crf 28` |
-| VP8/VP9 | `-b:v` + `minrate`/`maxrate` | `-b:v` only | `-crf 31` + `-b:v 0` |
-| AV1 | `-b:v` + `maxrate`/`bufsize` | `-b:v` only | `-crf 30` |
+**Notes:**
+- If the file is missing, no overrides are applied.
+- The file is loaded from `process.cwd()` or from `WEB_CODECS_FFMPEG_QUALITY`.
 
 ---
 
@@ -131,7 +147,7 @@ The `alpha` option controls how transparent pixels are handled during encoding.
 
 ## Output Bitstream Format
 
-By default the encoders emit Annex B (video) and ADTS/OGG (audio) streams directly from FFmpeg. If you need MP4-style payloads (length-prefixed NAL units, raw AAC frames) you can opt-in via the `format` config field.
+By default the encoders emit Annex B (video) and ADTS/OGG (audio) bitstreams. If you need MP4-style payloads (length-prefixed NAL units, raw AAC frames) you can opt-in via the `format` config field.
 
 ### VideoEncoder `format`
 
@@ -266,11 +282,11 @@ encoder.configure({
 
 | Codec | Quality Mode | Realtime Mode |
 |-------|--------------|---------------|
-| H.264 | Default (B-frames, lookahead) | `-tune zerolatency`, no B-frames |
-| H.265 | Default settings | `-tune zerolatency`, no B-frames |
-| VP8 | Default | `-deadline realtime`, `-cpu-used 8` |
-| VP9 | Row multithreading, tile columns | `-deadline realtime`, `-cpu-used 8` |
-| AV1 | Default | `-usage realtime`, `-cpu-used 8` |
+| H.264 | B-frames, lookahead enabled | Zero-latency tuning, no B-frames |
+| H.265 | Default settings | Zero-latency tuning, no B-frames |
+| VP8 | Default | Realtime deadline, fast encoding |
+| VP9 | Row multithreading, tile columns | Realtime deadline, fast encoding |
+| AV1 | Default | Realtime usage, fast encoding |
 
 **Example for live streaming:**
 

package/examples/README.md

@@ -22,6 +22,8 @@ npx tsx examples/transparent-video.ts
 npx tsx examples/streaming.ts
 npx tsx examples/hardware-encoding.ts
 npx tsx examples/hardware-decoding.ts
+npx tsx examples/canvas-encoding.ts
+npx tsx examples/offscreen-canvas.ts
 ```
 
 ## Examples
@@ -87,6 +89,27 @@ GPU-accelerated decoding. Demonstrates:
 - Benchmarking hardware vs software decoding
 - Real-time decoding capability analysis
 
+### canvas-encoding.ts
+
+GPU-accelerated canvas rendering with skia-canvas. Demonstrates:
+- GPU acceleration detection (Metal/Vulkan/D3D)
+- Creating GPU-accelerated canvas with `createCanvas()`
+- Using `FrameLoop` with backpressure for smooth encoding
+- Animated canvas drawing with gradients and shapes
+- Proper memory lifecycle (frame closing)
+- Real-time encoding speed measurement
+
+### offscreen-canvas.ts
+
+Browser-compatible OffscreenCanvas API. Demonstrates:
+- Using `OffscreenCanvasPolyfill` (matches browser API)
+- Installing polyfill globally with `installOffscreenCanvasPolyfill()`
+- `ImageDataPolyfill` with `Uint8ClampedArray`
+- `createPixelBuffer()` utilities for pixel manipulation
+- `convertToBlob()` for PNG/JPEG/WebP export
+- `validateEvenDimensions()` for YUV420 compatibility
+- Direct `VideoFrame` creation from OffscreenCanvas
+
 ## Additional Demos
 
 The `demos/` folder contains more complete demos that can be run via npm scripts:
@@ -95,6 +118,10 @@ The `demos/` folder contains more complete demos that can be run via npm scripts
 npm run demo:webcodecs      # Basic WebCodecs demo
 npm run demo:image          # Image decoding demo
 npm run demo:streaming      # Streaming comparison
-npm run demo:conversion     # File conversion with Mediabunny
 npm run demo:hwaccel        # Hardware acceleration detection
+npm run demo:fourcorners    # Video quadrant compositor
+npm run demo:containers     # Container demuxing/muxing
+npm run demo:1080p          # 1080p transcoding benchmark
+npm run demo:dvd            # Bouncing DVD logo animation
+npm run demo:visualizer     # Audio visualizer with waveform
 ```

package/examples/canvas-encoding.ts

@@ -0,0 +1,222 @@
+/**
+ * Canvas Encoding Example
+ *
+ * Demonstrates GPU-accelerated canvas rendering with skia-canvas
+ * and encoding to H.264 video using the FrameLoop helper.
+ *
+ * Features shown:
+ * - GPU acceleration detection (Metal/Vulkan/D3D)
+ * - Creating GPU-accelerated canvas
+ * - FrameLoop with backpressure for smooth encoding
+ * - Animated canvas drawing
+ * - Proper memory lifecycle (frame closing)
+ *
+ * Run: npx tsx examples/canvas-encoding.ts
+ */
+
+import {
+  VideoEncoder,
+  EncodedVideoChunk,
+  createCanvas,
+  createFrameLoop,
+  detectGpuAcceleration,
+  isGpuAvailable,
+  getGpuApi,
+  ensureEvenDimensions,
+} from '../src/index.js';
+
+async function main() {
+  // ============================================
+  // 1. Detect GPU Acceleration
+  // ============================================
+  console.log('=== GPU Detection ===\n');
+
+  const gpuInfo = detectGpuAcceleration();
+  console.log(`Renderer: ${gpuInfo.renderer}`);
+  console.log(`GPU Available: ${isGpuAvailable()}`);
+
+  const gpuApi = getGpuApi();
+  if (gpuApi) {
+    console.log(`GPU API: ${gpuApi}`);
+    if (gpuInfo.device) {
+      console.log(`Device: ${gpuInfo.device}`);
+    }
+  } else {
+    console.log('GPU API: None (CPU fallback)');
+  }
+
+  // ============================================
+  // 2. Setup Video Parameters
+  // ============================================
+  console.log('\n=== Video Setup ===\n');
+
+  // Ensure even dimensions for YUV420 compatibility
+  const { width, height } = ensureEvenDimensions(1280, 720);
+  const frameRate = 30;
+  const totalFrames = 90; // 3 seconds
+  const maxQueueSize = 8; // Backpressure limit
+
+  console.log(`Resolution: ${width}x${height}`);
+  console.log(`Frame Rate: ${frameRate} fps`);
+  console.log(`Total Frames: ${totalFrames}`);
+  console.log(`Duration: ${(totalFrames / frameRate).toFixed(1)}s`);
+
+  // ============================================
+  // 3. Create Encoder
+  // ============================================
+  const chunks: EncodedVideoChunk[] = [];
+  let encodedCount = 0;
+
+  const encoder = new VideoEncoder({
+    output: (chunk, metadata) => {
+      chunks.push(chunk);
+      encodedCount++;
+
+      if (metadata?.decoderConfig) {
+        console.log('\nDecoder config received');
+      }
+    },
+    error: (err) => {
+      console.error('Encoding error:', err);
+    },
+  });
+
+  encoder.configure({
+    codec: 'avc1.42001f', // H.264 Baseline Level 3.1
+    width,
+    height,
+    bitrate: 5_000_000, // 5 Mbps
+    framerate: frameRate,
+    bitrateMode: 'variable',
+  });
+
+  // ============================================
+  // 4. Create FrameLoop with Canvas Drawing
+  // ============================================
+  console.log('\n=== Encoding ===\n');
+
+  const startTime = Date.now();
+
+  const loop = createFrameLoop({
+    width,
+    height,
+    frameRate,
+    maxQueueSize,
+    gpu: isGpuAvailable(), // Use GPU if available
+
+    // This callback draws each frame
+    onFrame: (ctx, timing) => {
+      const { frameIndex, timestamp } = timing;
+      const t = frameIndex / totalFrames; // Progress 0-1
+
+      // Background gradient that shifts over time
+      const gradient = ctx.createLinearGradient(0, 0, width, height);
+      gradient.addColorStop(0, `hsl(${(frameIndex * 3) % 360}, 70%, 20%)`);
+      gradient.addColorStop(1, `hsl(${(frameIndex * 3 + 180) % 360}, 70%, 40%)`);
+      ctx.fillStyle = gradient;
+      ctx.fillRect(0, 0, width, height);
+
+      // Animated circles
+      const numCircles = 5;
+      for (let i = 0; i < numCircles; i++) {
+        const phase = (i / numCircles) * Math.PI * 2;
+        const x = width / 2 + Math.cos(frameIndex * 0.05 + phase) * 200;
+        const y = height / 2 + Math.sin(frameIndex * 0.07 + phase) * 150;
+        const radius = 30 + Math.sin(frameIndex * 0.1 + i) * 20;
+
+        ctx.beginPath();
+        ctx.arc(x, y, radius, 0, Math.PI * 2);
+        ctx.fillStyle = `hsla(${(i * 72 + frameIndex * 2) % 360}, 80%, 60%, 0.7)`;
+        ctx.fill();
+      }
+
+      // Central pulsing circle
+      const pulseRadius = 80 + Math.sin(frameIndex * 0.15) * 30;
+      ctx.beginPath();
+      ctx.arc(width / 2, height / 2, pulseRadius, 0, Math.PI * 2);
+      ctx.fillStyle = 'rgba(255, 255, 255, 0.3)';
+      ctx.fill();
+
+      // Frame counter text
+      ctx.fillStyle = 'white';
+      ctx.font = 'bold 24px sans-serif';
+      ctx.textAlign = 'left';
+      ctx.textBaseline = 'top';
+      ctx.fillText(`Frame: ${frameIndex + 1}/${totalFrames}`, 20, 20);
+
+      // Timestamp
+      ctx.font = '18px sans-serif';
+      ctx.fillText(`Time: ${(timestamp / 1_000_000).toFixed(3)}s`, 20, 50);
+
+      // GPU status
+      ctx.textAlign = 'right';
+      ctx.fillText(`GPU: ${gpuApi || 'CPU'}`, width - 20, 20);
+    },
+  });
+
+  // ============================================
+  // 5. Generate and Encode Frames
+  // ============================================
+
+  // Start frame generation
+  loop.start(totalFrames);
+
+  // Process frames as they become available
+  let processedCount = 0;
+
+  while (loop.getState() !== 'stopped' || loop.getQueueSize() > 0) {
+    const frame = loop.takeFrame();
+
+    if (frame) {
+      try {
+        // Request keyframe every second
+        const keyFrame = processedCount % frameRate === 0;
+        encoder.encode(frame, { keyFrame });
+        processedCount++;
+
+        // Progress indicator
+        if (processedCount % 10 === 0 || processedCount === totalFrames) {
+          process.stdout.write(`\rProcessed: ${processedCount}/${totalFrames} frames`);
+        }
+      } finally {
+        // Always close the frame to free memory
+        frame.close();
+      }
+    } else {
+      // No frame available, wait briefly
+      await new Promise((resolve) => setTimeout(resolve, 1));
+    }
+  }
+
+  console.log('\n');
+
+  // ============================================
+  // 6. Flush and Close
+  // ============================================
+  await encoder.flush();
+  encoder.close();
+
+  const endTime = Date.now();
+  const elapsed = (endTime - startTime) / 1000;
+
+  // ============================================
+  // 7. Results
+  // ============================================
+  console.log('=== Results ===\n');
+
+  const totalBytes = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+  const keyFrames = chunks.filter((c) => c.type === 'key').length;
+  const deltaFrames = chunks.filter((c) => c.type === 'delta').length;
+  const actualBitrate = (totalBytes * 8) / (totalFrames / frameRate);
+
+  console.log(`Total chunks: ${chunks.length}`);
+  console.log(`Key frames: ${keyFrames}`);
+  console.log(`Delta frames: ${deltaFrames}`);
+  console.log(`Total size: ${(totalBytes / 1024).toFixed(2)} KB`);
+  console.log(`Actual bitrate: ${(actualBitrate / 1000).toFixed(0)} kbps`);
+  console.log(`Encoding time: ${elapsed.toFixed(2)}s`);
+  console.log(`Speed: ${(totalFrames / elapsed).toFixed(1)} fps`);
+  console.log(`Realtime: ${((totalFrames / frameRate) / elapsed).toFixed(2)}x`);
+}
+
+main().catch(console.error);