realtime-avatar 0.1.0

package/dist/browser/player.d.ts

@@ -0,0 +1,162 @@
+import type { RealtimeTurnStream } from "../client";
+import type { AvatarTurnEventSource } from "../session-socket";
+import { type PlayoutDelay } from "./audio";
+/** True when this browser can hardware-decode the h264 avatar stream. */
+export declare function supportsH264Playback(): boolean;
+export type AvatarPlayerState = "idle" | "thinking" | "speaking" | "done" | "error";
+export type AvatarPlayerMetrics = {
+    /** Total video frames received this turn. */
+    frames: number;
+    /** Frames dropped to stay in sync with the audio clock. */
+    droppedFrames: number;
+    /** ms from turn start to the first audio chunk. */
+    firstAudioMs: number | null;
+    /** ms from turn start to the first video frame. */
+    firstVideoMs: number | null;
+    /** ms from turn start to the first frame DRAWN on the running audio clock —
+     *  the user-perceived time-to-first-frame (audio is playing by then). */
+    firstFrameDrawnMs: number | null;
+    /** Currently buffered audio, in ms. */
+    audioQueueMs: number;
+    /** Accumulated audio underrun (gaps the scheduler had to paper over), in ms. */
+    audioUnderrunMs: number;
+    width: number | null;
+    height: number | null;
+};
+export type AvatarPlayHandlers = {
+    /** Streaming assistant text: `delta` is the new text, `full` the accumulated reply. */
+    onText?: (delta: string, full: string) => void;
+    onState?: (state: AvatarPlayerState) => void;
+    onMetrics?: (metrics: AvatarPlayerMetrics) => void;
+};
+/** Final source-video playback position from the server's `done` event. */
+export type AvatarSourceVideoState = {
+    /** Ping-pong cursor; pass back as the next turn's `source_start_frame`. */
+    cursor: number;
+    /** Source frame index the render stopped on. */
+    index: number;
+    direction: "forward" | "reverse";
+    /** Source frame count in the cache. */
+    frames: number;
+};
+export type AvatarPlaySummary = {
+    text: string;
+    frames: number;
+    elapsedMs: number;
+    metrics: AvatarPlayerMetrics;
+    /** Present on source-video turns: where to resume the idle loop seamlessly. */
+    sourceVideo?: AvatarSourceVideoState;
+};
+export type AvatarPlayerOptions = {
+    sampleRate?: number;
+    /**
+     * `"adaptive"` (default) starts audio the moment the first video frame is
+     * decodable (or after a short cap for audio-only turns), minimizing
+     * perceived first-frame latency. A number reproduces the legacy fixed
+     * playout delay in ms.
+     */
+    playoutDelayMs?: PlayoutDelay;
+};
+/**
+ * Renders an avatar turn stream to a canvas with audio-clocked video playback.
+ *
+ * This is the piece every integrator would otherwise hand-roll: it schedules
+ * PCM audio, decodes/queues video frames, and drives a `requestAnimationFrame`
+ * clock that draws each frame on its audio-aligned presentation time, dropping
+ * late frames so lip-sync never drifts.
+ *
+ *     const player = new AvatarPlayer();
+ *     player.attach(canvasEl);
+ *     await player.play(await session.chat("who are you?"));
+ */
+export declare class AvatarPlayer {
+    private readonly sampleRate;
+    private readonly playoutDelay;
+    private canvas;
+    private scheduler;
+    private renderer;
+    private queue;
+    private pendingJpeg;
+    private decoding;
+    /** Lazily-configured WebCodecs decoder for `codec: "h264"` streams. */
+    private videoDecoder;
+    private rafHandle;
+    private state;
+    private metricsState;
+    /** Persistent, gesture-unlocked AudioContext reused across turns. */
+    private audioContext;
+    /** Lazily-created recording tap; every scheduled audio buffer also feeds it. */
+    private audioTap;
+    private playStartedAt;
+    constructor(options?: AvatarPlayerOptions);
+    /** Bind (or rebind) the canvas the player draws into. */
+    attach(canvas: HTMLCanvasElement | null): void;
+    /**
+     * Unlock audio from inside a user gesture (click/tap). Browsers start an
+     * AudioContext suspended and only let it resume during a gesture; turns that
+     * fire later (e.g. a livestream auto-reaction from a timer/effect) would
+     * otherwise play silently. Call this from the first user interaction. Safe to
+     * call repeatedly. Resolves once the context is running (or no-ops off-DOM).
+     */
+    unlock(): Promise<void>;
+    get metrics(): AvatarPlayerMetrics;
+    /**
+     * A live `MediaStream` carrying everything the player schedules to the
+     * speakers — pair it with `canvas.captureStream()` + `MediaRecorder` to
+     * record a turn exactly as it played. The tap is additive (speaker output is
+     * unchanged) and persists across turns. Returns `null` until the player has
+     * an AudioContext — call `unlock()` (any user gesture) first.
+     */
+    captureAudioStream(): MediaStream | null;
+    /**
+     * Play a turn to completion. Accepts anything that exposes turn events —
+     * an HTTP `RealtimeTurnStream` or a `RealtimeSessionSocket.turn()` source.
+     * Resolves with a summary when the stream ends. Pass the same `signal` used
+     * to create the stream so `stop()` and the network abort together.
+     */
+    play(stream: RealtimeTurnStream | AvatarTurnEventSource, handlers?: AvatarPlayHandlers & {
+        signal?: AbortSignal;
+    }): Promise<AvatarPlaySummary>;
+    /**
+     * After the stream ends, keep the audio clock + render loop running until the
+     * buffered audio has fully played out and the video queue has drained, so the
+     * tail of the turn actually renders instead of freezing on the last frame.
+     */
+    private drainPlayback;
+    /** Stop playback: cancel the clock, close audio, clear the queue and canvas. */
+    stop(): void;
+    /** Fully release resources, including the persistent AudioContext. Call on
+     *  unmount; after this, unlock() must be called again before the next turn. */
+    dispose(): void;
+    private handleEvent;
+    private queueVideo;
+    /**
+     * Decodes the pending-JPEG queue into bitmaps with bounded concurrency, off
+     * the stream loop. Concurrency matters: serial `await createImageBitmap` for
+     * tall (e.g. 832px) frames can exceed the per-frame realtime budget, so decode
+     * falls permanently behind the audio clock and the render queue starves —
+     * which is exactly the "audio plays but video freezes" stall. Decoding a few
+     * frames in parallel keeps throughput ahead of realtime.
+     *
+     * Frames are inserted into the render queue in pts order regardless of which
+     * decode finishes first, and we never drop here — the audio-clocked render
+     * loop is the single place that drops late frames.
+     */
+    private pumpJpegDecode;
+    /**
+     * Feed h264 access units to a WebCodecs decoder. Each server chunk is a
+     * self-contained Annex-B stream (AUD + SPS/PPS + IDR + P-frames), so any
+     * chunk can start decode and dropped chunks never corrupt later ones.
+     * Decoded VideoFrames land in the same pts-ordered queue the render clock
+     * already drains; hardware decode happens off the event loop.
+     */
+    private queueH264;
+    /** Insert a decoded frame into the render queue keeping it sorted by pts.
+     *  A frame in the queue means there is something to lip-sync against, so
+     *  this is also where adaptive playout releases held audio. */
+    private insertOrdered;
+    private startVideoClock;
+    private draw;
+    private setState;
+}
+//# sourceMappingURL=player.d.ts.map

package/dist/browser/player.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"player.d.ts","sourceRoot":"","sources":["../../src/browser/player.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAuB,KAAK,YAAY,EAAE,MAAM,SAAS,CAAC;AAcjE,yEAAyE;AACzE,wBAAgB,oBAAoB,IAAI,OAAO,CAE9C;AAED,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpF,MAAM,MAAM,mBAAmB,GAAG;IAChC,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,mDAAmD;IACnD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B;6EACyE;IACzE,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,uCAAuC;IACvC,YAAY,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACpD,CAAC;AAEF,2EAA2E;AAC3E,MAAM,MAAM,sBAAsB,GAAG;IACnC,2EAA2E;IAC3E,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IACjC,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,mBAAmB,CAAC;IAC7B,+EAA+E;IAC/E,WAAW,CAAC,EAAE,sBAAsB,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,YAAY,CAAC;CAC/B,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,SAAS,CAAoC;IACrD,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,QAAQ,CAAS;IACzB,uEAAuE;IACvE,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,YAAY,CAAuC;IAC3D,qEAAqE;IACrE,OAAO,CAAC,YAAY,CAA6B;IACjD,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAgD;IAChE,OAAO,CAAC,aAAa,CAAK;gBAEd,OAAO,GAAE,mBAAwB;IAK7C,yDAAyD;IACzD,MAAM,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI,GAAG,IAAI;IAI9C;;;;;;OAMG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAY7B,IAAI,OAAO,IAAI,mBAAmB,CAEjC;IAED;;;;;;OAMG;IACH,kBAAkB,IAAI,WAAW,GAAG,IAAI;IAOxC;;;;;OAKG;IACG,IAAI,CACR,MAAM,EAAE,kBAAkB,GAAG,qBAAqB,EAClD,QAAQ,GAAE,kBAAkB,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO,GAC3D,OAAO,CAAC,iBAAiB,CAAC;IA+C7B;;;;OAIG;YACW,aAAa;IAoB3B,gFAAgF;IAChF,IAAI,IAAI,IAAI;IAsBZ;mFAC+E;IAC/E,OAAO,IAAI,IAAI;YAQD,WAAW;IA4DzB,OAAO,CAAC,UAAU;IA0ClB;;;;;;;;;;;OAWG;YACW,cAAc;IA2B5B;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;IA+CjB;;mEAE+D;IAC/D,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,eAAe;IA8BvB,OAAO,CAAC,IAAI;IAaZ,OAAO,CAAC,QAAQ;CAKjB"}

package/dist/browser/player.js

@@ -0,0 +1,514 @@
+import { Pcm16AudioScheduler } from "./audio";
+import { I420CanvasRenderer } from "./yuv-canvas";
+/** True when this browser can hardware-decode the h264 avatar stream. */
+export function supportsH264Playback() {
+    return typeof VideoDecoder === "function" && typeof EncodedVideoChunk === "function";
+}
+/**
+ * Renders an avatar turn stream to a canvas with audio-clocked video playback.
+ *
+ * This is the piece every integrator would otherwise hand-roll: it schedules
+ * PCM audio, decodes/queues video frames, and drives a `requestAnimationFrame`
+ * clock that draws each frame on its audio-aligned presentation time, dropping
+ * late frames so lip-sync never drifts.
+ *
+ *     const player = new AvatarPlayer();
+ *     player.attach(canvasEl);
+ *     await player.play(await session.chat("who are you?"));
+ */
+export class AvatarPlayer {
+    sampleRate;
+    playoutDelay;
+    canvas = null;
+    scheduler = null;
+    renderer = null;
+    queue = [];
+    pendingJpeg = [];
+    decoding = false;
+    /** Lazily-configured WebCodecs decoder for `codec: "h264"` streams. */
+    videoDecoder = null;
+    rafHandle = null;
+    state = "idle";
+    metricsState = emptyMetrics();
+    /** Persistent, gesture-unlocked AudioContext reused across turns. */
+    audioContext = null;
+    /** Lazily-created recording tap; every scheduled audio buffer also feeds it. */
+    audioTap = null;
+    playStartedAt = 0;
+    constructor(options = {}) {
+        this.sampleRate = options.sampleRate ?? 16_000;
+        this.playoutDelay = options.playoutDelayMs ?? "adaptive";
+    }
+    /** Bind (or rebind) the canvas the player draws into. */
+    attach(canvas) {
+        this.canvas = canvas;
+    }
+    /**
+     * Unlock audio from inside a user gesture (click/tap). Browsers start an
+     * AudioContext suspended and only let it resume during a gesture; turns that
+     * fire later (e.g. a livestream auto-reaction from a timer/effect) would
+     * otherwise play silently. Call this from the first user interaction. Safe to
+     * call repeatedly. Resolves once the context is running (or no-ops off-DOM).
+     */
+    async unlock() {
+        const AudioContextCtor = typeof window !== "undefined"
+            ? window.AudioContext || window.webkitAudioContext
+            : undefined;
+        if (!AudioContextCtor)
+            return;
+        this.audioContext ??= new AudioContextCtor({ latencyHint: "interactive" });
+        if (this.audioContext.state !== "running") {
+            await this.audioContext.resume().catch(() => { });
+        }
+    }
+    get metrics() {
+        return { ...this.metricsState };
+    }
+    /**
+     * A live `MediaStream` carrying everything the player schedules to the
+     * speakers — pair it with `canvas.captureStream()` + `MediaRecorder` to
+     * record a turn exactly as it played. The tap is additive (speaker output is
+     * unchanged) and persists across turns. Returns `null` until the player has
+     * an AudioContext — call `unlock()` (any user gesture) first.
+     */
+    captureAudioStream() {
+        const context = this.audioContext;
+        if (!context || typeof context.createMediaStreamDestination !== "function")
+            return null;
+        this.audioTap ??= context.createMediaStreamDestination();
+        return this.audioTap.stream;
+    }
+    /**
+     * Play a turn to completion. Accepts anything that exposes turn events —
+     * an HTTP `RealtimeTurnStream` or a `RealtimeSessionSocket.turn()` source.
+     * Resolves with a summary when the stream ends. Pass the same `signal` used
+     * to create the stream so `stop()` and the network abort together.
+     */
+    async play(stream, handlers = {}) {
+        this.stop();
+        const startMs = now();
+        this.playStartedAt = startMs;
+        const signal = handlers.signal;
+        this.metricsState = emptyMetrics();
+        this.scheduler = new Pcm16AudioScheduler(this.sampleRate, this.playoutDelay, this.audioContext, {}, this.audioTap);
+        await this.scheduler.prepare();
+        this.setState("thinking", handlers);
+        this.startVideoClock();
+        let text = "";
+        let frames = 0;
+        let elapsedMs = 0;
+        let sourceVideo;
+        let streamErrored = false;
+        try {
+            for await (const event of stream.events) {
+                if (signal?.aborted)
+                    break;
+                const handled = await this.handleEvent(event, startMs, text, handlers);
+                text = handled.text;
+                if (handled.frames !== undefined)
+                    frames = handled.frames;
+                if (handled.elapsedMs !== undefined)
+                    elapsedMs = handled.elapsedMs;
+                if (handled.sourceVideo)
+                    sourceVideo = handled.sourceVideo;
+                if (handled.done)
+                    break;
+            }
+        }
+        catch (error) {
+            streamErrored = true;
+            throw error;
+        }
+        finally {
+            // CRITICAL: the network stream ends well before playback does — there is
+            // ~playoutDelayMs of buffered audio plus a video queue still waiting for
+            // its presentation time. Do NOT tear the scheduler down here, or the
+            // render clock loses its time source and the buffered video tail freezes
+            // on the last drawn frame while the audio keeps playing. Instead keep the
+            // clock alive and drain to the end of playback first (unless aborted or
+            // the stream errored).
+            if (!signal?.aborted && !streamErrored) {
+                await this.drainPlayback(signal);
+            }
+            this.scheduler?.close();
+            this.scheduler = null;
+        }
+        this.setState("done", handlers);
+        return { text, frames, elapsedMs: elapsedMs || now() - startMs, metrics: this.metrics, sourceVideo };
+    }
+    /**
+     * After the stream ends, keep the audio clock + render loop running until the
+     * buffered audio has fully played out and the video queue has drained, so the
+     * tail of the turn actually renders instead of freezing on the last frame.
+     */
+    async drainPlayback(signal) {
+        const scheduler = this.scheduler;
+        if (!scheduler)
+            return;
+        const deadline = now() + 12_000; // hard cap so we never hang
+        while (now() < deadline) {
+            if (signal?.aborted)
+                return;
+            // Finish decoding anything still pending (JPEG pool or WebCodecs queue).
+            if (this.pendingJpeg.length || this.decoding || (this.videoDecoder?.decodeQueueSize ?? 0) > 0) {
+                await sleep(30);
+                continue;
+            }
+            const mediaTime = scheduler.mediaTimeSeconds;
+            const lastPts = this.queue.length ? this.queue[this.queue.length - 1].pts : null;
+            const audioRemainingMs = scheduler.queuedMs;
+            const videoDrained = lastPts === null || (mediaTime !== null && mediaTime >= lastPts);
+            if (videoDrained && audioRemainingMs <= 30)
+                return; // everything played out
+            await sleep(30);
+        }
+    }
+    /** Stop playback: cancel the clock, close audio, clear the queue and canvas. */
+    stop() {
+        if (this.rafHandle !== null) {
+            cancelAnimationFrame(this.rafHandle);
+            this.rafHandle = null;
+        }
+        this.queue.forEach(closeQueuedFrame);
+        this.queue = [];
+        this.pendingJpeg = [];
+        if (this.videoDecoder && this.videoDecoder.state !== "closed") {
+            try {
+                this.videoDecoder.close();
+            }
+            catch {
+                // already closing
+            }
+        }
+        this.videoDecoder = null;
+        this.scheduler?.close();
+        this.scheduler = null;
+        this.renderer?.reset();
+        clearCanvas(this.canvas);
+    }
+    /** Fully release resources, including the persistent AudioContext. Call on
+     *  unmount; after this, unlock() must be called again before the next turn. */
+    dispose() {
+        this.stop();
+        this.audioTap = null; // bound to the context being closed below
+        const context = this.audioContext;
+        this.audioContext = null;
+        if (context && context.state !== "closed")
+            void context.close();
+    }
+    async handleEvent(event, startMs, priorText, handlers) {
+        const header = event.header;
+        switch (header.type) {
+            case "start":
+                this.setState("speaking", handlers);
+                return { text: priorText };
+            case "text_delta": {
+                const delta = new TextDecoder().decode(event.payload);
+                const text = priorText + delta;
+                handlers.onText?.(delta, text);
+                return { text };
+            }
+            case "text_done": {
+                const decoded = new TextDecoder().decode(event.payload);
+                const text = decoded || priorText;
+                if (decoded)
+                    handlers.onText?.("", text);
+                return { text };
+            }
+            case "audio": {
+                const scheduled = await this.scheduler?.schedule(event.payload);
+                this.metricsState.firstAudioMs ??= Math.round(now() - startMs);
+                if (scheduled) {
+                    this.metricsState.audioQueueMs = scheduled.queuedMs;
+                    if (scheduled.underrunMs > 0)
+                        this.metricsState.audioUnderrunMs += scheduled.underrunMs;
+                }
+                this.setState("speaking", handlers);
+                handlers.onMetrics?.(this.metrics);
+                return { text: priorText };
+            }
+            case "video":
+                this.queueVideo(header, event.payload);
+                this.metricsState.firstVideoMs ??= Math.round(now() - startMs);
+                handlers.onMetrics?.(this.metrics);
+                return { text: priorText };
+            case "done":
+                return {
+                    text: priorText,
+                    frames: header.frames,
+                    elapsedMs: header.elapsedMs,
+                    done: true,
+                    sourceVideo: header.sourceVideo,
+                };
+            case "error":
+                throw new Error(header.message);
+            default:
+                return { text: priorText };
+        }
+    }
+    queueVideo(header, payload) {
+        this.metricsState.width = header.width;
+        this.metricsState.height = header.height;
+        if (header.pixelFormat === "h264") {
+            this.queueH264(header, payload);
+            return;
+        }
+        if (header.pixelFormat === "jpeg") {
+            // Enqueue COPIES of each frame's bytes and decode them off the event loop.
+            // Two reasons this must be a copy, not a subarray view:
+            //   1. the stream's read buffer is reused, so a deferred decode of a view
+            //      could read bytes that a later chunk has already overwritten;
+            //   2. decoding inline here (await per frame) would block the stream's
+            //      for-await loop, starving the audio scheduler and stalling playback.
+            let offset = 0;
+            for (let i = 0; i < header.frames; i += 1) {
+                const size = header.frameSizes?.[i] ?? 0;
+                this.pendingJpeg.push({
+                    pts: (header.startFrame + i) / header.fps,
+                    bytes: payload.slice(offset, offset + size),
+                    width: header.width,
+                    height: header.height,
+                });
+                offset += size;
+            }
+            this.metricsState.frames += header.frames;
+            void this.pumpJpegDecode();
+            return;
+        }
+        const frameBytes = header.frameBytes ?? 0;
+        const frames = [];
+        for (let i = 0; i < header.frames; i += 1) {
+            const start = i * frameBytes;
+            // i420 frames are drawn straight from the queue; copy so a reused stream
+            // buffer can't corrupt a not-yet-drawn frame.
+            frames.push({ kind: "i420", pts: (header.startFrame + i) / header.fps, frame: payload.slice(start, start + frameBytes), width: header.width, height: header.height });
+        }
+        this.queue.push(...frames);
+        this.metricsState.frames += frames.length;
+        if (frames.length)
+            this.scheduler?.startPlayout();
+    }
+    /**
+     * Decodes the pending-JPEG queue into bitmaps with bounded concurrency, off
+     * the stream loop. Concurrency matters: serial `await createImageBitmap` for
+     * tall (e.g. 832px) frames can exceed the per-frame realtime budget, so decode
+     * falls permanently behind the audio clock and the render queue starves —
+     * which is exactly the "audio plays but video freezes" stall. Decoding a few
+     * frames in parallel keeps throughput ahead of realtime.
+     *
+     * Frames are inserted into the render queue in pts order regardless of which
+     * decode finishes first, and we never drop here — the audio-clocked render
+     * loop is the single place that drops late frames.
+     */
+    async pumpJpegDecode() {
+        if (this.decoding)
+            return;
+        this.decoding = true;
+        const CONCURRENCY = 4;
+        try {
+            while (this.pendingJpeg.length) {
+                const batch = this.pendingJpeg.splice(0, CONCURRENCY);
+                const decoded = await Promise.all(batch.map(async (frame) => {
+                    try {
+                        const bitmap = await decodeJpegFrame(frame.bytes);
+                        return { kind: "bitmap", pts: frame.pts, bitmap, width: frame.width, height: frame.height };
+                    }
+                    catch {
+                        return null;
+                    }
+                }));
+                for (const frame of decoded) {
+                    if (frame)
+                        this.insertOrdered(frame);
+                    else
+                        this.metricsState.droppedFrames += 1;
+                }
+            }
+        }
+        finally {
+            this.decoding = false;
+        }
+    }
+    /**
+     * Feed h264 access units to a WebCodecs decoder. Each server chunk is a
+     * self-contained Annex-B stream (AUD + SPS/PPS + IDR + P-frames), so any
+     * chunk can start decode and dropped chunks never corrupt later ones.
+     * Decoded VideoFrames land in the same pts-ordered queue the render clock
+     * already drains; hardware decode happens off the event loop.
+     */
+    queueH264(header, payload) {
+        if (!supportsH264Playback()) {
+            this.metricsState.droppedFrames += header.frames;
+            return;
+        }
+        if (!this.videoDecoder || this.videoDecoder.state === "closed") {
+            this.videoDecoder = new VideoDecoder({
+                output: (frame) => {
+                    this.insertOrdered({
+                        kind: "videoframe",
+                        pts: frame.timestamp / 1_000_000,
+                        frame,
+                        width: header.width,
+                        height: header.height,
+                    });
+                },
+                error: () => {
+                    // A decoder fault drops the rest of this turn's h264 frames; audio
+                    // keeps playing and the next turn reconfigures a fresh decoder.
+                    this.videoDecoder = null;
+                },
+            });
+            // Annex-B is implied when no description is attached. The stream is
+            // baseline profile level 3.1 (see avtr1_modal/realtime/h264.py).
+            this.videoDecoder.configure({ codec: "avc1.42001f", optimizeForLatency: true });
+        }
+        let offset = 0;
+        for (let i = 0; i < header.frames; i += 1) {
+            const size = header.frameSizes?.[i] ?? 0;
+            const bytes = payload.slice(offset, offset + size);
+            offset += size;
+            try {
+                this.videoDecoder.decode(new EncodedVideoChunk({
+                    // Chunk-leading AUs carry the IDR; the rest are P-frames.
+                    type: i === 0 ? "key" : "delta",
+                    timestamp: Math.round(((header.startFrame + i) / header.fps) * 1_000_000),
+                    data: bytes,
+                }));
+            }
+            catch {
+                this.metricsState.droppedFrames += 1;
+            }
+        }
+        this.metricsState.frames += header.frames;
+    }
+    /** Insert a decoded frame into the render queue keeping it sorted by pts.
+     *  A frame in the queue means there is something to lip-sync against, so
+     *  this is also where adaptive playout releases held audio. */
+    insertOrdered(frame) {
+        this.scheduler?.startPlayout();
+        const queue = this.queue;
+        if (queue.length === 0 || queue[queue.length - 1].pts <= frame.pts) {
+            queue.push(frame);
+            return;
+        }
+        let lo = 0;
+        let hi = queue.length;
+        while (lo < hi) {
+            const mid = (lo + hi) >> 1;
+            if (queue[mid].pts <= frame.pts)
+                lo = mid + 1;
+            else
+                hi = mid;
+        }
+        queue.splice(lo, 0, frame);
+    }
+    startVideoClock() {
+        if (this.rafHandle !== null)
+            cancelAnimationFrame(this.rafHandle);
+        const tick = () => {
+            const mediaTime = this.scheduler?.mediaTimeSeconds;
+            const canvas = this.canvas;
+            if (mediaTime !== null && mediaTime !== undefined && canvas) {
+                let drawable = null;
+                let dropped = 0;
+                while (this.queue.length && this.queue[0].pts <= mediaTime + 0.03) {
+                    if (drawable) {
+                        closeQueuedFrame(drawable);
+                        dropped += 1;
+                    }
+                    drawable = this.queue.shift() ?? null;
+                }
+                while (this.queue.length > 36 && this.queue[0].pts < mediaTime - 0.08) {
+                    closeQueuedFrame(this.queue.shift() ?? null);
+                    dropped += 1;
+                }
+                if (dropped)
+                    this.metricsState.droppedFrames += dropped;
+                if (drawable) {
+                    this.metricsState.firstFrameDrawnMs ??= Math.round(now() - this.playStartedAt);
+                    this.draw(canvas, drawable);
+                }
+            }
+            this.rafHandle = requestAnimationFrame(tick);
+        };
+        this.rafHandle = requestAnimationFrame(tick);
+    }
+    draw(canvas, frame) {
+        if (frame.kind === "bitmap") {
+            drawBitmapFrame(canvas, frame);
+            return;
+        }
+        if (frame.kind === "videoframe") {
+            drawVideoFrame(canvas, frame);
+            return;
+        }
+        this.renderer ??= new I420CanvasRenderer();
+        this.renderer.draw(canvas, frame.frame, frame.width, frame.height, Math.round(frame.pts * 1_000_000));
+    }
+    setState(state, handlers) {
+        if (this.state === state)
+            return;
+        this.state = state;
+        handlers.onState?.(state);
+    }
+}
+function emptyMetrics() {
+    return {
+        frames: 0,
+        droppedFrames: 0,
+        firstAudioMs: null,
+        firstVideoMs: null,
+        firstFrameDrawnMs: null,
+        audioQueueMs: 0,
+        audioUnderrunMs: 0,
+        width: null,
+        height: null,
+    };
+}
+function closeQueuedFrame(frame) {
+    if (frame?.kind === "bitmap")
+        frame.bitmap.close();
+    if (frame?.kind === "videoframe")
+        frame.frame.close();
+}
+function drawVideoFrame(canvas, frame) {
+    if (canvas.width !== frame.width)
+        canvas.width = frame.width;
+    if (canvas.height !== frame.height)
+        canvas.height = frame.height;
+    const context = canvas.getContext("2d", { alpha: false });
+    context?.drawImage(frame.frame, 0, 0, frame.width, frame.height);
+    frame.frame.close();
+}
+function clearCanvas(canvas) {
+    if (!canvas)
+        return;
+    const context = canvas.getContext("2d");
+    context?.clearRect(0, 0, canvas.width, canvas.height);
+    // eslint-disable-next-line no-self-assign -- reset draw state cheaply
+    canvas.width = canvas.width;
+}
+function drawBitmapFrame(canvas, frame) {
+    if (canvas.width !== frame.width)
+        canvas.width = frame.width;
+    if (canvas.height !== frame.height)
+        canvas.height = frame.height;
+    const context = canvas.getContext("2d", { alpha: false });
+    context?.drawImage(frame.bitmap, 0, 0, frame.width, frame.height);
+    frame.bitmap.close();
+}
+async function decodeJpegFrame(bytes) {
+    if (typeof createImageBitmap !== "function") {
+        throw new Error("This browser does not support realtime JPEG frame decoding.");
+    }
+    // `.slice()` does a single fast buffer copy (the Blob must own the bytes,
+    // since the underlying stream buffer is reused); avoid element-wise copies.
+    return createImageBitmap(new Blob([bytes.slice()], { type: "image/jpeg" }));
+}
+function now() {
+    return typeof performance !== "undefined" ? performance.now() : Date.now();
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=player.js.map