react-native-tv-audio-description 0.1.0

package/src/vega/SegmentBuffer.ts

@@ -0,0 +1,195 @@
+import type { AssetSegment, AssetSource } from '../MediaAdapter';
+import { log } from '../log';
+
+/**
+ * Keep a bounded window of the asset in the SourceBuffer, and never the whole
+ * thing.
+ *
+ * Two platform facts shape this and neither was guessable (PLATFORM.md):
+ *   - `url_mode_broken` — the player fetches nothing, so the app delivers
+ *     every byte;
+ *   - `no_range_requests` — a Range request returns the WHOLE file with status
+ *     200, so the app cannot read part of one. The window is therefore made of
+ *     whole segments cut at build time. A window built on byte ranges would
+ *     appear to work and hold the entire film in memory.
+ *
+ * The window is measured in TIME, never in segment count. ffmpeg cuts on
+ * keyframes, so a clip asked for 6 s segments came out 9.94 s, 4.17 s, 5.29 s
+ * and 0.65 s — "keep five segments" would mean anything between 3 and 50
+ * seconds of media, which is not a memory bound.
+ */
+
+/**
+ * Only what this class actually uses, declared structurally.
+ *
+ * `MediaSource.addSourceBuffer` is typed as returning the W3C `SourceBuffer`
+ * interface, which does not declare `addEventListener` — but the
+ * implementation behind it does, and it was measured to work on the device.
+ * Naming the five members this class touches says what it needs and fails
+ * loudly if any of them disappears.
+ */
+export interface AppendTarget {
+  readonly updating: boolean;
+  appendBuffer(data: Uint8Array): void;
+  remove(start: number, end: number): void;
+  addEventListener(type: string, listener: () => void): void;
+  removeEventListener(type: string, listener: () => void): void;
+}
+
+export interface BufferWindow {
+  /** media kept ahead of the playhead (default 30 s) */
+  aheadMs: number;
+  /** media kept behind it before eviction (default 10 s) */
+  behindMs: number;
+}
+
+export const DEFAULT_WINDOW: BufferWindow = { aheadMs: 30_000, behindMs: 10_000 };
+
+/** the two members of MediaSource this class touches, for the same reason */
+export interface SourceLike {
+  readonly readyState: string;
+  endOfStream(): void;
+}
+
+export class SegmentBuffer {
+  private appended = new Set<number>();
+  private queue: Promise<void> = Promise.resolve();
+  private disposed = false;
+
+  constructor(
+    private readonly source: SourceLike,
+    private readonly buffer: AppendTarget,
+    private readonly asset: AssetSource,
+    private readonly window: BufferWindow = DEFAULT_WINDOW,
+  ) {}
+
+  /**
+   * Append the init segment, then enough of the asset to start playing.
+   *
+   * Both steps are queued NOW, behind anything already queued, so a caller
+   * that feeds positions before `start()` settles can neither overlap two
+   * SourceBuffer operations nor have position 0 processed after a later one.
+   * Unlike a segment, a failed init REJECTS: without it nothing after it can
+   * decode, and the caller has to say so.
+   */
+  async start(): Promise<void> {
+    const initUri = this.asset.initUri;
+    let init: Promise<void> = Promise.resolve();
+    if (initUri) {
+      init = this.queue.then(async () => {
+        await this.append(await this.fetchBytes(initUri), 'init');
+      });
+      this.queue = init.catch(() => undefined);
+    }
+    const first = this.ensure(0);
+    await init;
+    await first;
+  }
+
+  /**
+   * Bring the window up to date for a playhead position.
+   *
+   * Safe to call on every `timeupdate`: segments already appended are
+   * skipped, and the work is serialised behind one promise chain because a
+   * SourceBuffer accepts exactly one operation at a time. Calling
+   * `appendBuffer` while `updating` is true throws `InvalidStateError`, which
+   * on Vega surfaces as a media error indistinguishable from a bad file.
+   */
+  ensure(positionMs: number): Promise<void> {
+    this.queue = this.queue
+      .then(() => this.sync(positionMs))
+      .catch((err) => {
+        log(`buffer.failed err=${(err as Error).message}`);
+      });
+    return this.queue;
+  }
+
+  /** segment indexes currently held, for diagnostics and tests */
+  held(): number[] {
+    return [...this.appended].sort((a, b) => a - b);
+  }
+
+  private async sync(positionMs: number): Promise<void> {
+    if (this.disposed) return;
+
+    for (const segment of this.wanted(positionMs)) {
+      if (this.appended.has(segment.index)) continue;
+      const bytes = await this.fetchBytes(segment.uri);
+      if (this.disposed) return;
+      await this.append(bytes, `seg${segment.index}`);
+      this.appended.add(segment.index);
+    }
+
+    await this.evict(positionMs);
+
+    // `endOfStream` only once the last segment is in, or the player waits
+    // forever for media that is not coming.
+    const last = this.asset.segments[this.asset.segments.length - 1];
+    // and never while the SourceBuffer is mid-operation: that throws, and the
+    // platform reports it the same way as a bad file
+    if (
+      last &&
+      this.appended.has(last.index) &&
+      this.source.readyState === 'open' &&
+      !this.buffer.updating
+    ) {
+      this.source.endOfStream();
+      log(`buffer.complete segments=${this.appended.size}`);
+    }
+  }
+
+  /** every segment overlapping [position, position + ahead] */
+  private wanted(positionMs: number): AssetSegment[] {
+    return this.asset.segments.filter(
+      (s) => s.end_ms > positionMs && s.start_ms <= positionMs + this.window.aheadMs,
+    );
+  }
+
+  private async evict(positionMs: number): Promise<void> {
+    const cutoff = positionMs - this.window.behindMs;
+    if (cutoff <= 0) return;
+
+    const stale = this.asset.segments.filter(
+      (s) => this.appended.has(s.index) && s.end_ms < cutoff,
+    );
+    if (stale.length === 0) return;
+
+    const end = Math.max(...stale.map((s) => s.end_ms));
+    await this.operation(() => this.buffer.remove(0, end / 1000));
+    for (const s of stale) this.appended.delete(s.index);
+
+    log(`buffer.evicted n=${stale.length} up_to_ms=${end} held=${this.appended.size}`);
+  }
+
+  private async fetchBytes(uri: string): Promise<Uint8Array> {
+    const response = await fetch(uri);
+    return new Uint8Array(await response.arrayBuffer());
+  }
+
+  private async append(bytes: Uint8Array, label: string): Promise<void> {
+    await this.operation(() => this.buffer.appendBuffer(bytes));
+    log(`buffer.appended ${label} bytes=${bytes.byteLength} held=${this.appended.size}`);
+  }
+
+  /** run one SourceBuffer operation and wait for `updateend` */
+  private operation(run: () => void): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+      const done = () => {
+        this.buffer.removeEventListener('updateend', done);
+        resolve();
+      };
+      this.buffer.addEventListener('updateend', done);
+      try {
+        run();
+      } catch (err) {
+        this.buffer.removeEventListener('updateend', done);
+        reject(err as Error);
+      }
+    });
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    this.appended.clear();
+  }
+}

package/src/vega/index.tsx

@@ -0,0 +1,382 @@
+import React, { useCallback } from 'react';
+import { AppState } from 'react-native';
+import {
+  AudioContentType,
+  AudioPlayer,
+  AudioUsageType,
+  KeplerVideoSurfaceView,
+  MediaSource,
+  VideoPlayer as VegaVideoPlayer,
+} from '@amazon-devices/react-native-w3cmedia';
+import type {
+  AppLifecycle,
+  AssetSource,
+  ClipPlayer,
+  MediaAdapter,
+  Unsubscribe,
+  VideoPlayer,
+  VideoSurfaceProps,
+} from '../MediaAdapter';
+import { DEFAULT_WINDOW, SegmentBuffer, type AppendTarget, type BufferWindow } from './SegmentBuffer';
+import { log } from '../log';
+
+export {
+  SegmentBuffer,
+  DEFAULT_WINDOW,
+  type AppendTarget,
+  type BufferWindow,
+  type SourceLike,
+} from './SegmentBuffer';
+
+/**
+ * The Vega OS implementation of `MediaAdapter`.
+ *
+ * EXTRACTED from code that already ran on the Vega Virtual Device, not written
+ * from the interface down. Each of the following is a bug somebody already
+ * paid for (details in PLATFORM.md):
+ *   - `srcObject`, never `src` — `url_mode_broken`: assigning a URL to `src`
+ *     fails before a single byte is requested;
+ *   - dual readiness tracking — `surface_races_init`: the surface can arrive
+ *     before `initialize()` resolves, and playing before both have landed
+ *     yields MEDIA_ERR_SRC_NOT_SUPPORTED, which reads exactly like an
+ *     unsupported file and is not one;
+ *   - fragmented MP4 through MediaSource only — `mse_path`;
+ *   - clips on an AudioPlayer built for accessibility speech, as a second
+ *     stream over the film — `concurrent_streams`.
+ */
+
+export interface VegaAdapterOptions {
+  /**
+   * The MSE type of YOUR asset, with full codec parameters. A bare
+   * 'video/mp4' is rejected at addSourceBuffer. Default: H.264 Constrained
+   * Baseline L3.0 + AAC-LC. Read yours with
+   * `ffprobe -show_streams` and build the RFC 6381 string.
+   */
+  videoMime?: string;
+  /** the MSE type of the description clips (default fragmented AAC-LC) */
+  clipMime?: string;
+  /** how much media to hold around the playhead */
+  window?: BufferWindow;
+}
+
+const DEFAULT_VIDEO_MIME = 'video/mp4; codecs="avc1.42C01E,mp4a.40.2"';
+const DEFAULT_CLIP_MIME = 'audio/mp4; codecs="mp4a.40.2"';
+
+type Listener = () => void;
+
+/** whole-file append, for description CLIPS — they are seconds long, not films */
+function fetchAndAppend(source: MediaSource, mime: string, uri: string): Promise<number> {
+  return new Promise<number>((resolve, reject) => {
+    source.addEventListener('sourceopen', () => {
+      (async () => {
+        try {
+          const buffer = source.addSourceBuffer(mime);
+
+          const response = await fetch(uri);
+          const bytes = new Uint8Array(await response.arrayBuffer());
+
+          (buffer as unknown as AppendTarget).addEventListener('updateend', () => {
+            if (source.readyState !== 'open') return;
+            source.endOfStream();
+            resolve(bytes.byteLength);
+          });
+
+          buffer.appendBuffer(bytes);
+        } catch (err) {
+          reject(err as Error);
+        }
+      })().catch(reject);
+    });
+  });
+}
+
+class VegaVideo implements VideoPlayer {
+  readonly player = new VegaVideoPlayer();
+
+  private ready = { player: false, surface: false };
+  private pendingPlay: (() => void) | null = null;
+  private listeners = new Map<string, Set<Listener>>();
+  private initialised: Promise<void> | null = null;
+  private lastError: Error | null = null;
+  private segments: SegmentBuffer | null = null;
+
+  constructor(
+    private readonly mime: string,
+    private readonly window: BufferWindow,
+  ) {}
+
+  private async initialize(): Promise<void> {
+    if (!this.initialised) {
+      this.initialised = this.player.initialize().then(() => {
+        this.player.addEventListener('error', () => {
+          const code = this.player.error?.code ?? -1;
+          const msg = (this.player.error as { message?: string } | null)?.message ?? 'none';
+          log(`player.error code=${code} msg=${msg}`);
+          this.lastError = new Error(`media error ${code}: ${msg}`);
+          this.emit('error');
+        });
+        for (const type of ['timeupdate', 'seeked', 'waiting', 'stalled', 'playing', 'ended'] as const) {
+          this.player.addEventListener(type, () => this.emit(type));
+        }
+      });
+    }
+    return this.initialised;
+  }
+
+  private emit(type: string): void {
+    for (const cb of this.listeners.get(type) ?? []) cb();
+  }
+
+  private on(type: string, cb: Listener): Unsubscribe {
+    const set = this.listeners.get(type) ?? new Set<Listener>();
+    set.add(cb);
+    this.listeners.set(type, set);
+    return () => set.delete(cb);
+  }
+
+  /** called by the surface component when the platform hands over a handle */
+  attachSurface(handle: string): void {
+    log(`player.surface created handle=${handle}`);
+    this.player.setSurfaceHandle(handle);
+    this.ready.surface = true;
+    this.release();
+  }
+
+  detachSurface(handle: string): void {
+    log(`player.surface destroyed handle=${handle}`);
+    this.player.clearSurfaceHandle(handle);
+    this.ready.surface = false;
+  }
+
+  /** whichever of the two readiness signals lands second lets playback start */
+  private release(): void {
+    if (!this.ready.player || !this.ready.surface) return;
+    const go = this.pendingPlay;
+    this.pendingPlay = null;
+    go?.();
+  }
+
+  async open(asset: AssetSource): Promise<void> {
+    await this.initialize();
+
+    const source = new MediaSource();
+
+    const opened = new Promise<void>((resolve, reject) => {
+      let handled = false;
+      source.addEventListener('sourceopen', () => {
+        // ONCE. Per MSE, appendBuffer() or remove() on an 'ended' source
+        // reopens it and fires 'sourceopen' again — and evicting behind the
+        // playhead after the last segment is in does exactly that. Handling it
+        // twice builds a second SourceBuffer and re-appends the whole asset,
+        // which evicts, which reopens: a loop, measured on the Virtual Device
+        // at the end of a 20 s clip (PLATFORM.md `sourceopen_refires`).
+        if (handled) return;
+        handled = true;
+        (async () => {
+          const buffer = source.addSourceBuffer(this.mime);
+          this.segments = new SegmentBuffer(
+            source,
+            buffer as unknown as AppendTarget,
+            asset,
+            this.window,
+          );
+          await this.segments.start();
+          resolve();
+        })().catch(reject);
+      });
+    });
+
+    // srcObject, NOT src — see PLATFORM.md `url_mode_broken`.
+    this.player.srcObject = source;
+
+    await opened;
+    log(`player.opened segments=${asset.segments.length} init=${asset.initUri ? 'yes' : 'none'}`);
+
+    // Keep the window fed as the playhead moves. The bounded window is what
+    // stops a feature-length asset becoming a feature-length ArrayBuffer.
+    this.on('timeupdate', () => {
+      void this.segments?.ensure(this.positionMs());
+    });
+
+    this.ready.player = true;
+    this.release();
+  }
+
+  play(): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+      const start = () => {
+        this.player
+          .play()
+          .then(() => {
+            log('player.play resolved');
+            resolve();
+          })
+          .catch(reject);
+      };
+      if (this.ready.player && this.ready.surface) start();
+      else this.pendingPlay = start; // the surface race, handled once, here
+    });
+  }
+
+  pause(): void {
+    this.player.pause();
+  }
+
+  positionMs(): number {
+    // currentTime is NaN until the player has media; a NaN position would
+    // reach the scheduler as a seek to nowhere
+    const t = this.player.currentTime;
+    return Number.isFinite(t) ? Math.round(t * 1000) : 0;
+  }
+
+  durationMs(): number {
+    const d = this.player.duration;
+    return Number.isFinite(d) ? Math.round(d * 1000) : 0;
+  }
+
+  isPlaying(): boolean {
+    return !this.player.paused;
+  }
+
+  setVolumePct(pct: number): void {
+    this.player.volume = Math.max(0, Math.min(100, pct)) / 100;
+  }
+
+  onPosition(cb: (ms: number) => void): Unsubscribe {
+    return this.on('timeupdate', () => cb(this.positionMs()));
+  }
+
+  onSeek(cb: (ms: number) => void): Unsubscribe {
+    return this.on('seeked', () => cb(this.positionMs()));
+  }
+
+  onStalled(cb: () => void): Unsubscribe {
+    const offWaiting = this.on('waiting', cb);
+    const offStalled = this.on('stalled', cb);
+    return () => {
+      offWaiting();
+      offStalled();
+    };
+  }
+
+  onPlaying(cb: () => void): Unsubscribe {
+    return this.on('playing', cb);
+  }
+
+  onEnded(cb: () => void): Unsubscribe {
+    return this.on('ended', cb);
+  }
+
+  onError(cb: (err: Error) => void): Unsubscribe {
+    return this.on('error', () => cb(this.lastError ?? new Error('media error')));
+  }
+
+  async destroy(): Promise<void> {
+    this.segments?.dispose();
+    this.segments = null;
+    this.listeners.clear();
+    this.ready = { player: false, surface: false };
+    this.pendingPlay = null;
+    await this.player.deinitialize().catch(() => {
+      // best effort during teardown; must not throw into React's cleanup path
+    });
+  }
+}
+
+class VegaClips implements ClipPlayer {
+  private current: AudioPlayer | null = null;
+  /** settles the clip that is playing; set for exactly as long as one is */
+  private interrupt: (() => void) | null = null;
+
+  constructor(private readonly mime: string) {}
+
+  async play(uri: string): Promise<void> {
+    const player = new AudioPlayer(
+      AudioContentType.CONTENT_TYPE_SPEECH,
+      AudioUsageType.USAGE_ACCESSIBILITY,
+    );
+    this.current = player;
+
+    try {
+      await player.initialize();
+
+      if (!MediaSource.isTypeSupported(this.mime)) {
+        throw new Error(`cue container unsupported: ${this.mime}`);
+      }
+
+      const source = new MediaSource();
+      const appended = fetchAndAppend(source, this.mime, uri);
+      player.srcObject = source;
+      const bytes = await appended;
+      log(`cue.audio bytes=${bytes}`);
+
+      await new Promise<void>((resolve, reject) => {
+        // A paused AudioPlayer never emits 'ended', so without this a stopped
+        // clip's promise never settles and its player is never torn down —
+        // one leaked player per interrupted cue, measured on the Virtual Device.
+        this.interrupt = () => {
+          log('cue.audio state=stopped');
+          resolve();
+        };
+        player.addEventListener('error', () => {
+          const code = player.error?.code ?? -1;
+          reject(new Error(`cue media error ${code}`));
+        });
+        player.addEventListener('ended', () => {
+          log(`cue.audio state=ended t=${player.currentTime.toFixed(2)}`);
+          resolve();
+        });
+        player.play().catch(reject);
+      });
+    } finally {
+      this.current = null;
+      this.interrupt = null;
+      // Not awaited: tearing the clip player down took ~130 ms on the Virtual
+      // Device, and every one of them kept the film ducked after the speech
+      // had ended. A cue that cannot be torn down must not stop the film.
+      void player.deinitialize().catch(() => undefined);
+    }
+  }
+
+  stop(): void {
+    this.current?.pause();
+    this.interrupt?.();
+  }
+}
+
+class VegaLifecycle implements AppLifecycle {
+  onBackground(cb: () => void): Unsubscribe {
+    const sub = AppState.addEventListener('change', (state) => {
+      if (state !== 'active') cb();
+    });
+    return () => sub.remove();
+  }
+}
+
+export function createVegaAdapter(options: VegaAdapterOptions = {}): MediaAdapter {
+  const video = new VegaVideo(
+    options.videoMime ?? DEFAULT_VIDEO_MIME,
+    options.window ?? DEFAULT_WINDOW,
+  );
+
+  function VideoSurface({ style }: VideoSurfaceProps) {
+    const onCreated = useCallback((handle: string) => video.attachSurface(handle), []);
+    const onDestroyed = useCallback((handle: string) => video.detachSurface(handle), []);
+
+    return (
+      <KeplerVideoSurfaceView
+        style={style}
+        scalingmode="fit"
+        onSurfaceViewCreated={onCreated}
+        onSurfaceViewDestroyed={onDestroyed}
+      />
+    );
+  }
+
+  return {
+    video,
+    clips: new VegaClips(options.clipMime ?? DEFAULT_CLIP_MIME),
+    lifecycle: new VegaLifecycle(),
+    VideoSurface,
+  };
+}