react-native-tv-audio-description 0.1.0

package/src/DescriptionAudio.ts

@@ -0,0 +1,97 @@
+import type { MediaAdapter } from './MediaAdapter';
+import { AD } from './budget';
+import type { DescriptionCue } from './track';
+import { rampVolumePct } from './duck';
+import { log } from './log';
+
+export interface DescriptionAudioOptions {
+  /** main track level while a cue speaks, % of full (default 25) */
+  duckTargetPct?: number;
+  /** fade length into and out of the duck (default 200) */
+  rampMs?: number;
+}
+
+/**
+ * Duck, speak, restore.
+ *
+ * The cue plays as a SECOND stream over the film rather than by switching to
+ * a pre-mixed track. On Vega that was measured to work: a video player and an
+ * audio player built for accessibility speech ran simultaneously with zero
+ * dropped frames (PLATFORM.md `concurrent_streams`).
+ */
+export class DescriptionAudio {
+  private active = false;
+  /**
+   * Bumped by `stop()`. A cue in flight when the screen unmounts keeps running
+   * — its `await` chain does not know the component is gone — and its
+   * `finally` would then ramp the volume on a player that has been destroyed,
+   * after `stop()` already restored it.
+   */
+  private generation = 0;
+  private readonly duckPct: number;
+  private readonly rampMs: number;
+
+  constructor(
+    private readonly media: MediaAdapter,
+    options: DescriptionAudioOptions = {},
+  ) {
+    this.duckPct = options.duckTargetPct ?? AD.DUCK_TARGET_PCT;
+    this.rampMs = options.rampMs ?? AD.DUCK_RAMP_MS;
+    // Backgrounding during a cue must stop the clip and restore the main
+    // level, or the app returns to the foreground ducked and silent.
+    this.media.lifecycle.onBackground(() => {
+      if (!this.active) return;
+      log('audio.background active=true');
+      void this.stop();
+    });
+  }
+
+  async speak(cue: DescriptionCue): Promise<void> {
+    if (this.active) {
+      // a cue already speaking is never interrupted by another — but the one
+      // that lost is said out loud in the log, not dropped in silence
+      log(`audio.busy id=${cue.id}`);
+      return;
+    }
+    this.active = true;
+    const mine = this.generation;
+    const stale = () => mine !== this.generation;
+
+    try {
+      log(`audio.duck id=${cue.id} to_pct=${this.duckPct}`);
+      await rampVolumePct(this.media.video, 100, this.duckPct, this.rampMs, stale);
+      await this.media.clips.play(cue.audio_uri);
+      log(stale() ? `audio.interrupted id=${cue.id}` : `audio.spoke id=${cue.id} words=${cue.words}`);
+    } catch (err) {
+      log(`audio.failed id=${cue.id} err=${(err as Error).message}`);
+    } finally {
+      // The main track ALWAYS returns to full, including on failure. A cue
+      // that fails must not leave the film at 25% for the rest of the runtime.
+      //
+      // Unless stop() already did it: then this cue is stale, the player may
+      // be torn down, and ramping again is work against a dead object.
+      if (!stale()) {
+        await rampVolumePct(this.media.video, this.duckPct, 100, this.rampMs, stale);
+        log(`audio.restored id=${cue.id}`);
+        this.active = false;
+      }
+    }
+  }
+
+  /**
+   * Stop the cue that is speaking, if any, and bring the film back to full.
+   * Safe to call at any time — turning description off, leaving the screen.
+   */
+  async stop(): Promise<void> {
+    this.generation++; // anything in flight is now stale and must not restore
+    this.media.clips.stop();
+    // Nothing speaking means the film is already at full. Ramping "back" from
+    // the duck level anyway would first SET it to the duck level: an audible
+    // dip every time description is switched off between cues.
+    if (!this.active) return;
+    log('audio.stopped');
+    await rampVolumePct(this.media.video, this.duckPct, 100, this.rampMs);
+    this.active = false;
+    log('audio.restored after=stop');
+  }
+}

package/src/MediaAdapter.ts

@@ -0,0 +1,136 @@
+import type { ComponentType } from 'react';
+import type { StyleProp, ViewStyle } from 'react-native';
+
+/**
+ * The only surface this library uses to reach a platform's media stack.
+ *
+ * Everything outside `src/vega/` is written against this interface and never
+ * against a platform package; a test fails if that stops being true. Bring
+ * your own implementation for another platform, or use the Vega one:
+ *
+ *   import { createVegaAdapter } from 'react-native-tv-audio-description/vega';
+ *
+ * Each platform defect the Vega implementation works around is kept OUT of
+ * this interface on purpose — a caller that had to know about them would leak
+ * one platform's defects into every other. They are listed in PLATFORM.md.
+ */
+
+export type Unsubscribe = () => void;
+
+/** one piece of an asset, with times read from the emitted playlist */
+export interface AssetSegment {
+  index: number;
+  start_ms: number;
+  end_ms: number;
+  uri: string;
+}
+
+/**
+ * What to play, and how it is cut up.
+ *
+ * ALWAYS A LIST, even for a single file — a whole-file asset is one segment
+ * with no init. One code path for both shapes means a short clip and a
+ * feature-length film exercise the same buffering logic, rather than the
+ * feature-length one taking a path nothing has ever run.
+ */
+export interface AssetSource {
+  /** appended once, before any segment; absent for a whole-file asset */
+  initUri?: string;
+  segments: AssetSegment[];
+}
+
+export interface VideoPlayer {
+  /**
+   * Point the player at an asset and get it ready to play.
+   *
+   * Implementations must NOT assume the platform will fetch anything. On Vega
+   * it will not (PLATFORM.md `url_mode_broken`) and the implementation reads
+   * the bytes itself — nor can it read PART of a file (`no_range_requests`),
+   * which is why the asset arrives already cut into segments.
+   */
+  open(source: AssetSource): Promise<void>;
+  play(): Promise<void>;
+  pause(): void;
+
+  /** current playback position, ms */
+  positionMs(): number;
+  durationMs(): number;
+  isPlaying(): boolean;
+
+  /**
+   * Set the main track volume as a percentage of full, EFFECTIVE IMMEDIATELY.
+   *
+   * There is no ramp parameter, and its absence is measured rather than
+   * chosen: the W3C volume setter is instantaneous and Vega exposes no fade
+   * (PLATFORM.md `no_volume_ramp`). The fade is implemented once, in JS, in
+   * `duck.ts`. An interface that accepted `rampMs` would invite every
+   * implementation to reimplement the same loop and would imply a capability
+   * no platform here has.
+   */
+  setVolumePct(pct: number): void;
+
+  /** fires on every position update the platform emits ('timeupdate') */
+  onPosition(cb: (ms: number) => void): Unsubscribe;
+  /** fires after a seek settles, with the new position ('seeked') */
+  onSeek(cb: (ms: number) => void): Unsubscribe;
+  /**
+   * Fires when playback cannot continue because the buffer ran dry
+   * ('waiting' / 'stalled').
+   *
+   * With the app owning byte delivery this is a REACHABLE state and it is NOT
+   * an error — no `error` event follows it. A UI that only listens for
+   * `onError` shows a frozen picture and says nothing, which for a blind
+   * viewer is indistinguishable from a quiet scene.
+   */
+  onStalled(cb: () => void): Unsubscribe;
+  /**
+   * Fires when playback is actually running ('playing').
+   *
+   * The counterpart to `onStalled`, and not optional: MSE emits `waiting` at
+   * the START of normal playback while the first frames decode, so a screen
+   * that treats stalling as terminal announces "Buffering" over a film that is
+   * playing fine (PLATFORM.md `waiting_fires_at_start`).
+   */
+  onPlaying(cb: () => void): Unsubscribe;
+  onEnded(cb: () => void): Unsubscribe;
+  onError(cb: (err: Error) => void): Unsubscribe;
+
+  /** release everything; safe to call twice */
+  destroy(): Promise<void>;
+}
+
+export interface ClipPlayer {
+  /**
+   * Plays one description clip to completion; rejects if it cannot be played.
+   * Same rule as `VideoPlayer.open`: no implementation may assume the
+   * platform fetches the URI.
+   */
+  play(uri: string): Promise<void>;
+  stop(): void;
+}
+
+export interface AppLifecycle {
+  /** fires when the app leaves the foreground */
+  onBackground(cb: () => void): Unsubscribe;
+}
+
+export interface VideoSurfaceProps {
+  style?: StyleProp<ViewStyle>;
+}
+
+export interface MediaAdapter {
+  video: VideoPlayer;
+  clips: ClipPlayer;
+  lifecycle: AppLifecycle;
+  /**
+   * The platform's own view that decoded pixels render into, already wired to
+   * `video`.
+   *
+   * It is part of the adapter rather than something a screen imports, because
+   * mounting it IS platform code: on Vega it is `KeplerVideoSurfaceView` and
+   * the handle it hands back has to reach the player — and it has to be
+   * tracked against `initialize()`, which races it (PLATFORM.md
+   * `surface_races_init`).
+   */
+  VideoSurface: ComponentType<VideoSurfaceProps>;
+}

package/src/TrackLoader.ts

@@ -0,0 +1,156 @@
+import type { DescriptionCue, DescriptionTrack, Verbosity } from './track';
+import { trackFileName, VERBOSITY_LEVELS } from './track';
+import { MAX_CLIPS_IN_MEMORY } from './budget';
+import { log } from './log';
+
+/**
+ * Load a track file and refuse to believe it.
+ *
+ * Validation here is not defensive decoration. The loader does not trust that
+ * the producer was well-behaved, and a malformed or missing track must end in
+ * a STATED visible and spoken state, never in silence. A silent app with
+ * nothing to say is indistinguishable from a working app in a quiet scene.
+ */
+
+export type LoadResult =
+  | { ok: true; track: DescriptionTrack; loaded_verbosity: Verbosity }
+  | { ok: false; reason: 'missing' | 'malformed'; detail: string };
+
+export const CUE_KEYS = [
+  'id',
+  'start_ms',
+  'end_ms',
+  'words',
+  'text',
+  'audio_uri',
+  'source_frames_ms',
+  'status',
+] as const;
+
+export const TRACK_KEYS = [
+  'version',
+  'asset_id',
+  'generated_at',
+  'source_subtitles',
+  'verbosity',
+  'model_id',
+  'cues',
+] as const;
+
+function validCue(v: unknown): v is DescriptionCue {
+  if (typeof v !== 'object' || v === null) return false;
+  const c = v as Record<string, unknown>;
+  if (CUE_KEYS.some((k) => !(k in c))) return false;
+  return (
+    typeof c.id === 'string' &&
+    typeof c.start_ms === 'number' &&
+    typeof c.end_ms === 'number' &&
+    typeof c.words === 'number' &&
+    typeof c.text === 'string' &&
+    typeof c.audio_uri === 'string' &&
+    Array.isArray(c.source_frames_ms) &&
+    c.source_frames_ms.every((n) => typeof n === 'number') &&
+    (c.status === 'ok' || c.status === 'failed')
+  );
+}
+
+/** Validates a parsed track field for field. */
+export function validateTrack(raw: unknown): LoadResult {
+  if (typeof raw !== 'object' || raw === null) {
+    return { ok: false, reason: 'malformed', detail: 'not an object' };
+  }
+  const t = raw as Record<string, unknown>;
+
+  const missing = TRACK_KEYS.filter((k) => !(k in t));
+  if (missing.length) {
+    return { ok: false, reason: 'malformed', detail: `missing ${missing.join(',')}` };
+  }
+  if (!VERBOSITY_LEVELS.includes(t.verbosity as Verbosity)) {
+    return { ok: false, reason: 'malformed', detail: `verbosity=${String(t.verbosity)}` };
+  }
+  if (!Array.isArray(t.cues) || !t.cues.every(validCue)) {
+    return { ok: false, reason: 'malformed', detail: 'cues' };
+  }
+
+  return {
+    ok: true,
+    track: raw as DescriptionTrack,
+    loaded_verbosity: t.verbosity as Verbosity,
+  };
+}
+
+/**
+ * Resolve a level by `<asset_id>.<verbosity>.track.json` beside the asset,
+ * FALLING BACK to `standard`. The fallback is what stops a missing `detailed`
+ * file from turning the feature off.
+ *
+ * `readJson` is yours: on Vega a packaged file is read with `fetch`, and the
+ * platform's own player cannot be pointed at it (PLATFORM.md
+ * `url_mode_broken`), so there is no platform loader to delegate to.
+ */
+export async function loadTrack(
+  readJson: (path: string) => Promise<unknown>,
+  assetDir: string,
+  assetId: string,
+  verbosity: Verbosity,
+): Promise<LoadResult> {
+  const levels: Verbosity[] =
+    verbosity === 'standard' ? [verbosity] : [verbosity, 'standard'];
+
+  for (const level of levels) {
+    const path = `${assetDir}/${trackFileName(assetId, level)}`;
+    let raw: unknown;
+    try {
+      raw = await readJson(path);
+    } catch {
+      log(`loader.miss path=${path}`);
+      continue;
+    }
+
+    const result = validateTrack(raw);
+    log(
+      `loader.load path=${path} ok=${result.ok}` +
+        (result.ok ? ` cues=${result.track.cues.length}` : ` reason=${result.reason}`),
+    );
+
+    // A malformed file is an error, not a reason to fall back: something
+    // produced a file that is not a track, and silently loading a different
+    // one would hide it.
+    return result;
+  }
+
+  return { ok: false, reason: 'missing', detail: trackFileName(assetId, verbosity) };
+}
+
+/**
+ * A bounded clip cache, oldest out first.
+ *
+ * It holds BYTES, not URIs, and on Vega that is forced rather than chosen:
+ * there is no URI a player will fetch (PLATFORM.md `url_mode_broken`), so
+ * every clip is read by the app and appended through a MediaSource.
+ */
+export class ClipCache {
+  private order: string[] = [];
+  private held = new Map<string, ArrayBuffer>();
+
+  constructor(private readonly max: number = MAX_CLIPS_IN_MEMORY) {}
+
+  put(uri: string, bytes: ArrayBuffer): void {
+    if (this.held.has(uri)) this.order = this.order.filter((u) => u !== uri);
+    this.held.set(uri, bytes);
+    this.order.push(uri);
+    while (this.order.length > this.max) {
+      const evicted = this.order.shift()!;
+      this.held.delete(evicted);
+      log(`cache.evict uri=${evicted} size=${this.held.size}`);
+    }
+  }
+
+  get(uri: string): ArrayBuffer | undefined {
+    return this.held.get(uri);
+  }
+
+  get size(): number {
+    return this.held.size;
+  }
+}

package/src/budget.ts

@@ -0,0 +1,61 @@
+import type { Verbosity } from './track';
+
+/**
+ * The numbers a player and a track producer have to agree on.
+ *
+ * The playback half (`DUCK_*`, `MAX_CLIPS_IN_MEMORY`) is what this library
+ * uses at runtime. The word budget is exported for whoever PRODUCES tracks: a
+ * cue longer than its gap either overruns into dialogue or gets cut off, and
+ * both are worse than a shorter sentence.
+ */
+export const AD = {
+  /** a dialogue gap shorter than this holds nothing worth saying */
+  MIN_GAP_MS: 1500,
+  /** narration pace used to turn a gap into a word count */
+  SPEAKING_RATE_WPM: 160,
+  /** how far the film is lowered while a cue speaks, as % of full */
+  DUCK_TARGET_PCT: 25,
+  /** the fade into and out of the duck; stepped in JS, see `duck.ts` */
+  DUCK_RAMP_MS: 200,
+  /** subtracted from every gap before budgeting: the duck ramp plus 100 ms */
+  BUDGET_MARGIN_MS: 300,
+} as const;
+
+/**
+ * Word TARGETS as a fraction of the physical ceiling — never multipliers of it.
+ * Every value is <= 1.0 by construction, so no level can ask for more words
+ * than the gap holds and no clamp is needed.
+ */
+export const VERBOSITY_SCALES: Record<Verbosity, number> = {
+  concise: 0.6,
+  standard: 0.85,
+  detailed: 1.0,
+};
+
+/**
+ * Description clips held in memory at once. A Fire TV Stick is a 32-bit
+ * process with a small heap, so only the next few clips are resident.
+ */
+export const MAX_CLIPS_IN_MEMORY = 8;
+
+function wordsIn(ms: number): number {
+  return Math.floor(((ms / 1000) * AD.SPEAKING_RATE_WPM) / 60);
+}
+
+/** floor((gap_ms - 300) / 1000 * 160 / 60), never negative */
+export function baseWordBudget(gapMs: number): number {
+  return Math.max(0, wordsIn(gapMs - AD.BUDGET_MARGIN_MS));
+}
+
+/** the number of words to ASK a describer for at this level */
+export function wordTarget(gapMs: number, verbosity: Verbosity): number {
+  return Math.floor(baseWordBudget(gapMs) * VERBOSITY_SCALES[verbosity]);
+}
+
+/**
+ * The number of words a cue may not exceed, at any level. It does not vary by
+ * verbosity: the gap is the gap.
+ */
+export function wordCeiling(gapMs: number): number {
+  return baseWordBudget(gapMs);
+}

package/src/duck.ts

@@ -0,0 +1,42 @@
+import type { VideoPlayer } from './MediaAdapter';
+import { AD } from './budget';
+
+/** how often the fade writes; a floor, since timers on a TV may fire late */
+const STEP_MS = 16;
+
+/**
+ * Fade the main track between two volume percentages over `rampMs`.
+ *
+ * The platform setter is instantaneous (PLATFORM.md `no_volume_ramp`), so the
+ * ramp is stepped here. Written once, in one place, because a fade duplicated
+ * per platform is a fade that differs per platform — which is also why
+ * `VideoPlayer.setVolumePct` takes no `rampMs`.
+ *
+ * Each step sets the level for the time ELAPSED, not for its step number. On
+ * the Vega Virtual Device a fade counted in 13 steps of 16 ms took about
+ * 510 ms instead of 200 — timers fire late there — and the film came back up
+ * after its dialogue window had closed (PLATFORM.md `coarse_timers`).
+ */
+export async function rampVolumePct(
+  video: VideoPlayer,
+  fromPct: number,
+  toPct: number,
+  rampMs: number = AD.DUCK_RAMP_MS,
+  /**
+   * Checked between steps. A fade is a loop that outlives the reason it
+   * started: unmount the screen mid-duck and it keeps stepping the volume of a
+   * player that is being torn down, toward a target nobody wants any more.
+   */
+  isCancelled: () => boolean = () => false,
+): Promise<void> {
+  const start = Date.now();
+  for (;;) {
+    if (isCancelled()) return;
+    const t = rampMs > 0 ? Math.min(1, (Date.now() - start) / rampMs) : 1;
+    // t reaches exactly 1, so the fade lands exactly on the target: rounding
+    // must never leave the film at 99% forever.
+    video.setVolumePct(t >= 1 ? toPct : fromPct + (toPct - fromPct) * t);
+    if (t >= 1) return;
+    await new Promise((r) => setTimeout(r, STEP_MS));
+  }
+}

package/src/index.ts

@@ -0,0 +1,42 @@
+export type {
+  AppLifecycle,
+  AssetSegment,
+  AssetSource,
+  ClipPlayer,
+  MediaAdapter,
+  Unsubscribe,
+  VideoPlayer,
+  VideoSurfaceProps,
+} from './MediaAdapter';
+
+export type { CueStatus, DescriptionCue, DescriptionTrack, Verbosity } from './track';
+export { VERBOSITY_LEVELS, trackFileName } from './track';
+
+export {
+  AD,
+  MAX_CLIPS_IN_MEMORY,
+  VERBOSITY_SCALES,
+  baseWordBudget,
+  wordCeiling,
+  wordTarget,
+} from './budget';
+
+export {
+  CueScheduler,
+  coalesce,
+  estimateCueMs,
+  type SchedulerEvents,
+  type SchedulerOptions,
+} from './CueScheduler';
+export { DescriptionAudio, type DescriptionAudioOptions } from './DescriptionAudio';
+export { rampVolumePct } from './duck';
+export {
+  ClipCache,
+  CUE_KEYS,
+  TRACK_KEYS,
+  loadTrack,
+  validateTrack,
+  type LoadResult,
+} from './TrackLoader';
+export { ADControls, stateMessage, type ADControlsProps, type ADState } from './ADControls';
+export { setLogger, type Logger } from './log';

package/src/log.ts

@@ -0,0 +1,30 @@
+/**
+ * Where this library's diagnostic lines go. Nowhere, by default.
+ *
+ * It is injectable rather than `console.log` because on Vega there is no
+ * readable JavaScript console at all — `console.log` reaches neither the
+ * device log stream nor the Metro terminal, in Release or in Debug (see
+ * PLATFORM.md, `no_js_console`). An app that needs to see these lines on a
+ * device has to ship them somewhere itself, usually an HTTP beacon to the
+ * development host, and only the app knows where that is.
+ *
+ * Lines are `area.event key=value ...` with no prefix; add your own.
+ *
+ *   setLogger(line => beacon(`MYAPP.${line}`));
+ */
+export type Logger = (line: string) => void;
+
+const silent: Logger = () => undefined;
+let sink: Logger = silent;
+
+export function setLogger(logger: Logger | null): void {
+  sink = logger ?? silent;
+}
+
+export function log(line: string): void {
+  try {
+    sink(line);
+  } catch {
+    // diagnostics must never break playback
+  }
+}

package/src/messages.ts

@@ -0,0 +1,27 @@
+import type { Verbosity } from './track';
+
+/**
+ * What the description layer is doing, as one sentence a screen reader can
+ * say. Kept apart from `ADControls` so it has no React Native dependency:
+ * any surface — a custom control, a toast, a test, a Node script — can speak
+ * the same sentences.
+ */
+
+export type ADState =
+  | { kind: 'ready'; enabled: boolean; verbosity: Verbosity; cues: number }
+  | { kind: 'missing'; detail: string }
+  | { kind: 'malformed'; detail: string };
+
+/** every branch has a sentence, and the sentence is spoken, not only shown */
+export function stateMessage(state: ADState): string {
+  switch (state.kind) {
+    case 'ready':
+      return state.enabled
+        ? `Audio description on, ${state.verbosity}, ${state.cues} descriptions`
+        : 'Audio description off';
+    case 'missing':
+      return 'No description track was found for this title. Playback continues without description.';
+    case 'malformed':
+      return 'The description track for this title could not be read. Playback continues without description.';
+  }
+}

package/src/track.ts

@@ -0,0 +1,52 @@
+/**
+ * The description track: what a producer writes and what this library plays.
+ *
+ * One file per verbosity level, because the levels do not share a cue list: a
+ * short gap that holds a useful sentence at `detailed` may hold nothing useful
+ * at `concise`, so the concise track has fewer cues, not shorter ones. In the
+ * film this was extracted from the three levels came out at 47 / 55 / 60 cues.
+ */
+
+export type Verbosity = 'concise' | 'standard' | 'detailed';
+
+export type CueStatus = 'ok' | 'failed';
+
+export interface DescriptionCue {
+  id: string;
+  /** the dialogue gap this cue may speak in; `end_ms` is exclusive */
+  start_ms: number;
+  end_ms: number;
+  words: number;
+  text: string;
+  /** the spoken clip; empty for a `failed` cue */
+  audio_uri: string;
+  /** frames the description was written from, for traceability */
+  source_frames_ms: number[];
+  /**
+   * A cue whose description could not be produced is WRITTEN as `failed`, not
+   * dropped: a player can skip a failed cue, but it cannot tell a missing cue
+   * from a gap that was never meant to hold one.
+   */
+  status: CueStatus;
+}
+
+export interface DescriptionTrack {
+  version: string;
+  asset_id: string;
+  generated_at: string;
+  source_subtitles: string;
+  verbosity: Verbosity;
+  model_id: string;
+  cues: DescriptionCue[];
+}
+
+export const VERBOSITY_LEVELS: readonly Verbosity[] = ['concise', 'standard', 'detailed'];
+
+/**
+ * The lookup rule: one file per level, named `<asset_id>.<verbosity>.track.json`
+ * beside the asset. A level switch resolves by this name and falls back to
+ * `standard` (see `loadTrack`).
+ */
+export function trackFileName(assetId: string, verbosity: Verbosity): string {
+  return `${assetId}.${verbosity}.track.json`;
+}