react-x11 2.32.0 → 2.34.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-x11",
-  "version": "2.32.0",
+  "version": "2.34.0",
   "description": "react renderer with X11 as a target",
   "main": "./src/index.js",
   "files": [

package/src/Reconciler.js

@@ -33,6 +33,7 @@ import {
 } from './nodes/scope.js';
 import { TextNode, TextChunkNode, setTextStripBelow } from './nodes/text.js';
 import { ImageNode } from './nodes/image.js';
+import { VideoNode } from './nodes/video.js';
 import { CanvasNode } from './nodes/canvas.js';
 import { TextInputNode } from './nodes/textinput.js';
 import { TextAreaNode } from './nodes/textarea.js';
@@ -102,6 +103,7 @@ export const HOST_TYPES = [
   'box',
   'text',
   'image',
+  'video',
   'canvas',
   'textinput',
   'textarea',
@@ -316,6 +318,9 @@ const HostConfig = {
       case 'image':
         node = new ImageNode(props, rootContainer);
         break;
+      case 'video':
+        node = new VideoNode(props, rootContainer);
+        break;
       case 'canvas':
         node = new CanvasNode(props, rootContainer);
         break;

package/src/a11y.js

@@ -123,6 +123,7 @@ export const ATSPI_ROLE = Object.freeze({
   INFO_BAR: 102,
   LEVEL_BAR: 103,
   BLOCK_QUOTE: 105,
+  VIDEO: 107,
   ARTICLE: 109,
   LANDMARK: 110,
   LOG: 111,
@@ -307,6 +308,9 @@ const KIND_ROLES = {
   box: ATSPI_ROLE.FILLER,
   text: ATSPI_ROLE.LABEL,
   image: ATSPI_ROLE.IMAGE,
+  // AT-SPI's own role for one, which Chromium and Firefox give `<video>` too;
+  // ARIA has none, so there is no web role name to write in its place
+  video: ATSPI_ROLE.VIDEO,
   canvas: ATSPI_ROLE.DRAWING_AREA,
   textinput: ATSPI_ROLE.ENTRY,
   textarea: ATSPI_ROLE.ENTRY,

package/src/appcontext.js

@@ -27,6 +27,7 @@ import {
   watchCompositing,
 } from './compositing.js';
 import { canEmbed } from './embedding.js';
+import { canPlayMedia } from './mediaplayback.js';
 import { hasDirectGL, watchDirectGL } from './glbackend.js';
 import { canOverlay } from './gloverlay.js';
 
@@ -59,7 +60,7 @@ export function useApp() {
   return app;
 }
 
-// 'nativeControls', 'embedding' and 'glOverlay' are properties of the
+// 'nativeControls', 'embedding', 'glOverlay' and 'mediaPlayback' are properties of the
 // backend — and 'glOverlay' of the X server too, which `createRoot` asks
 // before the first render (src/gloverlay.js) — decided before anything
 // renders and never changing after, so their subscription has nothing to
@@ -81,6 +82,7 @@ const FEATURES = {
   },
   embedding: { watch: NEVER_CHANGES, read: canEmbed },
   glOverlay: { watch: NEVER_CHANGES, read: canOverlay },
+  mediaPlayback: { watch: NEVER_CHANGES, read: canPlayMedia },
   // Everything here is a property of the **display**: what the server, the
   // compositor and the drawing pipeline can do. A desktop's own surfaces --
   // a tray, a launcher icon and what hangs off it, a notification daemon --
@@ -183,6 +185,19 @@ const FEATURES = {
  *
  * A property of the backend and the display, settled before the first
  * render, and it never changes.
+ *
+ * `'mediaPlayback'` is true when this display plays a file or URL itself —
+ * a platform player, AVFoundation on macOS — which is what `<video src>`
+ * needs. It is false on X11 and Wayland for good, where no display-server
+ * player exists, and the question to ask before rendering a `<video src>`,
+ * which refuses with one `onError` and shows its poster where the answer is
+ * no. `<video frames>`, a sink the application decodes into, needs no
+ * capability: it works everywhere, at a cost that differs.
+ *
+ * ```jsx
+ * const playback = useSupports('mediaPlayback');
+ * {playback ? <video src={file} autoPlay /> : <video frames={decoded} />}
+ * ```
  */
 export function useSupports(feature) {
   const app = useApp();

package/src/cocoa/app.js

@@ -39,6 +39,7 @@ import { CocoaColorSampler } from './screencolor.js';
 import { CocoaFilePanels } from './filepanels.js';
 import { CocoaFontManager } from './fonts.js';
 import { releaseImageUpload } from '../backend/context2d.js';
+import { CocoaPlayer, canPlay } from './player.js';
 import { CocoaSurface } from './surface.js';
 import { CocoaSymbols } from './symbols.js';
 import { CocoaWindow, FRAME_SLACK_MS } from './window.js';
@@ -217,6 +218,16 @@ export class CocoaApp {
     // The tray items (src/cocoa/statusitem.js), by the bridge's handle —
     // which is what a click event names them by.
     this._statusItems = new Map();
+    // `<video src>`'s players (src/cocoa/player.js), by the bridge's id — what
+    // a `player-*` event names them by. `createPlayer` is there only over a
+    // bridge that plays (@windowkit/appkit's player verbs): its presence is
+    // what `useSupports('mediaPlayback')` and `<video src>` ask
+    // (src/mediaplayback.js), so an app over an older bridge, or a test's
+    // fake without them, refuses `src` the way X11 does.
+    this._players = new Map();
+    if (canPlay(native)) {
+      this.createPlayer = (src, options) => new CocoaPlayer(this, src, options);
+    }
 
     // 'surface' (the measured default) or 'layers' — the retained CALayer
     // presenter, opt-in while docs/macos.md's measure-first gate is open.
@@ -245,7 +256,7 @@ export class CocoaApp {
     // the app's own bridge, so an app over a fake one (the tests) needs no
     // real bridge on the machine — the manager's default loads it only when
     // it is built standalone
-    this.fonts = new CocoaFontManager(native);
+    this.fonts = new CocoaFontManager(native, { scale: this.scale });
     // `<image src={{ symbol }}>`'s names are SF Symbols here (src/symbols.js)
     this.symbols = new CocoaSymbols(native);
 
@@ -598,6 +609,42 @@ export class CocoaApp {
     return new CocoaSurface(this, options);
   }
 
+  /**
+   * The formats a `VideoFrames` sink is cheapest in here, first to last
+   * (`preferredFormats`, src/videoframes.js). Over a bridge with video
+   * surfaces a YCbCr frame goes onto a layer as it is and into a bitmap
+   * through the bridge's own conversion, so NV12 — what a hardware decoder
+   * emits — is first; without them every frame is converted in JavaScript on
+   * its way to a `putImageData`, and BGRA is the one that costs least.
+   */
+  videoFormats() {
+    if (typeof this._native.writeVideoSurface !== 'function') return ['BGRA'];
+    return ['NV12', 'I420', 'BGRA'];
+  }
+
+  /**
+   * A `<video>`'s frame into `surface` — its drawn presentation, a
+   * `CocoaSurface` at the frame's size — in the bridge's code: a copy for
+   * BGRA, and for YCbCr a conversion that keeps the colours Core Animation
+   * shows the same frame in on a layer (`writeVideoSurface`), so a video
+   * moving between the two does not change shade. False over a bridge
+   * without the verb, and the frame is converted in JavaScript instead
+   * (src/nodes/video.js).
+   */
+  writeVideoFrame(surface, sink, frame) {
+    const native = this._native;
+    const handle = surface?._surfaceHandle;
+    if (typeof native.writeVideoSurface !== 'function' || !handle) return false;
+    native.writeVideoSurface(handle, sink.format, frame.planes, {
+      strides: frame.strides,
+      width: sink.width,
+      height: sink.height,
+      colorSpace: sink.colorSpace,
+      range: sink.range,
+    });
+    return true;
+  }
+
   /**
    * The release seam for an `Image`'s upload (`freeImage`, src/imagesource.js).
    * `ctx.drawImage(image)` here composites from a CG bitmap made for the
@@ -1332,6 +1379,12 @@ export class CocoaApp {
         return this._window(ev)?._created(ev);
       case 'surface-released':
         return this._routeSurfaceReleased(ev);
+      case 'player-metadata':
+      case 'player-state':
+      case 'player-time':
+      case 'player-ended':
+      case 'player-error':
+        return this._players.get(ev.id)?._event(ev);
       default:
         return undefined;
     }

package/src/cocoa/fonts.js

@@ -735,9 +735,25 @@ function spacedFeatures(features, spacing) {
 const KEPT_TYPESETTER_UNITS = 1 << 20;
 
 export class CocoaFontManager {
-  /** @param native the @windowkit/appkit module; the tests hand in a fake */
-  constructor(native = loadNative()) {
+  /**
+   * @param native the @windowkit/appkit module; the tests hand in a fake
+   * @param options.scale the app's backing scale. A size this engine is
+   *   handed is in device pixels, and CoreText reads some of a face's data
+   *   by point size — San Francisco's optical size and tracking, Apple Color
+   *   Emoji's `trak` — so a font made at the device size is the face as it
+   *   is set at twice the size on a 2x display: a 13px label 8% narrower
+   *   than AppKit sets it, a 19px emoji 1em wide where it is 23pt at 19pt.
+   *   The bridge makes a font at `size / scale` points under a matrix of
+   *   `scale` instead (windowkit/appkit#106), and answers in device pixels
+   *   as before. A bridge older than that ignores the scale and sets the
+   *   text as it always did.
+   */
+  constructor(native = loadNative(), { scale = 1 } = {}) {
     this._native = native;
+    // the trailing argument cgFontWithSize and fontByPostScriptName take,
+    // and nothing at 1x, so a 1x app asks the bridge what it always asked
+    this._scale = scale > 0 && Number.isFinite(scale) ? scale : 1;
+    this._scaleArgs = this._scale === 1 ? [] : [this._scale];
     this._fonts = new Map(); // family|weight|italic|size -> handle
     // handle -> features key -> the handle with those features set
     this._featured = new WeakMap();
@@ -895,8 +911,15 @@ export class CocoaFontManager {
     const sizedKey = `${key}|${size}|${variations ? JSON.stringify(variations) : ''}`;
     let handle = this._sized.get(sizedKey);
     if (handle) return handle;
-    if (entry.ps) handle = this._native.fontByPostScriptName(entry.ps, size);
-    else if (entry.cg) handle = this._native.cgFontWithSize(entry.cg, size);
+    if (entry.ps) {
+      handle = this._native.fontByPostScriptName(
+        entry.ps,
+        size,
+        ...this._scaleArgs,
+      );
+    } else if (entry.cg) {
+      handle = this._native.cgFontWithSize(entry.cg, size, ...this._scaleArgs);
+    }
     if (!handle) return null;
     handle = this._withVariations(handle, variations);
     this._sized.set(sizedKey, handle);
@@ -1020,7 +1043,7 @@ export class CocoaFontManager {
             best = face;
           }
         }
-        handle = this._native.cgFontWithSize(best.cg, size);
+        handle = this._native.cgFontWithSize(best.cg, size, ...this._scaleArgs);
         break;
       }
       handle ??= this._native.matchFont({
@@ -1028,6 +1051,7 @@ export class CocoaFontManager {
         size,
         weight,
         italic,
+        ...(this._scale === 1 ? null : { scale: this._scale }),
       });
       this._fonts.set(key, handle);
     }
@@ -1115,7 +1139,7 @@ export class CocoaFontManager {
   _registeredFace(reg) {
     if (!reg.face) {
       reg.face = new CocoaFace(this, 'registered', (size) =>
-        this._native.cgFontWithSize(reg.cg, size),
+        this._native.cgFontWithSize(reg.cg, size, ...this._scaleArgs),
       );
       reg.face.key = `registered:${reg.face.postscriptName}`;
     }

package/src/cocoa/player.js

@@ -0,0 +1,177 @@
+// `<video src>` on macOS (docs/architecture/video.md §3.2, §9): a file or URL
+// played by AVFoundation through @windowkit/appkit's player verbs — the
+// platform's decoder, audio, seeking and HLS — with the picture on the
+// player's own AVPlayerLayer.
+//
+// The player is what a `<video>`'s sprite part shows as its `contents`, the
+// same way a `VideoFrames` sink is: lifted (src/cocoa/sprites.js), the
+// AVPlayerLayer goes inside the part's layer and fills it, and AVFoundation
+// puts every frame on the screen with nothing of ours in the way; declined,
+// the element draws the frame showing, which `copyFrame` converts into a
+// surface of its own in the colours the layer shows it in. Everything else —
+// state, time, the end, failure — comes back as the bridge's `player-*`
+// events, routed here by id (`CocoaApp._route`).
+
+/** Has `native` the verbs a player needs? */
+export function canPlay(native) {
+  return (
+    typeof native?.createPlayer === 'function' &&
+    typeof native.playerSet === 'function' &&
+    typeof native.playerSeek === 'function' &&
+    typeof native.playerCopyFrame === 'function' &&
+    typeof native.releasePlayer === 'function'
+  );
+}
+
+/**
+ * One file or URL, playing or not. `width`/`height` are the picture's once
+ * the item is ready (0 before), `duration` its length in seconds (`Infinity`
+ * for a live stream, NaN before it is known).
+ */
+export class CocoaPlayer {
+  constructor(app, src, options = {}) {
+    this.app = app;
+    this._native = app._native;
+    const made = this._native.createPlayer(String(src), {
+      autoPlay: Boolean(options.autoPlay),
+      loop: Boolean(options.loop),
+      muted: Boolean(options.muted),
+      volume: typeof options.volume === 'number' ? options.volume : 1,
+      rate: typeof options.playbackRate === 'number' ? options.playbackRate : 1,
+    });
+    this.id = made.id;
+    /** the AVPlayerLayer, a layer handle */
+    this.layer = made.layer;
+    this.width = 0;
+    this.height = 0;
+    this.duration = NaN;
+    this.currentTime = 0;
+    this.playing = false;
+    this.ended = false;
+    this.error = null;
+    this.released = false;
+    this._listeners = new Set();
+    app._players.set(this.id, this);
+  }
+
+  /** A `player-*` event from the bridge: the state it says, then whoever
+   *  listens. */
+  _event(ev) {
+    if (this.released) return;
+    switch (ev.type) {
+      case 'player-metadata':
+        this.width = ev.width;
+        this.height = ev.height;
+        this.duration = ev.duration;
+        break;
+      case 'player-state':
+        // AVFoundation reports every move of its time control — waiting,
+        // playing, a loop's restart — and HTML's `play` and `pause` are the
+        // moves between paused and not
+        if (Boolean(ev.playing) === this.playing) return;
+        this.playing = Boolean(ev.playing);
+        if (this.playing) this.ended = false;
+        break;
+      case 'player-time':
+        this.currentTime = ev.currentTime;
+        break;
+      case 'player-ended':
+        this.ended = true;
+        this.playing = false;
+        break;
+      case 'player-error':
+        this.error = new Error(
+          `react-x11: <video src> could not be played: ${ev.message}`,
+        );
+        break;
+      default:
+        return;
+    }
+    for (const fn of [...this._listeners]) fn(ev);
+  }
+
+  /** `fn(event)` for every event; answers the unsubscribe. */
+  _subscribe(fn) {
+    this._listeners.add(fn);
+    return () => this._listeners.delete(fn);
+  }
+
+  /** What `paused`, `playbackRate`, `volume`, `muted` and `loop` say now;
+   *  what is left out stays. */
+  set({ paused, playbackRate, volume, muted, loop } = {}) {
+    if (this.released) return;
+    const settings = {};
+    if (typeof paused === 'boolean') settings.paused = paused;
+    if (typeof playbackRate === 'number' && playbackRate > 0) {
+      settings.rate = playbackRate;
+    }
+    if (typeof volume === 'number') settings.volume = volume;
+    if (typeof muted === 'boolean') settings.muted = muted;
+    if (typeof loop === 'boolean') settings.loop = loop;
+    this._native.playerSet(this.id, settings);
+  }
+
+  play() {
+    // HTML plays an ended item from its start
+    if (this.ended) this.seek(0);
+    this.set({ paused: false });
+  }
+
+  pause() {
+    this.set({ paused: true });
+  }
+
+  seek(seconds) {
+    if (this.released || !Number.isFinite(seconds)) return;
+    this.currentTime = Math.max(0, seconds);
+    this._native.playerSeek(this.id, this.currentTime);
+  }
+
+  /**
+   * The frame showing, into `surface` (a `CocoaSurface` of the picture's
+   * size): `{ width, height, time, written }`, or null when nothing is newer
+   * than the last frame copied. `written` false means the surface is not the
+   * frame's size, which the caller remakes.
+   */
+  copyFrame(surface) {
+    if (this.released || !surface?._surfaceHandle) return null;
+    return this._native.playerCopyFrame(this.id, surface._surfaceHandle);
+  }
+
+  release() {
+    if (this.released) return;
+    this.released = true;
+    this._listeners.clear();
+    this.app._players.delete(this.id);
+    this._native.releasePlayer(this.id);
+  }
+}
+
+/**
+ * A player's picture on a sprite part's layer: its AVPlayerLayer inside the
+ * part's, filling it. The bridge plays into it on AVFoundation's clock; this
+ * only places it, and takes it out again when the part comes down.
+ */
+export class PlayerLift {
+  constructor(native, player, layer) {
+    this.native = native;
+    this.player = player;
+    this.layer = layer;
+    this._bounds = null;
+    native.addSublayer(layer, player.layer);
+  }
+
+  /** The part's layer is `bounds` (points) now: the picture fills it. */
+  layout(bounds) {
+    const was = this._bounds;
+    if (was && was[2] === bounds[2] && was[3] === bounds[3]) return;
+    this._bounds = bounds;
+    this.native.setLayerProps(this.player.layer, {
+      frame: [0, 0, bounds[2], bounds[3]],
+    });
+  }
+
+  release() {
+    this.native.removeFromSuperlayer(this.player.layer);
+  }
+}

package/src/cocoa/promotion.js

@@ -615,6 +615,10 @@ export class CocoaPromotion {
         !(parent.style?.borderRadius > 0)
       ) {
         const inner = insetRect(parent.abs, borderReach(parent));
+        // a clip with round corners cut again is no box's shape
+        if (cut.radius > 0 && cut.clip && !containsRect(inner, cut.clip)) {
+          return false;
+        }
         cut.clip = cut.clip ? intersectRects(cut.clip, inner) : inner;
         bounds = cut.clip && intersectRects(bounds, cut.clip);
         if (!bounds) return false;