react-x11 2.31.0 → 2.33.0

package/src/cocoa/sprites.js

@@ -17,7 +17,12 @@
 // A part's content is a raster the presenter paints by calling the part's
 // own `paint`, at opacity 1 and untransformed, and paints again when its
 // `version` changes; its opacity and its transform are the layer's, and so
-// are its animations, keyframes the bridge hands to Core Animation. A
+// are its animations, keyframes the bridge hands to Core Animation. Or it is
+// a **source**, the part's `contents` — a `<video>`'s frames, or its player —
+// which the presenter does not paint at all: the source shows itself on the
+// layer (src/cocoa/video.js, src/cocoa/player.js), between frames and on its
+// own clock, and the layer is exactly the part's rect. A source this bridge
+// cannot show is a part declined, drawn by its element like any other. A
 // transform is CSS's matrix, which the bridge takes from
 // `@windowkit/appkit` 0.19 (`transformForms()`) along with the negative
 // delay a part joined half way through needs, so that is the bridge sprites
@@ -25,8 +30,11 @@
 // element.
 
 import { RASTER_PAD, RasterState } from './presenter.js';
+import { CocoaPlayer, PlayerLift } from './player.js';
+import { VideoLift, canLiftVideo } from './video.js';
 import { intersectRects } from '../nodes/rects.js';
 import { DEV } from '../nodes/util.js';
+import { isVideoFrames } from '../videoframes.js';
 
 // Past this many device pixels on a side a part is drawn by its element: a
 // layer's contents are a bitmap, and a page-sized one is a page-sized
@@ -105,12 +113,26 @@ function isAnimation(a) {
   return true;
 }
 
-/** Is `sprite` a part as `Node.sprites()` documents one? */
+/** Is `sprite` a part as `Node.sprites()` documents one? Either it paints
+ *  or it has a source to show, never both. */
 function isSprite(sprite) {
   if (sprite == null || typeof sprite.key !== 'string') return false;
-  if (!isRect(sprite.rect) || typeof sprite.paint !== 'function') return false;
+  if (!isRect(sprite.rect)) return false;
+  if (
+    sprite.contents != null
+      ? sprite.paint !== undefined
+      : typeof sprite.paint !== 'function'
+  ) {
+    return false;
+  }
   if (sprite.reach !== undefined && !isRect(sprite.reach)) return false;
   if (sprite.clip !== undefined && !isRect(sprite.clip)) return false;
+  if (
+    sprite.clipRadius !== undefined &&
+    !(finite(sprite.clipRadius) && sprite.clipRadius >= 0)
+  ) {
+    return false;
+  }
   if (sprite.opacity !== undefined && !finite(sprite.opacity)) return false;
   if (sprite.transform !== undefined && !isMatrix(sprite.transform)) {
     return false;
@@ -204,8 +226,11 @@ function extentOf(sprite) {
 }
 
 /** The raster a part's content is painted into: its reach, out to whole
- *  pixels and padded for antialiasing, untransformed. */
+ *  pixels and padded for antialiasing, untransformed. A source's layer is
+ *  the part's rect, which the element has already put on whole pixels so
+ *  that the hole it leaves and the layer meet. */
 function rasterRectOf(sprite) {
+  if (sprite.contents != null) return sprite.rect;
   const reach = sprite.reach ?? sprite.rect;
   const x = Math.floor(reach.x) - RASTER_PAD;
   const y = Math.floor(reach.y) - RASTER_PAD;
@@ -236,6 +261,8 @@ export class SpriteLayers {
     this.supported = Boolean(
       this.native.transformForms?.()?.includes?.('matrix'),
     );
+    // and a video's frames on a layer need its video surfaces
+    this.videos = canLiftVideo(this.native);
     this.hosts = new Map(); // element -> Map(key -> state)
     this._warned = new WeakSet();
   }
@@ -304,26 +331,48 @@ export class SpriteLayers {
           continue;
         }
         if (now.has(sprite.key)) continue;
+        const contents = sprite.contents ?? null;
+        // a source this bridge cannot show: the element draws it
+        if (contents !== null && !this._canShow(contents)) continue;
         const raster = rasterRectOf(sprite);
-        if (raster.width > MAX_SIDE || raster.height > MAX_SIDE) continue;
+        // a raster's limit; a source's surface is its own size, whatever
+        // the layer is scaled to
+        if (
+          contents === null &&
+          (raster.width > MAX_SIDE || raster.height > MAX_SIDE)
+        ) {
+          continue;
+        }
         // what shows of everywhere it can be: inside its own clip, and the
         // clips of its element's ancestors, which cut its layer (`_clear`)
-        const cut = { clip: sprite.clip ?? null, bounds: null };
+        const cut = {
+          clip: sprite.clip ?? null,
+          radius: sprite.clip ? (sprite.clipRadius ?? 0) : 0,
+          bounds: null,
+        };
         const extent = extentOf(sprite);
         const bounds = cut.clip ? intersectRects(extent, cut.clip) : extent;
         if (!bounds || !this.promotion._clear(host, bounds, cut)) continue;
         const shown = cut.bounds;
         let state = was?.get(sprite.key);
         // a clip that comes or goes is a layer in a box or out of one: lifted
-        // again, rather than a running layer moved between parents
-        if (state && Boolean(state.clip) !== Boolean(cut.clip)) {
+        // again, rather than a running layer moved between parents — and so
+        // is a part that shows another source, or stops painting to show one
+        if (
+          state &&
+          (Boolean(state.clip) !== Boolean(cut.clip) ||
+            state.contents !== contents)
+        ) {
           this._drop(state, root);
           state = null;
         }
         const lifting = !state;
-        if (lifting) state = this._lift(host, sprite.key, cut.clip !== null);
+        if (lifting) {
+          state = this._lift(host, sprite.key, cut.clip !== null, contents);
+        }
         state.sprite = sprite;
         state.clip = cut.clip;
+        state.radius = cut.radius;
         state.order = [...place, AFTER_CHILDREN, index];
         // the bitmap under where it shows repaints without it, from this
         // frame on; where it went since, it is the element's own claims
@@ -370,7 +419,14 @@ export class SpriteLayers {
     }
   }
 
-  _lift(host, key, clipped) {
+  /** Can this bridge show `contents` on a layer by itself? A player always
+   *  can: it was made by this bridge, with the layer it plays into. */
+  _canShow(contents) {
+    if (contents instanceof CocoaPlayer) return !contents.released;
+    return isVideoFrames(contents) && this.videos;
+  }
+
+  _lift(host, key, clipped, contents = null) {
     const layer = this.native.createLayer();
     // a part cut to a clip is in a box of the clip's size that masks to it,
     // and the box is what stands among the layers above the bitmap
@@ -389,6 +445,7 @@ export class SpriteLayers {
       box,
       boxProps: {},
       clip: null, // device pixels, window coordinates
+      radius: 0, // the clip's corners, device pixels
       raster: new RasterState(),
       painted: UNPAINTED,
       rect: null, // the raster's rect, device pixels, window coordinates
@@ -402,6 +459,14 @@ export class SpriteLayers {
       sprite: null,
       extent: null,
       order: null,
+      // what shows itself on the layer, for a part with a source
+      contents,
+      lift:
+        contents === null
+          ? null
+          : contents instanceof CocoaPlayer
+            ? new PlayerLift(this.native, contents, layer)
+            : new VideoLift(this.native, contents, layer),
     };
   }
 
@@ -409,6 +474,7 @@ export class SpriteLayers {
    *  forgotten, and the bitmap under where it could be repaints with the
    *  element drawing it again. */
   _drop(state, root) {
+    state.lift?.release();
     this.native.removeFromSuperlayer(state.box ?? state.layer);
     state.raster.release(this.native);
     for (const run of state.runs.values()) this._ends().delete(run.caId);
@@ -427,13 +493,14 @@ export class SpriteLayers {
     const phase = `${reach.x - Math.floor(reach.x)},${reach.y - Math.floor(reach.y)}`;
     // painted again for a new version, and for a raster of another size or
     // with the part at another fraction of a pixel; a move by whole pixels
-    // is the layer's alone
+    // is the layer's alone. A source paints nothing: it shows itself.
     if (
-      state.painted === UNPAINTED ||
-      sprite.version !== state.painted ||
-      state.rect?.width !== rect.width ||
-      state.rect?.height !== rect.height ||
-      state.phase !== phase
+      !state.lift &&
+      (state.painted === UNPAINTED ||
+        sprite.version !== state.painted ||
+        state.rect?.width !== rect.width ||
+        state.rect?.height !== rect.height ||
+        state.phase !== phase)
     ) {
       this._paint(state, sprite, rect);
       state.phase = phase;
@@ -446,14 +513,23 @@ export class SpriteLayers {
     const clip = state.clip;
     if (state.box) {
       const frame = [clip.x / s, clip.y / s, clip.width / s, clip.height / s];
+      // its corners rounded where the clip's are: a circle's arc, as wide
+      // as the box allows
+      const cornerRadius =
+        Math.min(state.radius, clip.width / 2, clip.height / 2) / s;
       const was = state.boxProps;
-      if (!sameMatrix(was.frame, frame) || was.zPosition !== z) {
+      if (
+        !sameMatrix(was.frame, frame) ||
+        was.zPosition !== z ||
+        was.cornerRadius !== cornerRadius
+      ) {
         this.native.setLayerProps(state.box, {
           frame,
           masksToBounds: true,
+          cornerRadius,
           zPosition: z,
         });
-        state.boxProps = { frame, zPosition: z };
+        state.boxProps = { frame, zPosition: z, cornerRadius };
       }
     }
     const ox = state.box ? clip.x : 0;
@@ -490,6 +566,8 @@ export class SpriteLayers {
     }
     if (any) this.native.setLayerProps(state.layer, out);
     state.props = next;
+    // a source that fills the layer with a layer of its own
+    state.lift?.layout?.(next.bounds);
     // after the model went out, in the same transaction
     this._syncAnimations(state, sprite);
   }
@@ -595,6 +673,7 @@ export class SpriteLayers {
   destroy() {
     for (const lifted of this.hosts.values()) {
       for (const state of lifted.values()) {
+        state.lift?.release();
         this.native.removeFromSuperlayer(state.box ?? state.layer);
         state.raster.release(this.native);
         for (const run of state.runs.values()) this._ends().delete(run.caId);

package/src/cocoa/video.js

@@ -0,0 +1,122 @@
+// A video's frames on a layer of their own (docs/architecture/video.md §7):
+// what the surface presenter's sprites (src/cocoa/sprites.js) do with a part
+// whose `contents` is a `VideoFrames` sink instead of a `paint`.
+//
+// Every frame goes into a video surface — an IOSurface the render server
+// scans out as it is, NV12 for a YCbCr sink and BGRA for a BGRA one
+// (@windowkit/appkit's createVideoSurface) — and the layer is repointed at
+// it, inside the `push` that delivered it: a layer change of its own, with
+// no react-x11 frame, no paint and nothing for the frame pacer to price. At
+// 1080p that is the copy in (0.07ms for NV12, 0.27ms for an I420 frame
+// interleaved on the way) and a transaction.
+//
+// The surfaces are a ring, because a surface the render server is still
+// reading — the one on screen, and in threaded mode the one a queued frame
+// has not taken off the layer yet — is torn by a write. A frame goes into
+// one `videoSurfaceIsInUse` says nothing reads, a new one is made while the
+// ring is short, and with every one of them busy the frame waits for the
+// next push or a short retry: the newest frame is the frame, and a dropped
+// one is the cheapest thing to lose.
+//
+// The verbs are asked for by name (`canLiftVideo`), and a bridge without
+// them lifts nothing: the part is declined and the element draws its frames,
+// which is what it does on every other backend.
+
+// Two would do in pump mode — one on screen, one being written — and a
+// worker's queued flip holds a third.
+const RING_MAX = 4;
+// With the whole ring busy, how soon to try the newest frame again: about a
+// display frame, since that is how often the render server lets one go.
+const RETRY_MS = 8;
+
+/** Has `native` the verbs a lifted video needs? */
+export function canLiftVideo(native) {
+  return (
+    typeof native?.createVideoSurface === 'function' &&
+    typeof native.writeVideoSurface === 'function' &&
+    typeof native.videoSurfaceIsInUse === 'function' &&
+    typeof native.setLayerContentsIOSurface === 'function'
+  );
+}
+
+/**
+ * A sink's frames on `layer`, from the frame it holds now until `release()`.
+ * Placing the layer — its bounds, its place among the others, the box that
+ * cuts it to a clip — is the presenter's; this only ever sets what it shows.
+ */
+export class VideoLift {
+  constructor(native, sink, layer) {
+    this.native = native;
+    this.sink = sink;
+    this.layer = layer;
+    this.ring = []; // { handle, id }
+    this.current = null;
+    this.shown = 0; // the version on the layer
+    this.released = false;
+    this._retry = null;
+    this._unsubscribe = sink._subscribe((frame) => this._show(frame));
+    if (sink.frame) this._show(sink.frame);
+  }
+
+  /** A surface nothing reads, or a new one while the ring is short. */
+  _free() {
+    const native = this.native;
+    for (const s of this.ring) {
+      if (s !== this.current && !native.videoSurfaceIsInUse(s.handle)) {
+        return s;
+      }
+    }
+    if (this.ring.length >= RING_MAX) return null;
+    const sink = this.sink;
+    const made = native.createVideoSurface(sink.width, sink.height, {
+      // a YCbCr frame stays YCbCr, I420 interleaved by the write; the
+      // render server converts it, in the colour the sink says it is
+      format: sink.format === 'BGRA' ? 'BGRA' : 'NV12',
+      colorSpace: sink.colorSpace,
+      range: sink.range,
+    });
+    const s = { handle: made.handle, id: made.iosurfaceId };
+    this.ring.push(s);
+    return s;
+  }
+
+  _show(frame) {
+    if (this.released || frame === null) return;
+    const s = this._free();
+    if (!s) {
+      this._again();
+      return;
+    }
+    this.native.writeVideoSurface(s.handle, this.sink.format, frame.planes, {
+      strides: frame.strides,
+    });
+    // its own transaction, actions off, in pump mode; a command of its own
+    // from a worker
+    this.native.setLayerContentsIOSurface(this.layer, s.id);
+    this.current = s;
+    this.shown = frame.version;
+  }
+
+  _again() {
+    if (this._retry) return;
+    this._retry = setTimeout(() => {
+      this._retry = null;
+      const frame = this.sink.frame;
+      if (frame && frame.version !== this.shown) this._show(frame);
+    }, RETRY_MS);
+    this._retry.unref?.();
+  }
+
+  /** Off the layer: no more frames, and the surfaces freed — the layer
+   *  holds the one it shows until the presenter takes it away. */
+  release() {
+    if (this.released) return;
+    this.released = true;
+    this._unsubscribe();
+    if (this._retry) clearTimeout(this._retry);
+    this._retry = null;
+    for (const s of this.ring) this.native.releaseVideoSurface?.(s.handle);
+    this.ring = [];
+    this.current = null;
+  }
+}

package/src/host.js

@@ -24,6 +24,7 @@ const BUILT_IN = Object.freeze([
   'box',
   'text',
   'image',
+  'video',
   'canvas',
   'textinput',
   'textarea',

package/src/index.d.ts

@@ -32,6 +32,7 @@ export * from './types/tray.js';
 export * from './types/permissions.js';
 export * from './types/notifications.js';
 export * from './types/desktopcalendar.js';
+export * from './types/video.js';
 
 /**
  * The XID of the X11 window a ref points at, or `null` if there is not one
@@ -220,9 +221,21 @@ export function useClipboard(): Clipboard;
  * above everything the X server draws: nothing can be drawn over one there.
  * Ask it before handing a surface its HUD. A property of the backend and the
  * display, settled before the first render.
+ *
+ * `'mediaPlayback'` is whether this display plays a file or URL itself — a
+ * platform player, AVFoundation on macOS over a bridge with its player verbs
+ * — which `<video src>` needs. False on X11 and Wayland for good; ask it
+ * before rendering a `<video src>`, which refuses with one `onError` and
+ * shows its poster where it is false. `<video frames>` needs no capability.
+ * A property of the backend, and it never changes.
  */
 export type SupportsFeature =
-  'transparency' | 'shaders' | 'nativeControls' | 'embedding' | 'glOverlay';
+  | 'transparency'
+  | 'shaders'
+  | 'nativeControls'
+  | 'embedding'
+  | 'glOverlay'
+  | 'mediaPlayback';
 
 /**
  * Can this **display** do something? `'transparency'` is true when the

package/src/index.js

@@ -75,6 +75,13 @@ export {
 export { useDesktopCalendarEvents } from './desktopcalendarhooks.js';
 export { parseUriList } from './transfer.js';
 export { useApp, useClipboard, useSupports } from './appcontext.js';
+export {
+  createVideoFrames,
+  VIDEO_FORMATS,
+  VideoFrames,
+} from './videoframes.js';
+export { useVideoFrames } from './videohooks.js';
+export { NoMediaPlaybackError } from './mediaplayback.js';
 export { BusUnavailableError, closeBus, sessionBus, systemBus } from './bus.js';
 export { announce } from './a11y.js';
 // the standard Undo/Cut/Copy/Paste menu, for an element that edits or

package/src/mediaplayback.js

@@ -0,0 +1,37 @@
+// Whether this display plays a file or URL itself — `<video src>` — and the
+// error a `<video src>` reports where it does not.
+//
+// `<video>` asks before it opens anything, and `useSupports('mediaPlayback')`
+// asks for a component deciding what to render. One function for both, so
+// the two cannot disagree (#531's rule, src/embedding.js's shape). A feature
+// test on the app, not a question about which backend this is: the answer is
+// yes where the app can make a player — the Cocoa app over a bridge with
+// AVFoundation's player verbs — and no on X11 and Wayland for good, where no
+// display-server player exists and the application decodes into `frames`.
+//
+// A leaf module on purpose: it imports nothing of ours, so both
+// `appcontext.js` and the node layer can use it.
+
+/** Can `app` play a `<video src>` with a platform player? */
+export function canPlayMedia(app) {
+  return typeof app?.createPlayer === 'function';
+}
+
+/**
+ * `<video src>` where this backend has no platform player — the error
+ * `onError` is handed, so an application can tell it from a file that would
+ * not play.
+ */
+export class NoMediaPlaybackError extends Error {
+  constructor() {
+    super(
+      'react-x11: <video src> needs a platform media player, and this ' +
+        'backend has none — so the poster is shown instead. Ask ' +
+        "useSupports('mediaPlayback') before rendering one, and give a " +
+        'backend without a player frames you decode yourself: <video ' +
+        'frames={useVideoFrames(…)}> works everywhere (docs/elements.md#video).',
+    );
+    this.name = 'NoMediaPlaybackError';
+    this.code = 'ENOMEDIAPLAYBACK';
+  }
+}

package/src/node.d.ts

@@ -24,6 +24,7 @@ import type {
   MouseEvent,
   WheelEvent,
 } from './types/events.js';
+import type { VideoFrames } from './types/video.js';
 
 /** A gradient, as `createLinearGradient` answers with. */
 export interface CanvasGradientLike {
@@ -289,9 +290,23 @@ export interface Sprite {
    * every ancestor with square corners (a rounded one has to hold all of
    * it). */
   clip?: Rect;
+  /** The radius of `clip`'s corners, in device pixels: a circular arc at
+   * each of the four, as a card with a `borderRadius` cuts what it holds.
+   * 0 when left out. A clipping ancestor with square corners that would
+   * cut the clip again keeps the part with its element, since one box
+   * cannot take both. */
+  clipRadius?: number;
   /** Draws the part where it is in the window, as `paint` would, but at
-   * opacity 1 and untransformed. */
-  paint(ctx: Context2D): void;
+   * opacity 1 and untransformed. A part with `contents` has none. */
+  paint?(ctx: Context2D): void;
+  /**
+   * A source that shows itself on the part's layer instead of a paint — a
+   * `VideoFrames` sink, whose every push reaches the layer with no frame of
+   * the window's own. `rect` is the picture, on whole pixels; a presenter
+   * that cannot show the source declines the part, and the element draws
+   * the current frame itself (docs/extending.md).
+   */
+  contents?: VideoFrames;
   /** Compared with `===` from frame to frame; a change paints the part
    * again. Left out, it is painted once. */
   version?: unknown;

package/src/nodes/fit.js

@@ -0,0 +1,46 @@
+// `objectFit`: where a picture with a size of its own goes in a box that has
+// another — CSS's `object-fit`, for `<image>` and `<video>`, centred the way
+// CSS's default `object-position: 50% 50%` centres it. A leaf: no node, no
+// backend, rects in and a rect out.
+
+export const OBJECT_FITS = Object.freeze([
+  'fill',
+  'contain',
+  'cover',
+  'none',
+  'scale-down',
+]);
+
+/** The element's `objectFit`, or its default where the style names none. */
+export function objectFitOf(style, fallback) {
+  const fit = style?.objectFit;
+  return OBJECT_FITS.includes(fit) ? fit : fallback;
+}
+
+/**
+ * Where a `width` x `height` picture goes in `box` under `fit`: `fill`
+ * stretches it to the box, `contain` scales it to fit inside with bars,
+ * `cover` scales it to fill with the overflow cut (by the caller), `none`
+ * leaves it its own size, and `scale-down` is `none` or `contain`, whichever
+ * is smaller. Its edges are snapped to whole pixels, so a picture drawn
+ * there and a layer placed there cover exactly the same ones — except under
+ * `fill`, which is the box itself, as an `<image>` has always been drawn.
+ */
+export function fitRect(box, width, height, fit) {
+  if (fit === 'fill' || !(width > 0 && height > 0)) return { ...box };
+  let k;
+  if (fit === 'none') k = 1;
+  else {
+    const sx = box.width / width;
+    const sy = box.height / height;
+    k = fit === 'cover' ? Math.max(sx, sy) : Math.min(sx, sy);
+    if (fit === 'scale-down') k = Math.min(k, 1);
+  }
+  const w = width * k;
+  const h = height * k;
+  const x0 = Math.round(box.x + (box.width - w) / 2);
+  const y0 = Math.round(box.y + (box.height - h) / 2);
+  const x1 = Math.round(box.x + (box.width + w) / 2);
+  const y1 = Math.round(box.y + (box.height + h) / 2);
+  return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 };
+}

package/src/nodes/image.js

@@ -18,6 +18,7 @@ import {
 } from '../imagesource.js';
 import { loadImageFile } from '../imagedecode.js';
 import { symbolWeight, symbolsFor, warnOnce } from '../symbols.js';
+import { fitRect, objectFitOf } from './fit.js';
 import { intrinsicSize } from './layout.js';
 import { Node } from './node.js';
 import { DEV } from './util.js';
@@ -317,12 +318,42 @@ export class ImageNode extends Node {
     }
     if (!this.image) return;
     const content = this.contentBox();
-    ctx.drawImage(
-      this.image,
-      content.x,
-      content.y,
-      content.width,
-      content.height,
+    const fit = objectFitOf(this.style, 'fill');
+    if (fit === 'fill') {
+      ctx.drawImage(
+        this.image,
+        content.x,
+        content.y,
+        content.width,
+        content.height,
+      );
+      return;
+    }
+    // the source's size in device pixels, as `measureContent` takes it
+    const s = this.scale;
+    const rect = fitRect(
+      content,
+      (this.image.width ?? 0) * s,
+      (this.image.height ?? 0) * s,
+      fit,
     );
+    if (!(rect.width > 0 && rect.height > 0)) return;
+    const inside =
+      rect.x >= content.x &&
+      rect.y >= content.y &&
+      rect.x + rect.width <= content.x + content.width &&
+      rect.y + rect.height <= content.y + content.height;
+    if (!inside) {
+      // `cover`, and `none` larger than the box: cut to the content box
+      ctx.save();
+      ctx.beginPath();
+      ctx.rect(content.x, content.y, content.width, content.height);
+      ctx.clip();
+    }
+    try {
+      ctx.drawImage(this.image, rect.x, rect.y, rect.width, rect.height);
+    } finally {
+      if (!inside) ctx.restore();
+    }
   }
 }

package/src/nodes/kinds.js

@@ -13,6 +13,7 @@ export const DRAWN_KINDS = new Set([
   'box',
   'text',
   'image',
+  'video',
   'canvas',
   'textinput',
   'textarea',

package/src/nodes/layouthost.js

@@ -147,6 +147,7 @@ const NO_LAYOUT_KINDS = new Set([
   'text',
   'textchunk',
   'image',
+  'video',
   'svg',
   'canvas',
   'textinput',