react-x11 2.52.0 → 2.53.0

package/src/wayland/app.js

@@ -30,6 +30,7 @@ import { releaseDevice } from './device.js';
 import { WaylandSeat } from './seat.js';
 import { WaylandBackendWindow } from './backendwindow.js';
 import { WaylandSurface } from './surface.js';
+import { releaseVideoFrame, writeVideoFrame } from './video.js';
 import { InputRouter } from './input.js';
 import { WaylandTouch } from './touch.js';
 import { WaylandTablet } from './tablet.js';
@@ -440,6 +441,32 @@ export class WaylandApp extends EventEmitter {
     return new WaylandSurface(this, options);
   }
 
+  /**
+   * The formats a `VideoFrames` sink is cheapest in here
+   * (`preferredFormats`, src/videoframes.js): all three, YCbCr first. Every
+   * one goes up as textures as it is and is converted by a shader
+   * (src/wayland/video.js), and NV12 or I420 is less than half of BGRA's
+   * bytes on the way.
+   */
+  videoFormats() {
+    return ['NV12', 'I420', 'BGRA'];
+  }
+
+  /**
+   * A `<video>`'s frame into `surface`, its drawn presentation, on the GPU:
+   * planes as textures, the matrix in a fragment shader. False where the
+   * frame cannot be — a size past the GPU's textures — and the node
+   * converts it in JavaScript instead (src/nodes/video.js).
+   */
+  writeVideoFrame(surface, sink, frame) {
+    return writeVideoFrame(this, surface, sink, frame);
+  }
+
+  /** The node let `surface` go: its plane textures go too. */
+  releaseVideoFrame(surface) {
+    releaseVideoFrame(this, surface);
+  }
+
   /**
    * What the machine can do, as a promise like the X11 and Cocoa answers
    * (src/glbackend.js chains on it): here the answer was settled when the

package/src/wayland/video.js

@@ -0,0 +1,376 @@
+// `<video frames>` on Wayland: a frame's planes go up as textures as the
+// decoder made them, and the colour conversion is a fragment shader drawing
+// into the `<video>`'s own surface (design/architecture/video.md §5).
+//
+// The backend's 2D context is already a GLES rasterizer, so this is the GLX
+// transport of §2.3 made native: no conversion in JavaScript for any format,
+// and NV12 and I420 cross into the GPU at 1.5 bytes a pixel. Y is an R8
+// texture; NV12's chroma one RG8 texture at half size, I420's two R8 ones;
+// BGRA one RGBA8 texture read back to front. The shader applies the sink's
+// matrix and range exactly as `frameToRGBA` does — no transfer function,
+// because nothing else on this backend is colour-managed either — so a frame
+// drawn here and one converted in JavaScript are the same picture, chroma
+// apart: sampled bilinearly here, as a player does, and by nearest there.
+//
+// What it writes into is the surface the drawn presentation already keeps
+// (src/nodes/video.js): a render target, which the window's context then
+// composites at the picture's rect like any `drawImage`. Nothing lifts on
+// Wayland — no subsurfaces, by the backend's own design — so this is the
+// whole presentation.
+//
+// The GL is shared with every 2D context on the app (device.js): whatever
+// the device's owner still has buffered is drawn before the target is
+// written, the shader's vertex state lives in a vertex array of its own,
+// and the device is released afterwards so the next context to draw puts
+// its own state back. `UNPACK_ROW_LENGTH`, which a decoder's padded rows
+// need, is set back to 0 for the uploads that assume it.
+
+import { flushDevice, releaseDevice } from './device.js';
+
+const VERT = `
+attribute vec2 aPos;
+attribute vec2 aUV;
+varying vec2 vUV;
+void main() {
+  vUV = aUV;
+  gl_Position = vec4(aPos, 0.0, 1.0);
+}`;
+
+const FRAG = `
+#ifdef GL_FRAGMENT_PRECISION_HIGH
+precision highp float;
+#else
+precision mediump float;
+#endif
+varying vec2 vUV;
+uniform int uLayout;
+uniform sampler2D uY;
+uniform sampler2D uC;
+uniform sampler2D uV;
+uniform vec2 uChroma;
+uniform mat3 uMatrix;
+uniform vec3 uOffset;
+void main() {
+  if (uLayout == 0) {
+    // BGRA bytes uploaded as RGBA: blue is in r. Opaque, whatever the
+    // fourth byte held.
+    gl_FragColor = vec4(texture2D(uY, vUV).bgr, 1.0);
+    return;
+  }
+  float y = texture2D(uY, vUV).r;
+  vec2 cuv = vUV * uChroma;
+  vec2 c = uLayout == 1
+    ? texture2D(uC, cuv).rg
+    : vec2(texture2D(uC, cuv).r, texture2D(uV, cuv).r);
+  vec3 rgb = uMatrix * (vec3(y, c) - uOffset);
+  gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0);
+}`;
+
+const LAYOUT = { BGRA: 0, NV12: 1, I420: 2 };
+
+// Kr and Kb, as src/videoframes.js has them
+const MATRIX = {
+  bt709: [0.2126, 0.0722],
+  bt601: [0.299, 0.114],
+  bt2020: [0.2627, 0.0593],
+};
+
+const matrices = new Map();
+
+/**
+ * The column-major mat3 and the offset that take normalized Y'CbCr to
+ * R'G'B' for a matrix and a range — the same numbers `frameToRGBA`'s tables
+ * are made of, in units of 1 rather than of 255.
+ */
+export function videoMatrix(colorSpace, range) {
+  const key = `${colorSpace}/${range}`;
+  let m = matrices.get(key);
+  if (m) return m;
+  const [kr, kb] = MATRIX[colorSpace];
+  const kg = 1 - kr - kb;
+  const full = range === 'full';
+  const ys = full ? 1 : 255 / 219;
+  const cs = full ? 1 : 255 / 224;
+  m = {
+    matrix: Float32Array.of(
+      // the column Y' multiplies
+      ys,
+      ys,
+      ys,
+      // Cb
+      0,
+      -cs * ((2 * kb * (1 - kb)) / kg),
+      cs * 2 * (1 - kb),
+      // Cr
+      cs * 2 * (1 - kr),
+      -cs * ((2 * kr * (1 - kr)) / kg),
+      0,
+    ),
+    offset: Float32Array.of(full ? 0 : 16 / 255, 128 / 255, 128 / 255),
+  };
+  matrices.set(key, m);
+  return m;
+}
+
+/** The planes of a frame as textures: format, size and the bytes per texel
+ *  `UNPACK_ROW_LENGTH` counts a decoder's stride in. */
+function planesOf(gl, format, width, height) {
+  const cw = Math.ceil(width / 2);
+  const ch = Math.ceil(height / 2);
+  const r8 = { internal: gl.R8, format: gl.RED, texel: 1 };
+  switch (format) {
+    case 'NV12':
+      return [
+        { ...r8, width, height },
+        { internal: gl.RG8, format: gl.RG, texel: 2, width: cw, height: ch },
+      ];
+    case 'I420':
+      return [
+        { ...r8, width, height },
+        { ...r8, width: cw, height: ch },
+        { ...r8, width: cw, height: ch },
+      ];
+    case 'BGRA':
+      return [{ internal: gl.RGBA, format: gl.RGBA, texel: 4, width, height }];
+  }
+  return null;
+}
+
+/** One per GL device: the program, and the quad it draws. */
+const programs = new WeakMap();
+
+function programFor(gl) {
+  let p = programs.get(gl);
+  if (p) return p;
+  const vs = compile(gl, gl.VERTEX_SHADER, VERT, 'video vertex');
+  let fs;
+  try {
+    fs = compile(gl, gl.FRAGMENT_SHADER, FRAG, 'video fragment');
+  } catch (err) {
+    gl.deleteShader(vs);
+    throw err;
+  }
+  const program = gl.createProgram();
+  gl.attachShader(program, vs);
+  gl.attachShader(program, fs);
+  gl.linkProgram(program);
+  const linked = gl.getProgramParameter(program, gl.LINK_STATUS);
+  const log = linked ? '' : gl.getProgramInfoLog(program);
+  gl.deleteShader(vs);
+  gl.deleteShader(fs);
+  if (!linked) {
+    gl.deleteProgram(program);
+    throw new Error(`video program failed to link: ${log}`);
+  }
+  const u = (name) => gl.getUniformLocation(program, name);
+  const aPos = gl.getAttribLocation(program, 'aPos');
+  const aUV = gl.getAttribLocation(program, 'aUV');
+  // A strip over the whole target. The 2D context draws device row 0 at the
+  // top of clip space, so the frame's first row — texture row 0 — goes
+  // there: v = 0 at clip y = +1.
+  const buffer = gl.createBuffer();
+  gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
+  gl.bufferData(
+    gl.ARRAY_BUFFER,
+    Float32Array.of(-1, 1, 0, 0, -1, -1, 0, 1, 1, 1, 1, 0, 1, -1, 1, 1),
+    gl.STATIC_DRAW,
+  );
+  // its own vertex array, so the 2D contexts' attribute state — on the
+  // default one, re-pointed every batch — is never this program's
+  const vao = gl.createVertexArray();
+  gl.bindVertexArray(vao);
+  gl.enableVertexAttribArray(aPos);
+  gl.vertexAttribPointer(aPos, 2, gl.FLOAT, false, 16, 0);
+  gl.enableVertexAttribArray(aUV);
+  gl.vertexAttribPointer(aUV, 2, gl.FLOAT, false, 16, 8);
+  gl.bindVertexArray(null);
+  p = {
+    program,
+    buffer,
+    vao,
+    maxSize: gl.getParameter(gl.MAX_TEXTURE_SIZE),
+    loc: {
+      uLayout: u('uLayout'),
+      uY: u('uY'),
+      uC: u('uC'),
+      uV: u('uV'),
+      uChroma: u('uChroma'),
+      uMatrix: u('uMatrix'),
+      uOffset: u('uOffset'),
+    },
+  };
+  programs.set(gl, p);
+  return p;
+}
+
+/** Each surface's plane textures, kept across frames of one size. */
+const uploads = new WeakMap();
+
+function texturesFor(gl, surface, format, width, height) {
+  let kept = uploads.get(surface);
+  if (
+    kept &&
+    kept.format === format &&
+    kept.width === width &&
+    kept.height === height
+  ) {
+    return kept;
+  }
+  if (kept) deleteTextures(gl, kept);
+  const planes = planesOf(gl, format, width, height);
+  kept = {
+    format,
+    width,
+    height,
+    planes,
+    textures: planes.map(() => gl.createTexture()),
+    fresh: true,
+  };
+  uploads.set(surface, kept);
+  return kept;
+}
+
+function deleteTextures(gl, kept) {
+  for (const tex of kept.textures) gl.deleteTexture(tex);
+  kept.textures = [];
+}
+
+/**
+ * A plane's rows at a stride `UNPACK_ROW_LENGTH` cannot say — one that is
+ * not a whole number of texels — packed tight. Every decoder pads to whole
+ * pixels, so this is the case that should never come.
+ */
+function tight(data, stride, rowBytes, rows) {
+  const out = new Uint8Array(rowBytes * rows);
+  for (let r = 0; r < rows; r++) {
+    out.set(data.subarray(r * stride, r * stride + rowBytes), r * rowBytes);
+  }
+  return out;
+}
+
+/**
+ * `frame` from `sink` into `surface` (a WaylandSurface at the frame's size).
+ * False — and the frame is converted in JavaScript instead — where the
+ * surface is not one of this backend's, the app has no GL, or the frame is
+ * larger than a texture can be.
+ */
+export function writeVideoFrame(app, surface, sink, frame) {
+  const gl = app?.gl;
+  const target = surface?.target;
+  if (!gl || !target?.fbo || target.destroyed) return false;
+  if (!(sink.format in LAYOUT)) return false;
+  app.makeCurrent();
+  const p = programFor(gl);
+  if (sink.width > p.maxSize || sink.height > p.maxSize) return false;
+  // Anything a context still has buffered is drawn first: one of its draws
+  // may read this target, and must read the frame it was given (device.js).
+  flushDevice(gl);
+  const kept = texturesFor(gl, surface, sink.format, sink.width, sink.height);
+  try {
+    gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+    kept.planes.forEach((plane, i) => {
+      const rowBytes = plane.width * plane.texel;
+      const stride = frame.strides[i];
+      let data = frame.planes[i];
+      let rowLength = 0;
+      if (stride !== rowBytes) {
+        if (stride % plane.texel === 0) rowLength = stride / plane.texel;
+        else data = tight(data, stride, rowBytes, plane.height);
+      }
+      gl.pixelStorei(gl.UNPACK_ROW_LENGTH, rowLength);
+      gl.activeTexture(gl.TEXTURE0 + i);
+      gl.bindTexture(gl.TEXTURE_2D, kept.textures[i]);
+      if (kept.fresh) {
+        // Y and BGRA meet the target texel for texel; chroma is half size,
+        // and read between its samples as a player reads it
+        const filter = i === 0 ? gl.NEAREST : gl.LINEAR;
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texImage2D(
+          gl.TEXTURE_2D,
+          0,
+          plane.internal,
+          plane.width,
+          plane.height,
+          0,
+          plane.format,
+          gl.UNSIGNED_BYTE,
+          data,
+        );
+      } else {
+        gl.texSubImage2D(
+          gl.TEXTURE_2D,
+          0,
+          0,
+          0,
+          plane.width,
+          plane.height,
+          plane.format,
+          gl.UNSIGNED_BYTE,
+          data,
+        );
+      }
+    });
+    kept.fresh = false;
+
+    target.bind();
+    gl.disable(gl.BLEND);
+    gl.disable(gl.SCISSOR_TEST);
+    gl.disable(gl.STENCIL_TEST);
+    gl.disable(gl.DEPTH_TEST);
+    gl.disable(gl.CULL_FACE);
+    gl.colorMask(true, true, true, true);
+    gl.useProgram(p.program);
+    const { loc } = p;
+    gl.uniform1i(loc.uLayout, LAYOUT[sink.format]);
+    gl.uniform1i(loc.uY, 0);
+    gl.uniform1i(loc.uC, 1);
+    gl.uniform1i(loc.uV, 2);
+    // full-size texture coordinates onto a chroma plane rounded up: an odd
+    // frame's last chroma column covers one picture column, not two
+    gl.uniform2f(
+      loc.uChroma,
+      sink.width / (2 * Math.ceil(sink.width / 2)),
+      sink.height / (2 * Math.ceil(sink.height / 2)),
+    );
+    const { matrix, offset } = videoMatrix(sink.colorSpace, sink.range);
+    gl.uniformMatrix3fv(loc.uMatrix, false, matrix);
+    gl.uniform3fv(loc.uOffset, offset);
+    gl.bindVertexArray(p.vao);
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    gl.bindVertexArray(null);
+  } finally {
+    gl.pixelStorei(gl.UNPACK_ROW_LENGTH, 0);
+    gl.activeTexture(gl.TEXTURE0);
+    // the program, the blend and the framebuffer were the 2D contexts' to
+    // say: the next one to draw says them again
+    releaseDevice(gl);
+    app.rebindWindowTarget?.();
+  }
+  return true;
+}
+
+/** `surface` is let go: its plane textures with it. */
+export function releaseVideoFrame(app, surface) {
+  const kept = uploads.get(surface);
+  if (!kept) return;
+  uploads.delete(surface);
+  const gl = app?.gl;
+  if (!gl) return;
+  app.makeCurrent();
+  deleteTextures(gl, kept);
+}
+
+function compile(gl, type, src, what) {
+  const sh = gl.createShader(type);
+  gl.shaderSource(sh, src);
+  gl.compileShader(sh);
+  if (!gl.getShaderParameter(sh, gl.COMPILE_STATUS)) {
+    const log = gl.getShaderInfoLog(sh);
+    gl.deleteShader(sh);
+    throw new Error(`${what} shader failed to compile: ${log}`);
+  }
+  return sh;
+}

package/src/win32/a11y.js

@@ -7,7 +7,7 @@
 // property reads — so answering from JS would pace the screen reader by the
 // application's busiest moment. The bridge therefore *pushes*, and
 // `windows/src/uia.cc` answers from its copy. The reasoning is
-// docs/windows.md §"Accessibility: UI Automation"; what matters here is the
+// design/windows.md §"Accessibility: UI Automation"; what matters here is the
 // consequence: **this file's job is a diff, not an answer.**
 //
 // Everything it pushes comes from `src/a11y.js`, which is pure functions over

package/src/win32/app.js

@@ -362,7 +362,7 @@ export class Win32App {
     // `embeddable` is "created, not shown; somebody else will place me", and
     // on this backend that cannot be a window: a composition target stops
     // presenting the moment its window becomes a child, so an HWND is not
-    // something another process can take (docs/windows-embedding.md). What a
+    // something another process can take (design/windows-embedding.md). What a
     // host can take is the **buffer**, so an embeddable window here is the
     // same shared surface a `<Frame>` pane draws into — whether this process
     // was forked as a pane or is a guest somebody else started.
@@ -825,7 +825,7 @@ export class Win32App {
         break;
       }
       case 'dpichanged':
-        // The renderer cannot re-scale a live window yet (docs/windows.md
+        // The renderer cannot re-scale a live window yet (design/windows.md
         // §Layout, open question 8), so this is recorded and not acted on.
         wnd.emit('statechange', { scale: event.a / 96 });
         break;

package/src/win32/fonts.js

@@ -1,6 +1,6 @@
 // The Windows text engine: DirectWrite behind the same `app.fonts` contract
 // ntk's FontManager answers on X11 and CoreText answers on macOS. The renderer
-// touches exactly this surface (docs/windows.md §"Text"):
+// touches exactly this surface (design/windows.md §"Text"):
 //
 //   fonts.layout(spans, base, { maxWidth, align, lineHeight, maxLines,
 //                               overflow, direction })

package/src/win32/glarea.js

@@ -1,6 +1,6 @@
 // `<glarea>` on Windows: a WGL core context, composited as a visual.
 //
-// docs/windows-gl.md is the report behind this, and the route is the X11
+// design/windows-gl.md is the report behind this, and the route is the X11
 // model rather than ANGLE — a GL surface covers its rect and the tree draws
 // under it, positioned from the parent's yoga rect. src/glnodes.js already
 // expects exactly that, including the rule that the surface selects no
@@ -58,7 +58,7 @@ export class Win32GlWindow {
    * vendor driver: opengl32.dll is then the 1.1 software rasterizer, with no
    * shaders and no framebuffer objects — which is not a degraded direct
    * backend but the indirect one's feature set without its portability, and
-   * is docs/windows-gl.md's rung 2 rather than this one.
+   * is design/windows-gl.md's rung 2 rather than this one.
    */
   _onReady(ok, why = 0) {
     if (!ok) {
@@ -79,7 +79,7 @@ export class Win32GlWindow {
         'error',
         new Error(
           `react-x11: no OpenGL surface — ${REASONS[why] ?? 'the bridge refused it'}. ` +
-            'docs/windows-gl.md rung 2 (ANGLE) covers this case and is not built yet.',
+            'design/windows-gl.md rung 2 (ANGLE) covers this case and is not built yet.',
         ),
       );
       return;
@@ -540,7 +540,7 @@ export function installGl(app) {
       throw new Error(
         'react-x11: <glarea> has no GL surface — this machine has no vendor ' +
           'OpenGL driver, so only the 1.1 software rasterizer is available, ' +
-          'which has no shaders. docs/windows-gl.md rung 2 (ANGLE) covers ' +
+          'which has no shaders. design/windows-gl.md rung 2 (ANGLE) covers ' +
           'this case and is not built yet.',
       );
     });

package/src/win32/overlay.js

@@ -13,7 +13,7 @@
 // leave transparent shows the GL frame, and a translucent fill, an
 // antialiased edge or a shadow blends with it. Text on it is grayscale
 // rather than ClearType, which Direct2D does on any surface with alpha
-// (docs/windows.md §Text).
+// (design/windows.md §Text).
 //
 // Drawn like the window it lives in (src/win32/window.js): a DirectComposition
 // surface hands out a context per `BeginDraw`, valid until `EndDraw`, so the

package/src/win32/panewindow.js

@@ -1,4 +1,4 @@
-// The pane process's "window" on the Windows backend — docs/frame.md's child
+// The pane process's "window" on the Windows backend — design/frame.md's child
 // half, with the composition engine standing in for the X server.
 //
 // On X11 a pane renders into a real window of its own and the host reparents
@@ -6,7 +6,7 @@
 // the pane makes a DirectComposition surface handle, hangs a composition
 // swapchain off it, and draws through the same verb table a window draws
 // through; the host opens that handle and gives it to a visual of its own
-// (src/win32/panehost.js). See docs/windows-embedding.md for why this rather
+// (src/win32/panehost.js). See design/windows-embedding.md for why this rather
 // than a shared DXGI texture — in one line, nothing here has to synchronise.
 //
 // **The present is the hand-off.** Once the host holds the handle the
@@ -360,7 +360,7 @@ export class Win32PaneWindow {
    * On X11 that is the window's own id, because an XID means the same thing
    * in every process on the display. Windows has no such number: a window
    * cannot be embedded at all, because a composition target stops presenting
-   * the moment its window becomes a child (docs/windows-embedding.md, with
+   * the moment its window becomes a child (design/windows-embedding.md, with
    * the measurements). What crosses instead is the **buffer** — the same
    * composition surface handle a `<Frame>` pane publishes — and the host
    * binds it to a visual of its own rather than reparenting anything.

package/src/win32/shell.js

@@ -236,7 +236,7 @@ async function badgePixels(app, label, accent) {
  * What this rung cannot do, and says so by not implementing it: actions,
  * replacing a notification in place, and an icon of the app's own. Those want
  * the Windows App SDK's notification manager and an AppUserModelID the system
- * knows — docs/windows.md §"The desktop around the app". Until then the
+ * knows — design/windows.md §"The desktop around the app". Until then the
  * balloon is shown by the shell in the same place, with the same sound, and
  * kept in the same action centre.
  */

package/src/win32/window.js

@@ -252,7 +252,7 @@ export class Win32Window {
   }
 
   /** The size limits Windows asks for synchronously in WM_GETMINMAXINFO, so
-   * they are pushed ahead rather than answered on demand — docs/windows.md
+   * they are pushed ahead rather than answered on demand — design/windows.md
    * "What Windows asks synchronously, and JS can know in advance". */
   setSizeHints(hints = {}) {
     this._native.setSizeHints?.(this.id, hints);
@@ -691,7 +691,7 @@ export class Win32Window {
   }
 
   /** A DirectComposition surface is write-only, so a window's pixels are not
-   * readable the way an IOSurface's or an X drawable's are. docs/windows.md
+   * readable the way an IOSurface's or an X drawable's are. design/windows.md
    * answers this with PrintWindow once the commit has completed; not bound
    * yet, and it says so rather than answering with something wrong. */
   /**

package/src/windowid.js

@@ -78,7 +78,7 @@ export function windowOf(target) {
  * - **Windows** — a **composition surface handle**, already valid in the
  *   host process. A window cannot be embedded here at all: a composition
  *   target stops presenting the moment its window becomes a child, measured
- *   in docs/windows-embedding.md. So what crosses is the buffer, and the
+ *   in design/windows-embedding.md. So what crosses is the buffer, and the
  *   host binds it to a visual of its own instead of reparenting. The host is
  *   the parent process by default — `createRoot({ win32: { paneHostPid } })`
  *   when it is not.