react-x11 2.29.0 → 2.31.0

package/src/frame/index.js

@@ -529,7 +529,9 @@ function PaneHostView({
     }
     let host;
     try {
-      host = app.createPaneHost(wnd);
+      // `send` is for the windows the pane makes besides its own, which a
+      // backend whose pane process cannot show one shows here (Cocoa's)
+      host = app.createPaneHost(wnd, { send: (msg) => s.trySend?.(msg) });
     } catch (err) {
       embedErrorRef.current?.(err);
       return undefined;
@@ -539,17 +541,44 @@ function PaneHostView({
     // The pane's size is this box's laid-out size, told to the pane whenever
     // it changes — the same absolutize hook a <foreign> child window uses
     // (src/foreignnodes.js), because onViewport fires only for scrollers.
-    let sentSize = null;
+    //
+    // And where the box is on the screen, in device pixels (`screen`): the
+    // pane has no window of its own to ask, and a popup it anchors to one of
+    // its nodes is placed against it (`windowOrigin`, src/anchor.js) — a
+    // dropdown in a pane opened from the screen's corner without it. It
+    // moves when the box does and when the window does, so it is sent on
+    // layout, on a scroll that shifts the box, and on every anchor change
+    // of the window, which a move of the window is.
+    let sent = null;
     const syncSize = () => {
       host.setRect(node.abs);
       const sc = node.scale ?? 1;
       const width = Math.max(1, Math.round(node.abs.width / sc));
       const height = Math.max(1, Math.round(node.abs.height / sc));
-      if (sentSize && sentSize.width === width && sentSize.height === height) {
+      const origin = node.root?.window?._screenOrigin;
+      const screen = origin
+        ? {
+            x: Math.round(origin.x + node.abs.x),
+            y: Math.round(origin.y + node.abs.y),
+          }
+        : null;
+      if (
+        sent &&
+        sent.width === width &&
+        sent.height === height &&
+        sent.screen?.x === screen?.x &&
+        sent.screen?.y === screen?.y
+      ) {
         return;
       }
-      sentSize = { width, height };
-      s.trySend?.({ type: 'pane-rect', width, height, scale: sc });
+      sent = { width, height, screen };
+      s.trySend?.({
+        type: 'pane-rect',
+        width,
+        height,
+        scale: sc,
+        ...(screen && { screen }),
+      });
     };
     const origAbsolutize = node.absolutize.bind(node);
     node.absolutize = (ox, oy) => {
@@ -560,13 +589,35 @@ function PaneHostView({
     if (origShift) {
       node._shiftAbs = (dx, dy) => {
         origShift(dx, dy);
-        host.setRect(node.abs);
+        syncSize();
       };
     }
+    const offAnchor = node.root?.onAnchorChange?.(syncSize);
     if (node.abs?.width) syncSize();
 
+    // The window's focus, which the pane has no window to hear: a menu it
+    // keeps open on the strength of the window having it closes when the
+    // user goes to another application (`useDismissOnWindowBlur`).
+    const offFocus = node.root?.onWindowFocusChange?.((focused) =>
+      s.trySend?.({
+        type: 'pane-event',
+        name: focused ? 'focus' : 'blur',
+        ev: { buttons: 0, time: Date.now() },
+      }),
+    );
+
     const off = s.transport.onMessage((msg) => {
-      if (msg?.type === 'pane-present') {
+      if (msg?.type === 'pane-window') {
+        host.paneWindow?.(msg);
+      } else if (msg?.type === 'pane-menu') {
+        host.popUpMenu?.(msg);
+      } else if (msg?.type === 'pane-menu-cancel') {
+        host.cancelMenu?.(msg.menu);
+      } else if (msg?.type === 'pane-present' && msg.window != null) {
+        host.presentWindow?.(msg);
+      } else if (msg?.type === 'pane-cursor' && msg.window != null) {
+        host.windowCursor?.(msg);
+      } else if (msg?.type === 'pane-present') {
         host.setRect(node.abs);
         host.present(msg.id);
       } else if (msg?.type === 'pane-cursor') {
@@ -581,6 +632,8 @@ function PaneHostView({
     });
     return () => {
       off();
+      offAnchor?.();
+      offFocus?.();
       node.defaultCursor = undefined;
       node.root?.events?.refreshCursor();
       node.absolutize = origAbsolutize;

package/src/imagedecode.js

@@ -0,0 +1,255 @@
+// Encoded image bytes → an ntk `Image` of straight RGBA, through the best
+// decoder this runtime has. A ladder, tried in order:
+//
+//   1. The runtime's own: `Bun.Image` under Bun. libjpeg-turbo, spng and
+//      libwebp linked into the runtime, GIF and BMP everywhere, and TIFF,
+//      HEIC and AVIF through the system's codecs on macOS and Windows — all
+//      run on a worker thread. That is the reason to prefer it even for
+//      the formats the next rung can do: a 1080p JPEG costs the JavaScript
+//      thread ~7ms here against ~110ms inside jpeg-js (a 1080p PNG, ~5ms
+//      against ~150ms in pngjs), and a decode on the JavaScript thread is
+//      frames nobody gets. Found by its constructor, never by asking which
+//      runtime this is.
+//   2. JavaScript. ntk's PNG and JPEG decoders (pngjs, jpeg-js), which are
+//      synchronous, and WebP through image-in-browser's decoder — imported
+//      the first time a WebP arrives, because evaluating it takes ~140ms
+//      that an app showing no WebP should never pay at startup.
+//
+// Two things here are easy to undo:
+//
+// - `Bun.Image` has no raw-pixel output: every terminal re-encodes. The
+//   cheapest round trip is a PNG at compression level 0 — stored deflate
+//   blocks, every row filtered None — which `storedPngImage` unwraps with
+//   one native inflate and a row copy. Reading that PNG with pngjs instead
+//   costs 80-130ms of the JavaScript thread at 1080p — all of what moving
+//   the decode off it saved.
+// - Bun's WebP decode is libwebp's still-image decoder, which refuses an
+//   animation outright (Bun 1.4.0). An animated WebP that Bun turns down
+//   goes to the JavaScript decoder for its first frame, which is what an
+//   `<image>` shows of any animation; asking Bun first keeps a later Bun that
+//   can do it on the fast rung.
+import { Image, decodeImage } from 'ntk/image';
+
+/** `Bun.Image`, where the runtime has one. Read per call: it is one
+ * property lookup, and a cached answer would hide a test's stand-in. */
+function runtimeDecoder() {
+  const decoder = globalThis.Bun?.Image;
+  return typeof decoder === 'function' ? decoder : null;
+}
+
+const ascii = (bytes, at, text) => {
+  for (let i = 0; i < text.length; i++) {
+    if (bytes[at + i] !== text.charCodeAt(i)) return false;
+  }
+  return true;
+};
+
+/** ISO-BMFF major brands of a HEIF still image; AVIF is told apart first. */
+const HEIF_BRANDS = new Set(['heic', 'heix', 'heim', 'heis', 'mif1', 'msf1']);
+
+/**
+ * The format a buffer's first bytes name, or null. The bytes, never a file
+ * extension, say what an image is — which is how Bun and ntk tell too.
+ */
+export function imageFormat(bytes) {
+  if (bytes.length >= 8 && ascii(bytes, 1, 'PNG') && bytes[0] === 0x89) {
+    return 'png';
+  }
+  if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8) {
+    return 'jpeg';
+  }
+  if (
+    bytes.length >= 12 &&
+    ascii(bytes, 0, 'RIFF') &&
+    ascii(bytes, 8, 'WEBP')
+  ) {
+    return 'webp';
+  }
+  if (bytes.length >= 6 && ascii(bytes, 0, 'GIF8')) return 'gif';
+  if (bytes.length >= 14 && ascii(bytes, 0, 'BM')) return 'bmp';
+  if (
+    bytes.length >= 4 &&
+    ((ascii(bytes, 0, 'II') && bytes[2] === 42 && bytes[3] === 0) ||
+      (ascii(bytes, 0, 'MM') && bytes[2] === 0 && bytes[3] === 42))
+  ) {
+    return 'tiff';
+  }
+  if (bytes.length >= 12 && ascii(bytes, 4, 'ftyp')) {
+    if (ascii(bytes, 8, 'avif') || ascii(bytes, 8, 'avis')) return 'avif';
+    const brand = String.fromCharCode(...bytes.subarray(8, 12));
+    if (HEIF_BRANDS.has(brand)) return 'heic';
+  }
+  return null;
+}
+
+/** An extended-format WebP whose VP8X chunk says it is animated. */
+export function isAnimatedWebP(bytes) {
+  return (
+    bytes.length >= 21 &&
+    ascii(bytes, 0, 'RIFF') &&
+    ascii(bytes, 8, 'WEBP') &&
+    ascii(bytes, 12, 'VP8X') &&
+    (bytes[20] & 0x02) !== 0
+  );
+}
+
+const NAMES = {
+  png: 'PNG',
+  jpeg: 'JPEG',
+  webp: 'WebP',
+  gif: 'GIF',
+  bmp: 'BMP',
+  tiff: 'TIFF',
+  heic: 'HEIC',
+  avif: 'AVIF',
+};
+
+const hex = (bytes) =>
+  Array.from(bytes.subarray(0, 8), (b) => b.toString(16).padStart(2, '0'))
+    .join(' ')
+    .trim();
+
+/**
+ * Decode encoded bytes: an `Image`, or a promise of one where the decode
+ * runs off the JavaScript thread (Bun) or its decoder loads on demand (WebP
+ * elsewhere). Throws — or rejects — on bytes that are not an image this
+ * runtime can decode, with what to do about it.
+ *
+ * The bytes are read, never written, and under Bun they are read on another
+ * thread after this returns: the caller must not change them meanwhile,
+ * which `<image>`'s contract (a source is immutable content) already says.
+ */
+export function decodeImageBytes(bytes) {
+  if (bytes.length === 0) {
+    throw new Error('react-x11: an empty buffer is not an image.');
+  }
+  const decoder = runtimeDecoder();
+  if (decoder) return decodeWithRuntime(decoder, bytes);
+  const format = imageFormat(bytes);
+  if (format === 'png' || format === 'jpeg') return decodeImage(bytes);
+  if (format === 'webp') return decodeWebP(bytes);
+  if (format) {
+    throw new Error(
+      `react-x11: ${NAMES[format]} needs a decoder this runtime does not ` +
+        'have. Under Node, <image> decodes PNG, JPEG and WebP; under Bun it ' +
+        'decodes GIF and BMP too, and TIFF, HEIC and AVIF on macOS and ' +
+        'Windows. Convert the image, or run the app with Bun.',
+    );
+  }
+  throw new Error(
+    'react-x11: these bytes are not an image format react-x11 recognizes ' +
+      `(PNG, JPEG or WebP; more under Bun) — they begin ${hex(bytes)}.`,
+  );
+}
+
+/** Read a file path or file URL and decode it. */
+export async function loadImageFile(path) {
+  const { readFile } = await import('node:fs/promises');
+  return decodeImageBytes(await readFile(path));
+}
+
+async function decodeWithRuntime(BunImage, bytes) {
+  // Bun borrows the bytes on its worker thread and refuses a shared or a
+  // resizable buffer; those, and only those, it is handed a copy of
+  const input =
+    bytes.buffer instanceof ArrayBuffer && !bytes.buffer.resizable
+      ? bytes
+      : bytes.slice();
+  let png;
+  try {
+    png = await new BunImage(input).png({ compressionLevel: 0 }).bytes();
+  } catch (err) {
+    if (err?.code === 'ERR_IMAGE_DECODE_FAILED' && isAnimatedWebP(bytes)) {
+      return decodeWebP(bytes);
+    }
+    if (err?.code === 'ERR_IMAGE_FORMAT_UNSUPPORTED') {
+      const name = NAMES[imageFormat(bytes)] ?? 'This format';
+      throw new Error(
+        `react-x11: ${name} has no decoder on this machine. Bun decodes ` +
+          "TIFF, HEIC and AVIF through the system's codecs — ImageIO on " +
+          'macOS, WIC on Windows (HEIC and AVIF need the HEIF and AV1 ' +
+          'extensions from the Microsoft Store) — and Linux has none. ' +
+          'Convert the image to PNG, JPEG or WebP.',
+        { cause: err },
+      );
+    }
+    throw err;
+  }
+  return storedPngImage(png) ?? decodeImage(png);
+}
+
+const IHDR = 0x49484452;
+const IDAT = 0x49444154;
+const IEND = 0x49454e44;
+
+/**
+ * The pixels of the PNG `Bun.Image` re-encodes into, without pngjs: 8-bit
+ * RGBA, not interlaced, every row filtered None — what
+ * `png({ compressionLevel: 0 })` writes — read with one native inflate and a
+ * row copy. Null for any other PNG, which pngjs reads instead: correct, and
+ * slower, so a Bun that changes what it writes degrades rather than breaks.
+ */
+export function storedPngImage(png) {
+  const view = new DataView(png.buffer, png.byteOffset, png.byteLength);
+  let width = 0;
+  let height = 0;
+  const chunks = [];
+  let total = 0;
+  for (let at = 8; at + 12 <= png.length;) {
+    const length = view.getUint32(at);
+    const type = view.getUint32(at + 4);
+    const body = png.subarray(at + 8, at + 8 + length);
+    if (type === IHDR) {
+      width = view.getUint32(at + 8);
+      height = view.getUint32(at + 12);
+      // bit depth 8, colour type 6 (RGBA), no interlace
+      if (body[8] !== 8 || body[9] !== 6 || body[12] !== 0) return null;
+    } else if (type === IDAT) {
+      chunks.push(body);
+      total += length;
+    } else if (type === IEND) {
+      break;
+    }
+    at += 12 + length;
+  }
+  if (!width || !height || !chunks.length) return null;
+  let stream = chunks[0];
+  if (chunks.length > 1) {
+    stream = new Uint8Array(total);
+    let at = 0;
+    for (const chunk of chunks) {
+      stream.set(chunk, at);
+      at += chunk.length;
+    }
+  }
+  // windowBits 15: the zlib wrapper PNG puts around its deflate stream
+  const rows = globalThis.Bun.inflateSync(stream, { windowBits: 15 });
+  const stride = width * 4;
+  if (rows.length !== height * (stride + 1)) return null;
+  const data = new Uint8Array(height * stride);
+  for (let y = 0, from = 0; y < height; y++, from += stride + 1) {
+    if (rows[from] !== 0) return null;
+    data.set(rows.subarray(from + 1, from + 1 + stride), y * stride);
+  }
+  return new Image({ width, height, data });
+}
+
+let webpModule = null;
+
+/** A WebP through image-in-browser's decoder; an animation's first frame. */
+async function decodeWebP(bytes) {
+  webpModule ??= import('image-in-browser/lib/src/formats/webp-decoder.js');
+  const { WebPDecoder } = await webpModule;
+  // frame 0 alone: without an index it decodes every frame of an animation
+  // and still answers the first
+  const decoded = new WebPDecoder().decode({ bytes, frameIndex: 0 });
+  if (!decoded) {
+    throw new Error(
+      'react-x11: these WebP bytes did not decode — the file is damaged or ' +
+        'cut short.',
+    );
+  }
+  // order 0 is ChannelOrder.rgba; a WebP always decodes to four channels
+  const data = decoded.getBytes({ order: 0 });
+  return new Image({ width: decoded.width, height: decoded.height, data });
+}

package/src/imagesource.js

@@ -6,8 +6,9 @@
 // and the `cacheKey` cache live here, where a test needs no server; the node
 // half in nodes/image.js is only lifecycle — when to resolve, when to claim
 // damage, when to let go.
-import { Image, decodeImage } from 'ntk/image';
+import { Image } from 'ntk/image';
 
+import { decodeImageBytes } from './imagedecode.js';
 import { x11Ntk } from './ntkroot.js';
 
 /**
@@ -42,7 +43,7 @@ export function isPathImageSource(src) {
   );
 }
 
-/** …normalized for ntk's `loadImage`, which wants a string or a real URL. */
+/** …normalized for `readFile`, which wants a string or a real URL. */
 export const toLoadablePath = (src) =>
   typeof src === 'string' || src instanceof URL ? src : new URL(src.href);
 
@@ -139,9 +140,9 @@ const describe = (value) =>
 
 /** Stated once, so every error lists the same set of accepted forms. */
 const SRC_FORMS =
-  'a file path or file URL (PNG/JPEG), encoded PNG/JPEG bytes (Buffer or ' +
-  'Uint8Array), raw RGBA ({ width, height, data }), an ntk Image/Surface, ' +
-  "or a symbol by name ({ symbol: 'speaker.wave.3.fill' })";
+  'a file path or file URL (PNG/JPEG/WebP), encoded PNG/JPEG/WebP bytes ' +
+  '(Buffer or Uint8Array), raw RGBA ({ width, height, data }), an ntk ' +
+  "Image/Surface, or a symbol by name ({ symbol: 'speaker.wave.3.fill' })";
 
 function validateServerSource(kind, desc) {
   const shape =
@@ -198,7 +199,7 @@ function validateRawSource(src) {
       'react-x11: <image src={{ width, height, data }}> wants straight ' +
         '(non-premultiplied) RGBA bytes — data.length must be ' +
         `width × height × 4 = ${want}, got ${bytes ?? describe(data)}. ` +
-        'For encoded PNG/JPEG bytes pass the buffer itself as `src`.',
+        'For encoded PNG/JPEG/WebP bytes pass the buffer itself as `src`.',
     );
   }
 }
@@ -254,14 +255,17 @@ export function validateImageProps(props) {
 }
 
 /**
- * Decode a synchronous source into an ntk `Image`: encoded bytes through the
- * PNG/JPEG decoders, raw RGBA wrapped as-is (no copy — the object is treated
- * as immutable content from here on). File paths stay async and do not come
- * here. May throw on corrupt bytes; the caller treats that as a content
- * failure, because encoded bytes usually arrive from outside the program.
+ * Decode an in-memory source into an ntk `Image`: encoded bytes through the
+ * decoder ladder in imagedecode.js, raw RGBA wrapped as-is (no copy — the
+ * object is treated as immutable content from here on). Returns the `Image`,
+ * or a promise of one where the bytes decode off the JavaScript thread
+ * (under Bun) or their decoder loads on demand (WebP elsewhere). File paths
+ * do not come here. May throw or reject on bytes that do not decode; the
+ * caller treats that as a content failure, because encoded bytes usually
+ * arrive from outside the program.
  */
 export function decodeImageSource(src) {
-  if (src instanceof Uint8Array) return decodeImage(src);
+  if (src instanceof Uint8Array) return decodeImageBytes(src);
   return new Image({ width: src.width, height: src.height, data: src.data });
 }
 
@@ -317,9 +321,10 @@ const sourceCaches = new WeakMap();
 /**
  * Take a hold on the entry for `key`, creating it with `load()` on first
  * use. `load` returns `{ image }` for a synchronous source or `{ promise }`
- * for a file read — the promise resolves an `Image` or, on content failure,
- * `null` (never rejects; the loader reports the failure once, not per
- * holder). Every acquire is paired with a `releaseImageSource`.
+ * for a file read or a decode that lands later — the promise resolves an
+ * `Image` or, on content failure, `null` (never rejects; the loader reports
+ * the failure once, not per holder). Every acquire is paired with a
+ * `releaseImageSource`.
  */
 export function acquireImageSource(app, key, load) {
   let cache = sourceCaches.get(app);

package/src/node.d.ts

@@ -284,6 +284,11 @@ export interface Sprite {
   rect: Rect;
   /** What it draws, untransformed, when that is more than `rect`. */
   reach?: Rect;
+  /** Where it shows, when the element clips it: device pixels, window
+   * coordinates. Its layer is cut to the rect, and so it is to the clip of
+   * every ancestor with square corners (a rounded one has to hold all of
+   * it). */
+  clip?: Rect;
   /** Draws the part where it is in the window, as `paint` would, but at
    * opacity 1 and untransformed. */
   paint(ctx: Context2D): void;

package/src/nodes/image.js

@@ -16,11 +16,7 @@ import {
   toLoadablePath,
   validateImageProps,
 } from '../imagesource.js';
-// Namespace import for `Surface`, the same shape and the same reason as
-// `paintcache.js`: a named import of something an older ntk does not export
-// is a *load-time* SyntaxError, which would take the renderer down rather
-// than the one feature that needs it.
-import { loadImage } from 'ntk/image';
+import { loadImageFile } from '../imagedecode.js';
 import { symbolWeight, symbolsFor, warnOnce } from '../symbols.js';
 import { intrinsicSize } from './layout.js';
 import { Node } from './node.js';
@@ -32,6 +28,18 @@ import { DEV } from './util.js';
  * both "nothing changed". */
 const IMAGE_SOURCE_PROPS = new Set(['src', 'picture', 'drawable', 'cacheKey']);
 
+const BYTES_FAILED = 'react-x11: <image src> bytes did not decode:';
+
+/** A load that never rejects: a failure is logged and resolves null. */
+const reported = (promise, prefix) =>
+  promise.then(
+    (image) => image,
+    (err) => {
+      console.error(prefix, err.message);
+      return null;
+    },
+  );
+
 export class ImageNode extends Node {
   constructor(props, app) {
     super('image', props, app);
@@ -133,7 +141,7 @@ export class ImageNode extends Node {
     }
     if (cacheKey != null) {
       const entry = acquireImageSource(this.app, cacheKey, () =>
-        this._loadEntry(src),
+        this._load(src),
       );
       this._hold = entry;
       if (entry.image) {
@@ -148,63 +156,54 @@ export class ImageNode extends Node {
       }
       return;
     }
-    if (isPathImageSource(src)) {
-      this._loadFile(src);
-      return;
-    }
-    const image = this._decode(src);
+    const { image, promise } = this._load(src);
     if (image) {
       this._ownedImage = image;
       this.image = image;
+    } else if (promise) {
+      this._adopt(promise);
     }
   }
 
-  /** `{ image }` or `{ promise }` for the cache. Failures resolve to null
-   * and are reported once, here — not per node holding the key. */
-  _loadEntry(src) {
+  /**
+   * `{ image }` for a source decoded on the spot, `{ promise }` for a file
+   * read or a decode that lands later — off the JavaScript thread under
+   * Bun, or behind a decoder loaded on demand. The promise resolves an
+   * Image, or null on a content failure, which is reported once, here: not
+   * per node holding the key.
+   */
+  _load(src) {
     if (isPathImageSource(src)) {
       return {
-        promise: loadImage(toLoadablePath(src)).then(
-          (image) => image,
-          (err) => {
-            console.error(
-              `react-x11: failed to load image ${src}:`,
-              err.message,
-            );
-            return null;
-          },
+        promise: reported(
+          loadImageFile(toLoadablePath(src)),
+          `react-x11: failed to load image ${src}:`,
         ),
       };
     }
-    return { image: this._decode(src) };
-  }
-
-  _decode(src) {
+    let decoded;
     try {
-      return decodeImageSource(src);
+      decoded = decodeImageSource(src);
     } catch (err) {
       // corrupt or unrecognized bytes are a content failure, not a
       // programming error: log and show nothing, like a missing file
-      console.error(
-        'react-x11: <image src> bytes did not decode:',
-        err.message,
-      );
-      return null;
+      console.error(BYTES_FAILED, err.message);
+      return { image: null };
     }
+    return typeof decoded?.then === 'function'
+      ? { promise: reported(decoded, BYTES_FAILED) }
+      : { image: decoded };
   }
 
-  async _loadFile(src) {
+  /** Take an Image that lands after the frame that asked for it — unless
+   * the source changed or the node went first, in which case it was never
+   * painted, so nothing was uploaded and there is nothing to free. */
+  async _adopt(promise) {
     const token = ++this._loadToken;
-    try {
-      const image = await loadImage(toLoadablePath(src));
-      if (token !== this._loadToken || this.destroyed) return;
-      this._ownedImage = image;
-      this._setImage(image);
-    } catch (err) {
-      if (token === this._loadToken && !this.destroyed) {
-        console.error(`react-x11: failed to load image ${src}:`, err.message);
-      }
-    }
+    const image = await promise;
+    if (!image || token !== this._loadToken || this.destroyed) return;
+    this._ownedImage = image;
+    this._setImage(image);
   }
 
   /** Adopt pixels that arrived after the frame that asked for them. A size
@@ -238,7 +237,7 @@ export class ImageNode extends Node {
   }
 
   _releaseSource() {
-    this._loadToken++; // orphan any in-flight file read
+    this._loadToken++; // orphan any in-flight read or decode
     if (this._hold) {
       releaseImageSource(this.app, this._hold);
       this._hold = null;