tmux-ide 2.9.11 → 2.9.13

package/packages/daemon/dist/terminal/pane-stream/pane-stream-websocket.js

@@ -1,8 +1,12 @@
-import { PANE_STREAM_INPUT_LIFETIME_CAPABILITY_VERSION, } from "@tmux-ide/contracts";
+import { TERMINAL_SURFACE_MAX_CONTROL_BYTES, TERMINAL_SURFACE_CAPABILITY_VERSION, } from "@tmux-ide/contracts";
+import { NativeSurfaceViewer } from "../graphics/native-surface-viewer.js";
+import { compressGraphicsChunk } from "../graphics/compress-chunk.js";
+import { TERMINAL_GRAPHICS_COMPRESSED_CAPABILITY_VERSION, TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION, TERMINAL_PROJECTION_CAPABILITY_VERSION, TERMINAL_PROJECTION_LOCAL_CAPABILITY_VERSION, } from "@tmux-ide/contracts";
+import { PANE_STREAM_INPUT_LIFETIME_CAPABILITY_VERSION, PANE_STREAM_INPUT_BYTES_CAPABILITY_VERSION, } from "@tmux-ide/contracts";
 import { inflateSync } from "node:zlib";
 import { PaneStreamLayoutSnapshotFrameSchemaZ } from "@tmux-ide/contracts";
 import { z } from "zod";
-import { PANE_STREAM_MAX_HELD_DELTAS, PANE_STREAM_MAX_OUTPUT_BYTES, PANE_STREAM_MAX_PANES, PANE_STREAM_MAX_SEED_BYTES, PANE_STREAM_PROTOCOL_VERSION, PANE_STREAM_REDEEM_PATH, PANE_STREAM_WEBSOCKET_SUBPROTOCOL, TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES, TERMINAL_GRAPHICS_CELL_PIXELS_CAPABILITY_VERSION, TERMINAL_GRAPHICS_PRECISE_PIXELS_CAPABILITY_VERSION, TERMINAL_GRAPHICS_LEGACY_MAX_IMAGE_BYTES, TERMINAL_GRAPHICS_ZLIB_CAPABILITY_VERSION, PaneStreamClientFrameSchemaZ, PaneStreamLoopbackWebSocketUrlSchemaZ, PaneStreamRedeemFrameSchemaZ, PaneStreamServerFrameSchemaZ, sharedMonotonicMicros, } from "@tmux-ide/contracts";
+import { PANE_STREAM_MAX_HELD_DELTAS, PANE_STREAM_MAX_OUTPUT_BYTES, PANE_STREAM_MAX_PANES, PANE_STREAM_MAX_SEED_BYTES, PANE_STREAM_PROTOCOL_VERSION, PANE_STREAM_REDEEM_PATH, PANE_STREAM_WEBSOCKET_SUBPROTOCOL, TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES, TERMINAL_DELIVERY_BINARY_CAPABILITY_VERSION, TERMINAL_DELIVERY_BINARY_LOCAL_CAPABILITY_VERSION, TERMINAL_GRAPHICS_BINARY_CAPABILITY_VERSION, TERMINAL_GRAPHICS_LOCAL_CAPABILITY_VERSION, encodeTerminalGraphicsBinary, encodeTerminalDeliveryBinaryChunk, TERMINAL_GRAPHICS_CELL_PIXELS_CAPABILITY_VERSION, TERMINAL_GRAPHICS_PRECISE_PIXELS_CAPABILITY_VERSION, TERMINAL_GRAPHICS_LEGACY_MAX_IMAGE_BYTES, TERMINAL_GRAPHICS_ZLIB_CAPABILITY_VERSION, PaneStreamClientFrameSchemaZ, PaneStreamLoopbackWebSocketUrlSchemaZ, PaneStreamRedeemFrameSchemaZ, PaneStreamServerFrameSchemaZ, sharedMonotonicMicros, } from "@tmux-ide/contracts";
 import { canonicalOriginOrNull, digestSecret, digestsEqual, rawDataByteLength, rawDataToBuffer, safeCloseSocket, strictJsonParse, } from "../attachments/admission-util.js";
 import { SessionRuntimeControllerLeaseError } from "../session-runtime/registry.js";
 import { terminalDeliveryObservationOrdinal } from "../session-runtime/terminal-delivery-observation-identity.js";
@@ -115,6 +119,8 @@ function errorFrameCode(error) {
  * ticket digests; a daemon restart makes every prior descriptor inert.
  */
 export class PaneStreamAdmissionCoordinator {
+    #nativeSurfaces;
+    #localImages;
     #instanceId;
     #webSocketUrl;
     #leaseManager;
@@ -145,6 +151,10 @@ export class PaneStreamAdmissionCoordinator {
     #shuttingDown = false;
     #shutdownPromise = null;
     constructor(options) {
+        this.#nativeSurfaces = options.nativeSurfaces;
+        if (this.#nativeSurfaces && this.#nativeSurfaces.generation !== options.daemonInstanceId)
+            throw new Error("Native surface provider generation mismatch");
+        this.#localImages = options.localImages;
         this.#instanceId = BindingIdSchemaZ.parse(options.daemonInstanceId);
         this.#webSocketUrl = PaneStreamLoopbackWebSocketUrlSchemaZ.parse(options.webSocketUrl);
         this.#leaseManager = options.leaseManager;
@@ -407,6 +417,8 @@ export class PaneStreamAdmissionCoordinator {
                 let live;
                 try {
                     live = new PaneStreamLiveConnection({
+                        nativeSurfaces: this.#nativeSurfaces,
+                        localImages: this.#localImages,
                         clientId: `psc-${this.#clientCounter}`,
                         socket,
                         descriptor: structuredClone(descriptor),
@@ -614,6 +626,15 @@ export function rawGraphicsState(state) {
 }
 const GRAPHICS_SEND_WINDOW_BYTES = 2 * 1024 * 1024;
 export class PaneStreamLiveConnection {
+    // One bounded native negotiation per redeemed socket; never reuse on reconnect.
+    #nativeSurfaces;
+    #surfaceViewer = null;
+    #surfaceDemands = new Map();
+    #surfaceNative = false;
+    #surfaceSession = null;
+    #surfaceReceiver = null;
+    #localImages;
+    #localLeases = new Map();
     #clientId;
     #socket;
     #descriptor;
@@ -701,6 +722,8 @@ export class PaneStreamLiveConnection {
         return elapsed;
     }
     constructor(options) {
+        this.#nativeSurfaces = options.nativeSurfaces;
+        this.#localImages = options.localImages;
         this.#clientId = options.clientId;
         this.#socket = options.socket;
         this.#descriptor = options.descriptor;
@@ -770,6 +793,10 @@ export class PaneStreamLiveConnection {
                     this.#descriptor.terminalDelivery?.protocolVersions.includes(PANE_STREAM_INPUT_LIFETIME_CAPABILITY_VERSION)
                     ? { inputLifetime: true }
                     : {}),
+                ...(this.#sessionRuntimeBinding &&
+                    this.#descriptor.terminalDelivery?.protocolVersions.includes(PANE_STREAM_INPUT_BYTES_CAPABILITY_VERSION)
+                    ? { inputBytes: true }
+                    : {}),
                 protocolVersion: PANE_STREAM_PROTOCOL_VERSION,
                 daemonInstanceId: this.#binding.daemonInstanceId,
                 requestId: this.#binding.requestId,
@@ -790,6 +817,8 @@ export class PaneStreamLiveConnection {
         }
         this.#stopAuthoritySnapshots =
             this.#sessionRuntimeBinding?.onAuthoritySnapshot?.((snapshot) => this.#usesExplicitAuthority ? this.#sendAuthoritySnapshot(snapshot) : undefined) ?? null;
+        if (!this.#descriptor.terminalDelivery)
+            this.#observeSurfaceDemand(this.#descriptor.panes);
         void this.#subscribeAll();
     }
     close(code = 1000, reason = "stream-closed") {
@@ -797,6 +826,13 @@ export class PaneStreamLiveConnection {
             return;
         this.#recordDiagnosticLifecycle("pane-stream-terminal", code, reason);
         this.#closed = true;
+        for (const demand of this.#surfaceDemands.values())
+            demand.close();
+        this.#surfaceDemands.clear();
+        this.#surfaceViewer?.close();
+        this.#surfaceViewer = null;
+        this.#surfaceSession?.close();
+        this.#surfaceSession = null;
         this.#cancelDrainTick?.();
         this.#cancelDrainTick = null;
         // Departure force-returns every ticket within this same tick.
@@ -812,6 +848,12 @@ export class PaneStreamLiveConnection {
         this.#stopAuthoritySnapshots?.();
         this.#stopAuthoritySnapshots = null;
         const closures = [];
+        this.#localLeases.clear();
+        if (this.#localImages)
+            closures.push(this.#localImages.releaseClient({
+                generation: this.#localImages.identity.generation,
+                client: this.#descriptor.leaseId,
+            }));
         const layoutSubscription = this.#layoutSubscription;
         this.#layoutSubscription = null;
         if (layoutSubscription)
@@ -822,6 +864,8 @@ export class PaneStreamLiveConnection {
             const delivery = channel.delivery;
             channel.delivery = null;
             channel.closed = true;
+            this.#surfaceDemands.get(channel.semanticPaneId)?.close();
+            this.#surfaceDemands.delete(channel.semanticPaneId);
             if (sub)
                 closures.push(sub.close().catch(() => undefined));
             if (delivery)
@@ -1024,7 +1068,7 @@ export class PaneStreamLiveConnection {
                 return true;
             let incarnation = null;
             for (const message of pending) {
-                if (message.type !== "terminal.delivery")
+                if (message.type !== "terminal.delivery" && message.type !== "terminal.projection")
                     continue;
                 if (message.workspaceName !== this.#descriptor.workspaceName ||
                     message.semanticPaneId !== channel.semanticPaneId ||
@@ -1132,7 +1176,15 @@ export class PaneStreamLiveConnection {
         this.#closeIfAllPanesGone();
     }
     async #sendTerminalDelivery(pane, message) {
-        if (message.type === "terminal.delivery") {
+        if (message.type === "terminal.projection") {
+            const channel = this.#panes.get(pane);
+            if (channel?.deliveryAddress)
+                channel.deliveryAddress.incarnation = message.incarnation;
+            if (channel)
+                channel.binaryCells = true;
+            this.#sendFrame(pane, { type: "terminal-projection-envelope", pane, envelope: message });
+        }
+        else if (message.type === "terminal.delivery") {
             if (!this.#diagnosticFirstSeedObserved) {
                 this.#diagnosticFirstSeedObserved = true;
                 this.#recordDiagnosticLifecycle("pane-stream-first-seed");
@@ -1140,6 +1192,8 @@ export class PaneStreamLiveConnection {
             const channel = this.#panes.get(pane);
             if (channel?.deliveryAddress)
                 channel.deliveryAddress.incarnation = message.incarnation;
+            if (channel)
+                channel.binaryCells = message.encoding === "semantic-binary-v1";
             // The trusted lease address was fixed at delivery admission; preserve the
             // exact envelope and its observation identity through transmission.
             const detailedDeliveryObservation = this.#observability?.enabled === true;
@@ -1215,13 +1269,18 @@ export class PaneStreamLiveConnection {
             }
         }
         else if (message.type === "terminal.delivery.chunk") {
-            this.#sendFrame(pane, {
-                type: "terminal-delivery-chunk",
-                pane,
-                transactionId: message.transactionId,
-                index: message.index,
-                data: Buffer.from(message.bytes).toString("base64"),
-            });
+            if (this.#panes.get(pane)?.binaryCells) {
+                this.#sendEncoded(pane, Buffer.from(encodeTerminalDeliveryBinaryChunk({ pane, transactionId: message.transactionId, index: message.index }, message.bytes)), true);
+            }
+            else {
+                this.#sendFrame(pane, {
+                    type: "terminal-delivery-chunk",
+                    pane,
+                    transactionId: message.transactionId,
+                    index: message.index,
+                    data: Buffer.from(message.bytes).toString("base64"),
+                });
+            }
         }
         else {
             this.#sendFrame(pane, { type: "terminal-delivery-fault", pane, fault: message });
@@ -1232,15 +1291,53 @@ export class PaneStreamLiveConnection {
         }
         await this.#awaitSemanticCredit(pane);
     }
+    #observeSurfaceDemand(panes) {
+        for (const pane of panes) {
+            if (this.#surfaceDemands.has(pane))
+                continue;
+            const demand = this.#nativeSurfaces?.demand.viewer(this.#descriptor.sessionName, [pane]);
+            if (!demand)
+                return;
+            demand.native(this.#surfaceNative);
+            this.#surfaceDemands.set(pane, demand);
+        }
+    }
+    #surfaceControl(type) {
+        if (type === "terminal-surface-offer")
+            return;
+        this.#surfaceNative = type === "terminal-surface-peer";
+        for (const demand of this.#surfaceDemands.values())
+            demand.native(this.#surfaceNative);
+    }
     /** Announce the pane's kitty graphics state; the client pulls bytes it lacks. */
     #startGraphics(channel) {
         const graphics = channel.delivery?.graphics;
         if (!graphics)
             return;
+        this.#observeSurfaceDemand([channel.semanticPaneId]);
         // Renderers that cannot inflate see raw formats; their bytes are inflated
         // when served.
         const zlib = this.#descriptor.terminalDelivery?.protocolVersions.includes(TERMINAL_GRAPHICS_ZLIB_CAPABILITY_VERSION) === true;
         channel.graphicsZlib = zlib;
+        channel.graphicsLocal =
+            !!this.#localImages &&
+                this.#descriptor.terminalDelivery?.protocolVersions.some((version) => version === TERMINAL_GRAPHICS_LOCAL_CAPABILITY_VERSION ||
+                    version === TERMINAL_DELIVERY_BINARY_LOCAL_CAPABILITY_VERSION ||
+                    version === TERMINAL_PROJECTION_LOCAL_CAPABILITY_VERSION ||
+                    version === TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION) === true;
+        channel.graphicsCompressed =
+            this.#descriptor.terminalDelivery?.protocolVersions.some((version) => version === TERMINAL_GRAPHICS_COMPRESSED_CAPABILITY_VERSION ||
+                version === TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION) === true;
+        channel.graphicsInline = new Set();
+        channel.graphicsBinary =
+            this.#descriptor.terminalDelivery?.protocolVersions.some((version) => version === TERMINAL_GRAPHICS_BINARY_CAPABILITY_VERSION ||
+                version === TERMINAL_GRAPHICS_LOCAL_CAPABILITY_VERSION ||
+                version === TERMINAL_DELIVERY_BINARY_CAPABILITY_VERSION ||
+                version === TERMINAL_PROJECTION_CAPABILITY_VERSION ||
+                version === TERMINAL_DELIVERY_BINARY_LOCAL_CAPABILITY_VERSION ||
+                version === TERMINAL_PROJECTION_LOCAL_CAPABILITY_VERSION ||
+                version === TERMINAL_GRAPHICS_COMPRESSED_CAPABILITY_VERSION ||
+                version === TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION) === true;
         // Renderers that report cell pixels learn up front that this daemon
         // accepts them, before any image exists.
         const cellPixels = this.#descriptor.terminalDelivery?.protocolVersions.includes(TERMINAL_GRAPHICS_CELL_PIXELS_CAPABILITY_VERSION) === true;
@@ -1268,6 +1365,9 @@ export class PaneStreamLiveConnection {
                     ...(zlib ? state : rawGraphicsState(state)),
                     ...(cellPixels ? { cellPixels: true } : {}),
                     ...(preciseCellPixels ? { preciseCellPixels: true } : {}),
+                    ...(channel.graphicsBinary ? { binaryBlobs: true } : {}),
+                    ...(channel.graphicsCompressed ? { compressedBlobs: true } : {}),
+                    ...(channel.graphicsLocal ? { localFiles: true } : {}),
                 },
             });
             if (pushed.length > 0)
@@ -1284,15 +1384,24 @@ export class PaneStreamLiveConnection {
                     placements: [],
                     cellPixels: true,
                     ...(preciseCellPixels ? { preciseCellPixels: true } : {}),
+                    ...(channel.graphicsBinary ? { binaryBlobs: true } : {}),
+                    ...(channel.graphicsCompressed ? { compressedBlobs: true } : {}),
+                    ...(channel.graphicsLocal ? { localFiles: true } : {}),
                 },
             });
         }
     }
-    #requestGraphics(pane, digests) {
+    #requestGraphics(pane, digests, preferInline = false) {
         const channel = this.#panes.get(pane);
         // Requests are only meaningful after an announcement; ignore stray ones.
         if (!channel || channel.closed || !channel.graphicsAnnounced)
             return;
+        if (preferInline && channel.graphicsLocal) {
+            for (const digest of digests) {
+                if ((channel.graphicsInline?.size ?? 0) < GRAPHICS_MAX_QUEUED_DIGESTS)
+                    channel.graphicsInline?.add(digest);
+            }
+        }
         this.#queueGraphics(channel, digests);
     }
     #queueGraphics(channel, digests) {
@@ -1313,6 +1422,7 @@ export class PaneStreamLiveConnection {
         try {
             while (!this.#closed && !channel.closed && channel.graphicsQueue.length > 0) {
                 const digest = channel.graphicsQueue.shift();
+                const inline = channel.graphicsInline?.delete(digest) ?? false;
                 const graphics = channel.delivery?.graphics;
                 const image = graphics?.image(digest);
                 if (!graphics || !image)
@@ -1339,6 +1449,49 @@ export class PaneStreamLiveConnection {
                     continue;
                 const format = raw ? (image.format === "rgba-zlib" ? "rgba" : "rgb") : image.format;
                 const total = data.byteLength;
+                if (!inline && channel.graphicsLocal && !raw && this.#localImages) {
+                    for (const [id, lease] of this.#localLeases)
+                        if (lease.expiresAt <= Date.now())
+                            this.#localLeases.delete(id);
+                    if (this.#localLeases.size < 64) {
+                        try {
+                            const lease = await this.#localImages.acquire({
+                                generation: this.#localImages.identity.generation,
+                                client: this.#descriptor.leaseId,
+                                digest,
+                                bytes: data,
+                            });
+                            if (this.#closed || channel.closed || graphics.image(digest) !== image) {
+                                await this.#localImages.release(lease);
+                                continue;
+                            }
+                            this.#localLeases.set(lease.id, {
+                                pane: channel.semanticPaneId,
+                                expiresAt: lease.expiresAt,
+                            });
+                            this.#sendFrame(null, {
+                                type: "terminal-graphics-local",
+                                pane: channel.semanticPaneId,
+                                generation: lease.generation,
+                                leaseId: lease.id,
+                                path: lease.path,
+                                expiresAt: lease.expiresAt,
+                                blob: {
+                                    digest,
+                                    format,
+                                    width: image.width,
+                                    height: image.height,
+                                    total,
+                                    offset: 0,
+                                },
+                            });
+                            continue;
+                        }
+                        catch {
+                            /* Optional transport: retain the negotiated inline fallback. */
+                        }
+                    }
+                }
                 for (let offset = 0; offset < total; offset += TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES) {
                     while ((this.#socket.bufferedAmount ?? 0) > GRAPHICS_SEND_WINDOW_BYTES) {
                         if (this.#closed || channel.closed)
@@ -1352,6 +1505,43 @@ export class PaneStreamLiveConnection {
                     if (offset > 0 && graphics.image(digest) !== image)
                         break;
                     const end = Math.min(total, offset + TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES);
+                    if (channel.graphicsBinary) {
+                        const header = {
+                            pane: channel.semanticPaneId,
+                            blob: {
+                                digest,
+                                format,
+                                width: image.width,
+                                height: image.height,
+                                total,
+                                offset,
+                            },
+                        };
+                        const compressed = channel.graphicsCompressed
+                            ? await compressGraphicsChunk(header, data.subarray(offset, end))
+                            : null;
+                        // Compression yields: a pane may close or replace this image meanwhile.
+                        if (this.#closed || channel.closed)
+                            return;
+                        if (graphics.image(digest) !== image)
+                            break;
+                        if (compressed) {
+                            this.#sendEncoded(null, Buffer.from(compressed), true);
+                            continue;
+                        }
+                        this.#sendEncoded(null, Buffer.from(encodeTerminalGraphicsBinary({
+                            pane: channel.semanticPaneId,
+                            blob: {
+                                digest,
+                                format,
+                                width: image.width,
+                                height: image.height,
+                                total,
+                                offset,
+                            },
+                        }, data.subarray(offset, end))), true);
+                        continue;
+                    }
                     this.#sendFrame(null, {
                         type: "terminal-graphics-blob",
                         pane: channel.semanticPaneId,
@@ -1508,6 +1698,8 @@ export class PaneStreamLiveConnection {
         if (channel.closed)
             return;
         channel.closed = true;
+        this.#surfaceDemands.get(channel.semanticPaneId)?.close();
+        this.#surfaceDemands.delete(channel.semanticPaneId);
         const sub = channel.sub;
         channel.sub = null;
         if (sub)
@@ -1671,7 +1863,12 @@ export class PaneStreamLiveConnection {
         if (this.#closed || this.#socket.readyState !== WS_OPEN)
             return;
         const encoded = JSON.stringify(frame);
-        const bytes = Buffer.byteLength(encoded, "utf8");
+        this.#sendEncoded(pane, encoded, false);
+    }
+    #sendEncoded(pane, encoded, binary) {
+        if (this.#closed || this.#socket.readyState !== WS_OPEN)
+            return;
+        const bytes = Buffer.byteLength(encoded);
         if (pane !== null) {
             this.#ledger.take(this.#clientId, pane, "ws-send-buffer", bytes);
             if (this.#deliveryAcks)
@@ -1680,7 +1877,7 @@ export class PaneStreamLiveConnection {
         this.#sendQueue.push({ pane, remaining: bytes });
         this.#sentBytesTotal += bytes;
         try {
-            this.#socket.send(encoded, { binary: false });
+            this.#socket.send(encoded, { binary });
         }
         catch {
             this.close(1011, "stream-unavailable");
@@ -1914,6 +2111,24 @@ export class PaneStreamLiveConnection {
             });
             return;
         }
+        if (frame.type === "terminal-projection-receipt") {
+            const channel = this.#deliveryChannel(frame.receipt);
+            if (!channel)
+                return;
+            if (!channel.delivery?.projectionReceipt)
+                return this.#failProtocol("protocol-error");
+            channel.delivery.projectionReceipt(frame.receipt);
+            return;
+        }
+        if (frame.type === "terminal-projection-materialize") {
+            const channel = this.#deliveryChannel(frame.request);
+            if (!channel)
+                return;
+            if (!channel.delivery?.materialize)
+                return this.#failProtocol("protocol-error");
+            channel.delivery.materialize(frame.request);
+            return;
+        }
         if (frame.type === "terminal-delivery-ack") {
             const channel = this.#deliveryChannel(frame.ack);
             if (!channel)
@@ -1945,8 +2160,138 @@ export class PaneStreamLiveConnection {
             this.#acceptSemanticIntent(frame.operationId, frame.intent);
             return;
         }
+        if (frame.type === "terminal-surface-watch") {
+            if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES ||
+                frame.generation !== this.#sessionRuntimeBinding?.generation ||
+                !this.#descriptor.terminalDelivery?.protocolVersions.includes(TERMINAL_SURFACE_CAPABILITY_VERSION) ||
+                !this.#nativeSurfaces ||
+                this.#surfaceReceiver)
+                return this.#failProtocol("protocol-error");
+            if (this.#surfaceViewer)
+                return;
+            this.#surfaceViewer = new NativeSurfaceViewer(this.#nativeSurfaces, this.#descriptor.panes, (control) => {
+                if (this.#closed)
+                    return false;
+                this.#surfaceControl(control.type);
+                this.#sendFrame(null, control);
+                return !this.#closed;
+            }, (pane, generation) => {
+                const channel = this.#panes.get(pane);
+                const address = channel?.deliveryAddress;
+                return !this.#closed &&
+                    channel &&
+                    !channel.closed &&
+                    channel.delivery &&
+                    address?.generation === generation
+                    ? address.deliveryNonce
+                    : null;
+            });
+            this.#surfaceViewer.start();
+            return;
+        }
+        if (frame.type === "terminal-surface-receiver") {
+            if (this.#surfaceViewer) {
+                if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES || !this.#surfaceViewer.receive(frame))
+                    this.#failProtocol("protocol-error");
+                return;
+            }
+            if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES ||
+                !this.#descriptor.terminalDelivery ||
+                frame.generation !== this.#sessionRuntimeBinding?.generation) {
+                return this.#failProtocol("protocol-error");
+            }
+            if (this.#surfaceReceiver) {
+                // Parsed schema order is deterministic. A duplicate cannot change the
+                // challenge, kernel identity or endpoint, nor amplify fallback replies.
+                if (JSON.stringify(frame) !== JSON.stringify(this.#surfaceReceiver)) {
+                    return this.#failProtocol("protocol-error");
+                }
+                return;
+            }
+            this.#surfaceReceiver = frame;
+            if (this.#descriptor.terminalDelivery.protocolVersions.includes(TERMINAL_SURFACE_CAPABILITY_VERSION)) {
+                const native = this.#nativeSurfaces?.open(frame, this.#descriptor.panes, (control) => {
+                    if (this.#closed)
+                        return false;
+                    this.#surfaceControl(control.type);
+                    this.#sendFrame(null, control);
+                    return !this.#closed;
+                }, (pane, generation) => {
+                    const channel = this.#panes.get(pane);
+                    const address = channel?.deliveryAddress;
+                    if (this.#closed ||
+                        !channel ||
+                        channel.closed ||
+                        !channel.delivery ||
+                        !address ||
+                        address.generation !== generation)
+                        return null;
+                    return address.deliveryNonce;
+                }) ?? null;
+                if (this.#closed) {
+                    native?.close();
+                    return;
+                }
+                this.#surfaceSession = native;
+                if (native)
+                    return;
+            }
+            // No provider/capability means no PID/slot lookup and no inferred locality.
+            this.#sendFrame(null, {
+                type: "terminal-surface-fallback",
+                version: 1,
+                generation: frame.generation,
+                connectionNonce: frame.connectionNonce,
+                reason: "unsupported",
+            });
+            return;
+        }
+        if (frame.type === "terminal-surface-fallback") {
+            if (this.#surfaceViewer) {
+                if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES || !this.#surfaceViewer.fallback(frame))
+                    this.#failProtocol("protocol-error");
+                return;
+            }
+            if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES ||
+                frame.generation !== this.#surfaceReceiver?.generation ||
+                frame.connectionNonce !== this.#surfaceReceiver.connectionNonce) {
+                return this.#failProtocol("protocol-error");
+            }
+            this.#surfaceSession?.fallback(frame.reason);
+            return;
+        }
+        if (frame.type === "terminal-surface-retired") {
+            if (this.#surfaceViewer) {
+                if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES || !this.#surfaceViewer.retire(frame))
+                    this.#failProtocol("protocol-error");
+                return;
+            }
+            if (byteLength <= TERMINAL_SURFACE_MAX_CONTROL_BYTES &&
+                frame.generation === this.#surfaceReceiver?.generation &&
+                frame.connectionNonce === this.#surfaceReceiver.connectionNonce &&
+                this.#surfaceSession?.retire(frame.sequence))
+                return;
+            // Unissued/replayed ACKs must not retire another socket's native storage.
+            return this.#failProtocol("protocol-error");
+        }
         if (frame.type === "terminal-graphics-request") {
-            this.#requestGraphics(frame.pane, frame.digests);
+            this.#requestGraphics(frame.pane, frame.digests, frame.preferInline === true);
+            return;
+        }
+        if (frame.type === "terminal-graphics-local-release") {
+            const lease = this.#localLeases.get(frame.leaseId);
+            if (lease &&
+                lease.pane === frame.pane &&
+                frame.generation === this.#localImages?.identity.generation) {
+                this.#localLeases.delete(frame.leaseId);
+                void this.#localImages
+                    ?.release({
+                    generation: frame.generation,
+                    client: this.#descriptor.leaseId,
+                    id: frame.leaseId,
+                })
+                    .catch(() => { });
+            }
             return;
         }
         if (frame.type === "terminal-cell-pixels") {
@@ -2127,6 +2472,12 @@ export class PaneStreamLiveConnection {
         this.#evaluateResume(pane);
     }
     #acceptInput(pane, seq, kind, data, frameBytes, performanceTraceId, causalProbe, ingressAtMicros = null, callbackAtSharedMicros = null, ingressAtSharedMicros = null, expectedLifetime) {
+        if (kind === "bytes" &&
+            (!this.#sessionRuntimeBinding ||
+                !this.#descriptor.terminalDelivery?.protocolVersions.includes(PANE_STREAM_INPUT_BYTES_CAPABILITY_VERSION))) {
+            this.#failProtocol("input-rejected");
+            return;
+        }
         if (expectedLifetime &&
             (!this.#sessionRuntimeBinding ||
                 !this.#descriptor.terminalDelivery?.protocolVersions.includes(PANE_STREAM_INPUT_LIFETIME_CAPABILITY_VERSION))) {
@@ -2227,8 +2578,10 @@ export class PaneStreamLiveConnection {
                     : undefined, expectedLifetime);
             else if (kind === "text")
                 channel.sub.sendText(data);
-            else
+            else if (kind === "key")
                 channel.sub.sendKey(data);
+            else
+                throw new Error("Byte input requires semantic runtime");
             let ackStartedAtMicros = null;
             let ackStartedAtSharedMicros = null;
             if (trace) {

package/packages/daemon/dist/terminal/pane-stream/runtime.js

@@ -1,3 +1,4 @@
+import { NativeSurfaceProviderRegistry } from "../graphics/native-surface-provider.js";
 import { SessionRuntimeTransportBinder } from "../session-runtime/transport-binding.js";
 import { PaneStreamLeaseManager } from "./lease-manager.js";
 import { PaneStreamAdmissionCoordinator, } from "./pane-stream-websocket.js";
@@ -8,14 +9,22 @@ import { PaneStreamAdmissionCoordinator, } from "./pane-stream-websocket.js";
  */
 export class PaneStreamRuntime {
     coordinator;
+    nativeSurfaces;
     #disposePromise = null;
     constructor(options) {
+        this.nativeSurfaces =
+            options.nativeSurfaces ??
+                (process.platform === "darwin"
+                    ? new NativeSurfaceProviderRegistry(options.daemonInstanceId)
+                    : undefined);
         const leaseManager = new PaneStreamLeaseManager({
             daemonInstanceId: options.daemonInstanceId,
         });
         const transportBinder = new SessionRuntimeTransportBinder(options.sessionRuntimeRegistry);
         this.coordinator = new PaneStreamAdmissionCoordinator({
             ...options.admission,
+            localImages: options.localImages,
+            nativeSurfaces: this.nativeSurfaces,
             daemonInstanceId: options.daemonInstanceId,
             webSocketUrl: options.webSocketUrl,
             leaseManager,
@@ -42,6 +51,7 @@ export class PaneStreamRuntime {
     dispose() {
         if (!this.#disposePromise) {
             this.#disposePromise = (async () => {
+                this.nativeSurfaces?.close();
                 await this.coordinator.shutdown();
             })();
         }

package/packages/daemon/dist/terminal/session-runtime/registry.js

@@ -1653,6 +1653,8 @@ class SessionRuntimeConsumerImpl {
             negotiation: upstream.negotiation,
             closed: upstream.closed,
             ack: (ack) => upstream.ack(ack),
+            ...(upstream.projectionReceipt ? { projectionReceipt: upstream.projectionReceipt } : {}),
+            ...(upstream.materialize ? { materialize: upstream.materialize } : {}),
             nack: (nack) => upstream.nack(nack),
             setVisibility: (visibility) => upstream.setVisibility(visibility),
             close: async () => {