tmux-ide 2.9.11 → 2.9.13

package/packages/daemon/src/terminal/pane-stream/pane-stream-websocket.ts

@@ -1,5 +1,27 @@
+import type { SurfaceViewerDemand } from "../graphics/native-surface-demand.ts";
+import {
+  TERMINAL_SURFACE_MAX_CONTROL_BYTES,
+  TERMINAL_SURFACE_CAPABILITY_VERSION,
+  type TerminalSurfaceReceiver,
+} from "@tmux-ide/contracts";
+import type {
+  NativeSurfaceProviderRegistry,
+  NativeSurfaceSession,
+} from "../graphics/native-surface-provider.ts";
+import { NativeSurfaceViewer } from "../graphics/native-surface-viewer.ts";
+import { compressGraphicsChunk } from "../graphics/compress-chunk.ts";
+import {
+  TERMINAL_GRAPHICS_COMPRESSED_CAPABILITY_VERSION,
+  TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION,
+  TERMINAL_PROJECTION_CAPABILITY_VERSION,
+  TERMINAL_PROJECTION_LOCAL_CAPABILITY_VERSION,
+  type TerminalProjectionReceipt,
+  type TerminalProjectionMaterialize,
+} from "@tmux-ide/contracts";
+import type { LocalImageOwner } from "../graphics/local-image-owner.ts";
 import {
   PANE_STREAM_INPUT_LIFETIME_CAPABILITY_VERSION,
+  PANE_STREAM_INPUT_BYTES_CAPABILITY_VERSION,
   type PaneStreamInputLifetime,
 } from "@tmux-ide/contracts";
 import { inflateSync } from "node:zlib";
@@ -15,6 +37,12 @@ import {
   PANE_STREAM_REDEEM_PATH,
   PANE_STREAM_WEBSOCKET_SUBPROTOCOL,
   TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES,
+  TERMINAL_DELIVERY_BINARY_CAPABILITY_VERSION,
+  TERMINAL_DELIVERY_BINARY_LOCAL_CAPABILITY_VERSION,
+  TERMINAL_GRAPHICS_BINARY_CAPABILITY_VERSION,
+  TERMINAL_GRAPHICS_LOCAL_CAPABILITY_VERSION,
+  encodeTerminalGraphicsBinary,
+  encodeTerminalDeliveryBinaryChunk,
   TERMINAL_GRAPHICS_CELL_PIXELS_CAPABILITY_VERSION,
   TERMINAL_GRAPHICS_PRECISE_PIXELS_CAPABILITY_VERSION,
   TERMINAL_GRAPHICS_LEGACY_MAX_IMAGE_BYTES,
@@ -245,6 +273,8 @@ export interface PaneStreamAdmissionSnapshot {
 }
 
 export interface PaneStreamAdmissionCoordinatorOptions {
+  readonly nativeSurfaces?: NativeSurfaceProviderRegistry;
+  readonly localImages?: LocalImageOwner;
   readonly daemonInstanceId: string;
   readonly webSocketUrl: string;
   readonly leaseManager: PaneStreamLeaseAuthority;
@@ -326,6 +356,8 @@ export interface SessionRuntimeTerminalDeliveryConnection {
   readonly closed?: Promise<"closed" | "sink-failed">;
   readonly negotiation: TerminalDeliveryNegotiationResult;
   ack(ack: TerminalDeliveryAck): void;
+  projectionReceipt?(receipt: TerminalProjectionReceipt): void;
+  materialize?(request: TerminalProjectionMaterialize): void;
   nack(nack: TerminalDeliveryNack): void;
   setVisibility(visibility: TerminalDeliveryVisibility): void;
   close(): Promise<void>;
@@ -388,6 +420,8 @@ function errorFrameCode(error: unknown): PaneStreamErrorFrameCode {
  * ticket digests; a daemon restart makes every prior descriptor inert.
  */
 export class PaneStreamAdmissionCoordinator {
+  readonly #nativeSurfaces: NativeSurfaceProviderRegistry | undefined;
+  readonly #localImages: LocalImageOwner | undefined;
   readonly #instanceId: string;
   readonly #webSocketUrl: string;
   readonly #leaseManager: PaneStreamLeaseAuthority;
@@ -421,6 +455,10 @@ export class PaneStreamAdmissionCoordinator {
   #shutdownPromise: Promise<void> | null = null;
 
   constructor(options: PaneStreamAdmissionCoordinatorOptions) {
+    this.#nativeSurfaces = options.nativeSurfaces;
+    if (this.#nativeSurfaces && this.#nativeSurfaces.generation !== options.daemonInstanceId)
+      throw new Error("Native surface provider generation mismatch");
+    this.#localImages = options.localImages;
     this.#instanceId = BindingIdSchemaZ.parse(options.daemonInstanceId);
     this.#webSocketUrl = PaneStreamLoopbackWebSocketUrlSchemaZ.parse(options.webSocketUrl);
     this.#leaseManager = options.leaseManager;
@@ -792,6 +830,8 @@ export class PaneStreamAdmissionCoordinator {
         let live: PaneStreamLiveConnection;
         try {
           live = new PaneStreamLiveConnection({
+            nativeSurfaces: this.#nativeSurfaces,
+            localImages: this.#localImages,
             clientId: `psc-${this.#clientCounter}`,
             socket,
             descriptor: structuredClone(descriptor),
@@ -1010,6 +1050,8 @@ class PreAuthAdmission implements PaneStreamPreAuthAdmission {
 }
 
 interface LiveConnectionOptions {
+  readonly nativeSurfaces?: NativeSurfaceProviderRegistry;
+  readonly localImages?: LocalImageOwner;
   readonly clientId: string;
   readonly socket: DirectTerminalSocket;
   readonly descriptor: PaneStreamLeaseDescriptor;
@@ -1066,6 +1108,11 @@ interface PaneChannel {
   graphicsServed: Map<number, string>;
   graphicsState: TerminalGraphicsState | null;
   graphicsZlib?: boolean;
+  graphicsBinary?: boolean;
+  graphicsCompressed?: boolean;
+  binaryCells?: boolean;
+  graphicsLocal?: boolean;
+  graphicsInline?: Set<string>;
 }
 
 interface QueuedSend {
@@ -1097,6 +1144,15 @@ export function rawGraphicsState(state: TerminalGraphicsState): TerminalGraphics
 const GRAPHICS_SEND_WINDOW_BYTES = 2 * 1024 * 1024;
 
 export class PaneStreamLiveConnection {
+  // One bounded native negotiation per redeemed socket; never reuse on reconnect.
+  readonly #nativeSurfaces: NativeSurfaceProviderRegistry | undefined;
+  #surfaceViewer: NativeSurfaceViewer | null = null;
+  #surfaceDemands = new Map<string, SurfaceViewerDemand>();
+  #surfaceNative = false;
+  #surfaceSession: NativeSurfaceSession | null = null;
+  #surfaceReceiver: TerminalSurfaceReceiver | null = null;
+  readonly #localImages: LocalImageOwner | undefined;
+  readonly #localLeases = new Map<string, { pane: string; expiresAt: number }>();
   readonly #clientId: string;
   readonly #socket: DirectTerminalSocket;
   readonly #descriptor: PaneStreamLeaseDescriptor;
@@ -1221,6 +1277,8 @@ export class PaneStreamLiveConnection {
   }
 
   constructor(options: LiveConnectionOptions) {
+    this.#nativeSurfaces = options.nativeSurfaces;
+    this.#localImages = options.localImages;
     this.#clientId = options.clientId;
     this.#socket = options.socket;
     this.#descriptor = options.descriptor;
@@ -1294,6 +1352,12 @@ export class PaneStreamLiveConnection {
         )
           ? { inputLifetime: true as const }
           : {}),
+        ...(this.#sessionRuntimeBinding &&
+        this.#descriptor.terminalDelivery?.protocolVersions.includes(
+          PANE_STREAM_INPUT_BYTES_CAPABILITY_VERSION,
+        )
+          ? { inputBytes: true as const }
+          : {}),
         protocolVersion: PANE_STREAM_PROTOCOL_VERSION,
         daemonInstanceId: this.#binding.daemonInstanceId,
         requestId: this.#binding.requestId,
@@ -1315,6 +1379,7 @@ export class PaneStreamLiveConnection {
       this.#sessionRuntimeBinding?.onAuthoritySnapshot?.((snapshot) =>
         this.#usesExplicitAuthority ? this.#sendAuthoritySnapshot(snapshot) : undefined,
       ) ?? null;
+    if (!this.#descriptor.terminalDelivery) this.#observeSurfaceDemand(this.#descriptor.panes);
     void this.#subscribeAll();
   }
 
@@ -1322,6 +1387,12 @@ export class PaneStreamLiveConnection {
     if (this.#closed) return;
     this.#recordDiagnosticLifecycle("pane-stream-terminal", code, reason);
     this.#closed = true;
+    for (const demand of this.#surfaceDemands.values()) demand.close();
+    this.#surfaceDemands.clear();
+    this.#surfaceViewer?.close();
+    this.#surfaceViewer = null;
+    this.#surfaceSession?.close();
+    this.#surfaceSession = null;
     this.#cancelDrainTick?.();
     this.#cancelDrainTick = null;
     // Departure force-returns every ticket within this same tick.
@@ -1336,6 +1407,14 @@ export class PaneStreamLiveConnection {
     this.#stopAuthoritySnapshots?.();
     this.#stopAuthoritySnapshots = null;
     const closures: Promise<unknown>[] = [];
+    this.#localLeases.clear();
+    if (this.#localImages)
+      closures.push(
+        this.#localImages.releaseClient({
+          generation: this.#localImages.identity.generation,
+          client: this.#descriptor.leaseId,
+        }),
+      );
     const layoutSubscription = this.#layoutSubscription;
     this.#layoutSubscription = null;
     if (layoutSubscription) closures.push(layoutSubscription.close().catch(() => undefined));
@@ -1345,6 +1424,8 @@ export class PaneStreamLiveConnection {
       const delivery = channel.delivery;
       channel.delivery = null;
       channel.closed = true;
+      this.#surfaceDemands.get(channel.semanticPaneId)?.close();
+      this.#surfaceDemands.delete(channel.semanticPaneId);
       if (sub) closures.push(sub.close().catch(() => undefined));
       if (delivery) closures.push(delivery.close().catch(() => undefined));
     }
@@ -1564,7 +1645,8 @@ export class PaneStreamLiveConnection {
       if (negotiated.generation !== this.#binding.daemonInstanceId) return true;
       let incarnation: string | null = null;
       for (const message of pending) {
-        if (message.type !== "terminal.delivery") continue;
+        if (message.type !== "terminal.delivery" && message.type !== "terminal.projection")
+          continue;
         if (
           message.workspaceName !== this.#descriptor.workspaceName ||
           message.semanticPaneId !== channel.semanticPaneId ||
@@ -1686,13 +1768,19 @@ export class PaneStreamLiveConnection {
   }
 
   async #sendTerminalDelivery(pane: string, message: TerminalDeliveryServerMessage): Promise<void> {
-    if (message.type === "terminal.delivery") {
+    if (message.type === "terminal.projection") {
+      const channel = this.#panes.get(pane);
+      if (channel?.deliveryAddress) channel.deliveryAddress.incarnation = message.incarnation;
+      if (channel) channel.binaryCells = true;
+      this.#sendFrame(pane, { type: "terminal-projection-envelope", pane, envelope: message });
+    } else if (message.type === "terminal.delivery") {
       if (!this.#diagnosticFirstSeedObserved) {
         this.#diagnosticFirstSeedObserved = true;
         this.#recordDiagnosticLifecycle("pane-stream-first-seed");
       }
       const channel = this.#panes.get(pane);
       if (channel?.deliveryAddress) channel.deliveryAddress.incarnation = message.incarnation;
+      if (channel) channel.binaryCells = message.encoding === "semantic-binary-v1";
       // The trusted lease address was fixed at delivery admission; preserve the
       // exact envelope and its observation identity through transmission.
       const detailedDeliveryObservation = this.#observability?.enabled === true;
@@ -1775,13 +1863,26 @@ export class PaneStreamLiveConnection {
         }
       }
     } else if (message.type === "terminal.delivery.chunk") {
-      this.#sendFrame(pane, {
-        type: "terminal-delivery-chunk",
-        pane,
-        transactionId: message.transactionId,
-        index: message.index,
-        data: Buffer.from(message.bytes).toString("base64"),
-      });
+      if (this.#panes.get(pane)?.binaryCells) {
+        this.#sendEncoded(
+          pane,
+          Buffer.from(
+            encodeTerminalDeliveryBinaryChunk(
+              { pane, transactionId: message.transactionId, index: message.index },
+              message.bytes,
+            ),
+          ),
+          true,
+        );
+      } else {
+        this.#sendFrame(pane, {
+          type: "terminal-delivery-chunk",
+          pane,
+          transactionId: message.transactionId,
+          index: message.index,
+          data: Buffer.from(message.bytes).toString("base64"),
+        });
+      }
     } else {
       this.#sendFrame(pane, { type: "terminal-delivery-fault", pane, fault: message });
       // Lifecycle/control truth must never queue behind bulk terminal bytes.
@@ -1792,10 +1893,27 @@ export class PaneStreamLiveConnection {
     await this.#awaitSemanticCredit(pane);
   }
 
+  #observeSurfaceDemand(panes: readonly string[]): void {
+    for (const pane of panes) {
+      if (this.#surfaceDemands.has(pane)) continue;
+      const demand = this.#nativeSurfaces?.demand.viewer(this.#descriptor.sessionName, [pane]);
+      if (!demand) return;
+      demand.native(this.#surfaceNative);
+      this.#surfaceDemands.set(pane, demand);
+    }
+  }
+
+  #surfaceControl(type: string): void {
+    if (type === "terminal-surface-offer") return;
+    this.#surfaceNative = type === "terminal-surface-peer";
+    for (const demand of this.#surfaceDemands.values()) demand.native(this.#surfaceNative);
+  }
+
   /** Announce the pane's kitty graphics state; the client pulls bytes it lacks. */
   #startGraphics(channel: PaneChannel): void {
     const graphics = channel.delivery?.graphics;
     if (!graphics) return;
+    this.#observeSurfaceDemand([channel.semanticPaneId]);
     // Renderers that cannot inflate see raw formats; their bytes are inflated
     // when served.
     const zlib =
@@ -1803,6 +1921,34 @@ export class PaneStreamLiveConnection {
         TERMINAL_GRAPHICS_ZLIB_CAPABILITY_VERSION,
       ) === true;
     channel.graphicsZlib = zlib;
+    channel.graphicsLocal =
+      !!this.#localImages &&
+      this.#descriptor.terminalDelivery?.protocolVersions.some(
+        (version) =>
+          version === TERMINAL_GRAPHICS_LOCAL_CAPABILITY_VERSION ||
+          version === TERMINAL_DELIVERY_BINARY_LOCAL_CAPABILITY_VERSION ||
+          version === TERMINAL_PROJECTION_LOCAL_CAPABILITY_VERSION ||
+          version === TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION,
+      ) === true;
+    channel.graphicsCompressed =
+      this.#descriptor.terminalDelivery?.protocolVersions.some(
+        (version) =>
+          version === TERMINAL_GRAPHICS_COMPRESSED_CAPABILITY_VERSION ||
+          version === TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION,
+      ) === true;
+    channel.graphicsInline = new Set();
+    channel.graphicsBinary =
+      this.#descriptor.terminalDelivery?.protocolVersions.some(
+        (version) =>
+          version === TERMINAL_GRAPHICS_BINARY_CAPABILITY_VERSION ||
+          version === TERMINAL_GRAPHICS_LOCAL_CAPABILITY_VERSION ||
+          version === TERMINAL_DELIVERY_BINARY_CAPABILITY_VERSION ||
+          version === TERMINAL_PROJECTION_CAPABILITY_VERSION ||
+          version === TERMINAL_DELIVERY_BINARY_LOCAL_CAPABILITY_VERSION ||
+          version === TERMINAL_PROJECTION_LOCAL_CAPABILITY_VERSION ||
+          version === TERMINAL_GRAPHICS_COMPRESSED_CAPABILITY_VERSION ||
+          version === TERMINAL_GRAPHICS_COMPRESSED_LOCAL_CAPABILITY_VERSION,
+      ) === true;
     // Renderers that report cell pixels learn up front that this daemon
     // accepts them, before any image exists.
     const cellPixels =
@@ -1834,6 +1980,9 @@ export class PaneStreamLiveConnection {
           ...(zlib ? state : rawGraphicsState(state)),
           ...(cellPixels ? { cellPixels: true as const } : {}),
           ...(preciseCellPixels ? { preciseCellPixels: true as const } : {}),
+          ...(channel.graphicsBinary ? { binaryBlobs: true as const } : {}),
+          ...(channel.graphicsCompressed ? { compressedBlobs: true as const } : {}),
+          ...(channel.graphicsLocal ? { localFiles: true as const } : {}),
         },
       });
       if (pushed.length > 0) this.#queueGraphics(channel, pushed);
@@ -1849,15 +1998,24 @@ export class PaneStreamLiveConnection {
           placements: [],
           cellPixels: true,
           ...(preciseCellPixels ? { preciseCellPixels: true as const } : {}),
+          ...(channel.graphicsBinary ? { binaryBlobs: true as const } : {}),
+          ...(channel.graphicsCompressed ? { compressedBlobs: true as const } : {}),
+          ...(channel.graphicsLocal ? { localFiles: true as const } : {}),
         },
       });
     }
   }
 
-  #requestGraphics(pane: string, digests: readonly string[]): void {
+  #requestGraphics(pane: string, digests: readonly string[], preferInline = false): void {
     const channel = this.#panes.get(pane);
     // Requests are only meaningful after an announcement; ignore stray ones.
     if (!channel || channel.closed || !channel.graphicsAnnounced) return;
+    if (preferInline && channel.graphicsLocal) {
+      for (const digest of digests) {
+        if ((channel.graphicsInline?.size ?? 0) < GRAPHICS_MAX_QUEUED_DIGESTS)
+          channel.graphicsInline?.add(digest);
+      }
+    }
     this.#queueGraphics(channel, digests);
   }
 
@@ -1881,6 +2039,7 @@ export class PaneStreamLiveConnection {
     try {
       while (!this.#closed && !channel.closed && channel.graphicsQueue.length > 0) {
         const digest = channel.graphicsQueue.shift()!;
+        const inline = channel.graphicsInline?.delete(digest) ?? false;
         const graphics = channel.delivery?.graphics;
         const image = graphics?.image(digest);
         if (!graphics || !image) continue;
@@ -1902,6 +2061,47 @@ export class PaneStreamLiveConnection {
           continue;
         const format = raw ? (image.format === "rgba-zlib" ? "rgba" : "rgb") : image.format;
         const total = data.byteLength;
+        if (!inline && channel.graphicsLocal && !raw && this.#localImages) {
+          for (const [id, lease] of this.#localLeases)
+            if (lease.expiresAt <= Date.now()) this.#localLeases.delete(id);
+          if (this.#localLeases.size < 64) {
+            try {
+              const lease = await this.#localImages.acquire({
+                generation: this.#localImages.identity.generation,
+                client: this.#descriptor.leaseId,
+                digest,
+                bytes: data,
+              });
+              if (this.#closed || channel.closed || graphics.image(digest) !== image) {
+                await this.#localImages.release(lease);
+                continue;
+              }
+              this.#localLeases.set(lease.id, {
+                pane: channel.semanticPaneId,
+                expiresAt: lease.expiresAt,
+              });
+              this.#sendFrame(null, {
+                type: "terminal-graphics-local",
+                pane: channel.semanticPaneId,
+                generation: lease.generation,
+                leaseId: lease.id,
+                path: lease.path,
+                expiresAt: lease.expiresAt,
+                blob: {
+                  digest,
+                  format,
+                  width: image.width,
+                  height: image.height,
+                  total,
+                  offset: 0,
+                },
+              });
+              continue;
+            } catch {
+              /* Optional transport: retain the negotiated inline fallback. */
+            }
+          }
+        }
         for (let offset = 0; offset < total; offset += TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES) {
           while ((this.#socket.bufferedAmount ?? 0) > GRAPHICS_SEND_WINDOW_BYTES) {
             if (this.#closed || channel.closed) return;
@@ -1912,6 +2112,50 @@ export class PaneStreamLiveConnection {
           // the renderer drops the partial image and shows the newer one.
           if (offset > 0 && graphics.image(digest) !== image) break;
           const end = Math.min(total, offset + TERMINAL_GRAPHICS_BLOB_CHUNK_BYTES);
+          if (channel.graphicsBinary) {
+            const header = {
+              pane: channel.semanticPaneId,
+              blob: {
+                digest,
+                format,
+                width: image.width,
+                height: image.height,
+                total,
+                offset,
+              },
+            };
+            const compressed = channel.graphicsCompressed
+              ? await compressGraphicsChunk(header, data.subarray(offset, end))
+              : null;
+            // Compression yields: a pane may close or replace this image meanwhile.
+            if (this.#closed || channel.closed) return;
+            if (graphics.image(digest) !== image) break;
+            if (compressed) {
+              this.#sendEncoded(null, Buffer.from(compressed), true);
+              continue;
+            }
+            this.#sendEncoded(
+              null,
+              Buffer.from(
+                encodeTerminalGraphicsBinary(
+                  {
+                    pane: channel.semanticPaneId,
+                    blob: {
+                      digest,
+                      format,
+                      width: image.width,
+                      height: image.height,
+                      total,
+                      offset,
+                    },
+                  },
+                  data.subarray(offset, end),
+                ),
+              ),
+              true,
+            );
+            continue;
+          }
           this.#sendFrame(null, {
             type: "terminal-graphics-blob",
             pane: channel.semanticPaneId,
@@ -2067,6 +2311,8 @@ export class PaneStreamLiveConnection {
   #emitClosed(channel: PaneChannel): void {
     if (channel.closed) return;
     channel.closed = true;
+    this.#surfaceDemands.get(channel.semanticPaneId)?.close();
+    this.#surfaceDemands.delete(channel.semanticPaneId);
     const sub = channel.sub;
     channel.sub = null;
     if (sub) void sub.close().catch(() => undefined);
@@ -2245,7 +2491,12 @@ export class PaneStreamLiveConnection {
   #sendFrame(pane: string | null, frame: Record<string, unknown>): void {
     if (this.#closed || this.#socket.readyState !== WS_OPEN) return;
     const encoded = JSON.stringify(frame);
-    const bytes = Buffer.byteLength(encoded, "utf8");
+    this.#sendEncoded(pane, encoded, false);
+  }
+
+  #sendEncoded(pane: string | null, encoded: string | Buffer, binary: boolean): void {
+    if (this.#closed || this.#socket.readyState !== WS_OPEN) return;
+    const bytes = Buffer.byteLength(encoded);
     if (pane !== null) {
       this.#ledger.take(this.#clientId, pane, "ws-send-buffer", bytes);
       if (this.#deliveryAcks) this.#ledger.take(this.#clientId, pane, "renderer-backlog", 1);
@@ -2253,7 +2504,7 @@ export class PaneStreamLiveConnection {
     this.#sendQueue.push({ pane, remaining: bytes });
     this.#sentBytesTotal += bytes;
     try {
-      this.#socket.send(encoded, { binary: false });
+      this.#socket.send(encoded, { binary });
     } catch {
       this.close(1011, "stream-unavailable");
       return;
@@ -2475,6 +2726,20 @@ export class PaneStreamLiveConnection {
       });
       return;
     }
+    if (frame.type === "terminal-projection-receipt") {
+      const channel = this.#deliveryChannel(frame.receipt);
+      if (!channel) return;
+      if (!channel.delivery?.projectionReceipt) return this.#failProtocol("protocol-error");
+      channel.delivery.projectionReceipt(frame.receipt);
+      return;
+    }
+    if (frame.type === "terminal-projection-materialize") {
+      const channel = this.#deliveryChannel(frame.request);
+      if (!channel) return;
+      if (!channel.delivery?.materialize) return this.#failProtocol("protocol-error");
+      channel.delivery.materialize(frame.request);
+      return;
+    }
     if (frame.type === "terminal-delivery-ack") {
       const channel = this.#deliveryChannel(frame.ack);
       if (!channel) return;
@@ -2503,8 +2768,163 @@ export class PaneStreamLiveConnection {
       this.#acceptSemanticIntent(frame.operationId, frame.intent);
       return;
     }
+    if (frame.type === "terminal-surface-watch") {
+      if (
+        byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES ||
+        frame.generation !== this.#sessionRuntimeBinding?.generation ||
+        !this.#descriptor.terminalDelivery?.protocolVersions.includes(
+          TERMINAL_SURFACE_CAPABILITY_VERSION,
+        ) ||
+        !this.#nativeSurfaces ||
+        this.#surfaceReceiver
+      )
+        return this.#failProtocol("protocol-error");
+      if (this.#surfaceViewer) return;
+      this.#surfaceViewer = new NativeSurfaceViewer(
+        this.#nativeSurfaces,
+        this.#descriptor.panes,
+        (control) => {
+          if (this.#closed) return false;
+          this.#surfaceControl(control.type);
+          this.#sendFrame(null, control);
+          return !this.#closed;
+        },
+        (pane, generation) => {
+          const channel = this.#panes.get(pane);
+          const address = channel?.deliveryAddress;
+          return !this.#closed &&
+            channel &&
+            !channel.closed &&
+            channel.delivery &&
+            address?.generation === generation
+            ? address.deliveryNonce
+            : null;
+        },
+      );
+      this.#surfaceViewer.start();
+      return;
+    }
+    if (frame.type === "terminal-surface-receiver") {
+      if (this.#surfaceViewer) {
+        if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES || !this.#surfaceViewer.receive(frame))
+          this.#failProtocol("protocol-error");
+        return;
+      }
+      if (
+        byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES ||
+        !this.#descriptor.terminalDelivery ||
+        frame.generation !== this.#sessionRuntimeBinding?.generation
+      ) {
+        return this.#failProtocol("protocol-error");
+      }
+      if (this.#surfaceReceiver) {
+        // Parsed schema order is deterministic. A duplicate cannot change the
+        // challenge, kernel identity or endpoint, nor amplify fallback replies.
+        if (JSON.stringify(frame) !== JSON.stringify(this.#surfaceReceiver)) {
+          return this.#failProtocol("protocol-error");
+        }
+        return;
+      }
+      this.#surfaceReceiver = frame;
+      if (
+        this.#descriptor.terminalDelivery.protocolVersions.includes(
+          TERMINAL_SURFACE_CAPABILITY_VERSION,
+        )
+      ) {
+        const native =
+          this.#nativeSurfaces?.open(
+            frame,
+            this.#descriptor.panes,
+            (control) => {
+              if (this.#closed) return false;
+              this.#surfaceControl(control.type);
+              this.#sendFrame(null, control);
+              return !this.#closed;
+            },
+            (pane, generation) => {
+              const channel = this.#panes.get(pane);
+              const address = channel?.deliveryAddress;
+              if (
+                this.#closed ||
+                !channel ||
+                channel.closed ||
+                !channel.delivery ||
+                !address ||
+                address.generation !== generation
+              )
+                return null;
+              return address.deliveryNonce;
+            },
+          ) ?? null;
+        if (this.#closed) {
+          native?.close();
+          return;
+        }
+        this.#surfaceSession = native;
+        if (native) return;
+      }
+      // No provider/capability means no PID/slot lookup and no inferred locality.
+      this.#sendFrame(null, {
+        type: "terminal-surface-fallback",
+        version: 1,
+        generation: frame.generation,
+        connectionNonce: frame.connectionNonce,
+        reason: "unsupported",
+      });
+      return;
+    }
+    if (frame.type === "terminal-surface-fallback") {
+      if (this.#surfaceViewer) {
+        if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES || !this.#surfaceViewer.fallback(frame))
+          this.#failProtocol("protocol-error");
+        return;
+      }
+      if (
+        byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES ||
+        frame.generation !== this.#surfaceReceiver?.generation ||
+        frame.connectionNonce !== this.#surfaceReceiver.connectionNonce
+      ) {
+        return this.#failProtocol("protocol-error");
+      }
+      this.#surfaceSession?.fallback(frame.reason);
+      return;
+    }
+    if (frame.type === "terminal-surface-retired") {
+      if (this.#surfaceViewer) {
+        if (byteLength > TERMINAL_SURFACE_MAX_CONTROL_BYTES || !this.#surfaceViewer.retire(frame))
+          this.#failProtocol("protocol-error");
+        return;
+      }
+      if (
+        byteLength <= TERMINAL_SURFACE_MAX_CONTROL_BYTES &&
+        frame.generation === this.#surfaceReceiver?.generation &&
+        frame.connectionNonce === this.#surfaceReceiver.connectionNonce &&
+        this.#surfaceSession?.retire(frame.sequence)
+      )
+        return;
+      // Unissued/replayed ACKs must not retire another socket's native storage.
+      return this.#failProtocol("protocol-error");
+    }
     if (frame.type === "terminal-graphics-request") {
-      this.#requestGraphics(frame.pane, frame.digests);
+      this.#requestGraphics(frame.pane, frame.digests, frame.preferInline === true);
+      return;
+    }
+    if (frame.type === "terminal-graphics-local-release") {
+      const lease = this.#localLeases.get(frame.leaseId);
+      if (
+        lease &&
+        lease.pane === frame.pane &&
+        frame.generation === this.#localImages?.identity.generation
+      ) {
+        this.#localLeases.delete(frame.leaseId);
+        void this.#localImages
+          ?.release({
+            generation: frame.generation,
+            client: this.#descriptor.leaseId,
+            id: frame.leaseId,
+          })
+          .catch(() => {});
+      }
       return;
     }
     if (frame.type === "terminal-cell-pixels") {
@@ -2720,7 +3140,7 @@ export class PaneStreamLiveConnection {
   #acceptInput(
     pane: string,
     seq: number,
-    kind: "text" | "key",
+    kind: "text" | "key" | "bytes",
     data: string,
     frameBytes: number,
     performanceTraceId?: string,
@@ -2730,6 +3150,16 @@ export class PaneStreamLiveConnection {
     ingressAtSharedMicros: number | null = null,
     expectedLifetime?: PaneStreamInputLifetime,
   ): void {
+    if (
+      kind === "bytes" &&
+      (!this.#sessionRuntimeBinding ||
+        !this.#descriptor.terminalDelivery?.protocolVersions.includes(
+          PANE_STREAM_INPUT_BYTES_CAPABILITY_VERSION,
+        ))
+    ) {
+      this.#failProtocol("input-rejected");
+      return;
+    }
     if (
       expectedLifetime &&
       (!this.#sessionRuntimeBinding ||
@@ -2862,7 +3292,8 @@ export class PaneStreamLiveConnection {
           expectedLifetime,
         );
       else if (kind === "text") channel.sub!.sendText(data);
-      else channel.sub!.sendKey(data);
+      else if (kind === "key") channel.sub!.sendKey(data);
+      else throw new Error("Byte input requires semantic runtime");
       let ackStartedAtMicros: number | null = null;
       let ackStartedAtSharedMicros: number | null = null;
       if (trace) {