tywrap 0.7.0 → 0.8.0

package/src/runtime/transport.ts

@@ -26,6 +26,28 @@ export const PROTOCOL_ID = 'tywrap/1';
  */
 export const TYWRAP_PROTOCOL_VERSION = Number.parseInt(PROTOCOL_ID.split('/')[1] ?? '', 10);
 
+/**
+ * Framing protocol identifier for chunked large-payload transport.
+ *
+ * This is DISTINCT from {@link PROTOCOL_ID}: the logical RPC stays `tywrap/1`,
+ * while `tywrap-frame/1` describes a separate layer (below {@link Transport.send})
+ * that fragments one logical message across multiple wire frames and reassembles
+ * it. An old bridge rejects any non-`tywrap/1` request, so the logical protocol
+ * must NOT be bumped to negotiate chunking — a separate framing protocol,
+ * advertised through a `tywrap/1` `meta` extension, is used instead.
+ *
+ * Subprocess-only for 0.8.0 (it is the only backend with a real frame ceiling —
+ * the JSONL line-length limit). See docs/transport-framing.md.
+ */
+export const FRAME_PROTOCOL_ID = 'tywrap-frame/1';
+
+/**
+ * Numeric framing-protocol version. Derived from the trailing number of
+ * {@link FRAME_PROTOCOL_ID} so the two cannot drift — bump FRAME_PROTOCOL_ID
+ * alone and this follows (same pattern as {@link TYWRAP_PROTOCOL_VERSION}).
+ */
+export const FRAME_PROTOCOL_VERSION = Number.parseInt(FRAME_PROTOCOL_ID.split('/')[1] ?? '', 10);
+
 // =============================================================================
 // PROTOCOL TYPES
 // =============================================================================
@@ -102,6 +124,78 @@ export interface ProtocolResponse {
   };
 }
 
+// =============================================================================
+// CHUNK FRAMING (tywrap-frame/1)
+// =============================================================================
+
+/**
+ * Per-frame encoding for a {@link ChunkFrame}'s `data` field.
+ *
+ * - `utf8-slice` (the chosen default for 0.8.0): `data` is a raw substring of the
+ *   complete logical JSON message, split on UTF-8 codepoint boundaries. Because
+ *   the logical payload is already valid-UTF-8 JSON, the slices reassemble by
+ *   simple concatenation — no inflation, no extra decode. See
+ *   docs/transport-framing.md for the rationale (decision #6).
+ * - `utf8-base64`: `data` is a base64-encoded chunk of the UTF-8 bytes, safe for
+ *   arbitrary byte splits but ~33% larger on the wire with a memory-amplification
+ *   cost. Reserved as an alternative; not emitted by tywrap in 0.8.0.
+ */
+export type ChunkFrameEncoding = 'utf8-base64' | 'utf8-slice';
+
+/**
+ * A single wire frame of the `tywrap-frame/1` framing protocol.
+ *
+ * A frame envelope is DISTINCT from the logical {@link ProtocolMessage} /
+ * {@link ProtocolResponse}: it carries a slice of the bytes of ONE complete
+ * logical JSON message (a request or a response), fragmented because the payload
+ * exceeds the transport's frame ceiling. The framing layer reassembles all
+ * frames for a given {@link ChunkFrame.id} back into the single logical message
+ * before the JSON/codec path ever sees it.
+ *
+ * Correlation reuses the existing RPC `id`. `seq` is zero-based; `total` and
+ * `totalBytes` are repeated on every frame so the receiver can validate the
+ * stream is complete (no missing/duplicate `seq`, exact frame count, exact
+ * reassembled byte length) before decoding.
+ *
+ * @see docs/transport-framing.md
+ */
+export interface ChunkFrame {
+  /**
+   * Frame-envelope discriminator.
+   * - `'chunk'`: a normal data-carrying frame.
+   * - `'error'`: a framing-layer error (e.g. the sender could not continue the
+   *   stream); carries no further data frames for this `id`.
+   */
+  __tywrap_frame__: 'chunk' | 'error';
+
+  /** Framing protocol identifier (must equal {@link FRAME_PROTOCOL_ID}). */
+  frameProtocol: string;
+
+  /** Which logical stream this frame belongs to. */
+  stream: 'request' | 'response';
+
+  /** RPC correlation id, shared with the logical {@link ProtocolMessage.id}. */
+  id: number;
+
+  /** Zero-based sequence index of this frame within its stream. */
+  seq: number;
+
+  /** Total number of frames in this stream (repeated on every frame). */
+  total: number;
+
+  /**
+   * Total byte length of the complete reassembled logical message (repeated on
+   * every frame). Used to validate the reassembled payload exactly.
+   */
+  totalBytes: number;
+
+  /** Per-frame payload encoding (see {@link ChunkFrameEncoding}). */
+  encoding: ChunkFrameEncoding;
+
+  /** This frame's slice of the logical message, encoded per {@link encoding}. */
+  data: string;
+}
+
 // =============================================================================
 // TRANSPORT CAPABILITIES
 // =============================================================================
@@ -118,9 +212,13 @@ export interface ProtocolResponse {
  * authoritative for transport-level flags; the meta report is authoritative for
  * library availability.
  *
- * Honest for TODAY's behavior: `supportsChunking` and `supportsStreaming` are
- * `false` on every backend — both are planned for 0.8.0 and no backend implements
- * them yet. See docs/transport-capabilities.md for the full matrix.
+ * Honest for TODAY's behavior: as of 0.8.0 `supportsChunking` is implemented for
+ * the subprocess backend and reports the *configured* `tywrap-frame/1` path
+ * (`enableChunking`) — static and lifecycle-independent, like `supportsArrow`;
+ * the per-bridge negotiated fact lives on {@link BridgeInfo}'s `transport` block.
+ * HTTP and Pyodide stay `false`. `supportsStreaming` is `false` on every backend
+ * (not implemented in 0.8.0). See docs/transport-capabilities.md for the full
+ * matrix.
  */
 export interface TransportCapabilities {
   /** Which backend this transport drives. */

package/src/types/index.ts

@@ -413,6 +413,25 @@ export interface TypeMappingConfig {
 /** Known bridge backends. Each speaks the identical "tywrap/1" protocol. */
 export type BridgeBackend = 'python-subprocess' | 'pyodide' | 'http';
 
+/**
+ * Optional chunked-transport negotiation block in {@link BridgeInfo}.
+ *
+ * Reported by a bridge that understands the `tywrap-frame/1` framing protocol
+ * (subprocess only, 0.8.0). It lets the JS side learn, via the `meta` probe,
+ * whether the bridge can reassemble chunked frames and the maximum single-frame
+ * size it will accept. Absent on old bridges (and on HTTP/Pyodide, which stay
+ * single-frame in 0.8.0) — absence means "no chunking", which is backward
+ * compatible. See docs/transport-framing.md.
+ */
+export interface BridgeTransportInfo {
+  /** Framing protocol the bridge speaks (e.g. `'tywrap-frame/1'`). */
+  frameProtocol: string;
+  /** Whether the bridge can fragment/reassemble chunked frames. */
+  supportsChunking: boolean;
+  /** Maximum size, in bytes, of a single wire frame the bridge will accept. */
+  maxFrameBytes: number;
+}
+
 export interface BridgeInfo {
   protocol: string;
   protocolVersion: number;
@@ -426,6 +445,12 @@ export interface BridgeInfo {
   torchAvailable: boolean;
   sklearnAvailable: boolean;
   instances: number;
+  /**
+   * Optional chunked-transport negotiation block. Present only when the bridge
+   * advertises `tywrap-frame/1` framing; absent on old bridges and on
+   * HTTP/Pyodide (single-frame in 0.8.0). See {@link BridgeTransportInfo}.
+   */
+  transport?: BridgeTransportInfo;
 }
 
 // Analysis and generation results

package/src/tywrap.ts

@@ -441,15 +441,7 @@ async function fetchPythonIr(
   try {
     const primary = await execAndParseIr(
       pythonPath,
-      [
-        '-m',
-        'tywrap_ir',
-        '--module',
-        moduleName,
-        '--ir-version',
-        TYWRAP_IR_VERSION,
-        '--no-pretty',
-      ],
+      ['-m', 'tywrap_ir', '--module', moduleName, '--ir-version', TYWRAP_IR_VERSION, '--no-pretty'],
       execOptions,
       'tywrap_ir output'
     );

package/src/utils/codec.ts

@@ -508,6 +508,31 @@ const decodeNdarrayEnvelope: EnvelopeHandler = (value, decodeArrow) => {
   throw new Error(`Invalid ndarray envelope: unsupported encoding ${String(encoding)}`);
 };
 
+/**
+ * Assert an array holds only integer indices within [0, bound). Used to re-validate
+ * scipy.sparse index arrays on the JS side so a corrupt/oversized index can never
+ * silently address out of the declared shape.
+ *
+ * Why: the JS decoder never reconstructs a Python object, but it IS the boundary a
+ * downstream consumer trusts — validating index ranges here turns a corrupt payload
+ * into a clear, early failure instead of a confusing downstream error.
+ */
+function assertIndexArrayInRange(arr: readonly unknown[], bound: number, label: string): void {
+  for (let i = 0; i < arr.length; i += 1) {
+    const idx = arr[i];
+    if (typeof idx !== 'number' || !Number.isInteger(idx)) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: ${label}[${i}] must be an integer, got ${String(idx)}`
+      );
+    }
+    if (idx < 0 || idx >= bound) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: ${label}[${i}]=${idx} is out of range [0, ${bound})`
+      );
+    }
+  }
+}
+
 const decodeScipySparseEnvelope: EnvelopeHandler = value => {
   const encoding = value.encoding;
   if (encoding !== 'json') {
@@ -522,10 +547,16 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
     !Array.isArray(shape) ||
     shape.length !== 2 ||
     typeof shape[0] !== 'number' ||
-    typeof shape[1] !== 'number'
+    !Number.isInteger(shape[0]) ||
+    shape[0] < 0 ||
+    typeof shape[1] !== 'number' ||
+    !Number.isInteger(shape[1]) ||
+    shape[1] < 0
   ) {
-    throw new Error('Invalid scipy.sparse envelope: shape must be a 2-item number[]');
+    throw new Error('Invalid scipy.sparse envelope: shape must be a 2-item non-negative integer[]');
   }
+  const rows = shape[0];
+  const cols = shape[1];
   const data = value.data;
   if (!Array.isArray(data)) {
     throw new Error('Invalid scipy.sparse envelope: data must be an array');
@@ -539,6 +570,16 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
     if (!Array.isArray(row) || !Array.isArray(col)) {
       throw new Error('Invalid scipy.sparse envelope: coo requires row and col arrays');
     }
+    // COO: one (row, col, value) triple per stored entry, so all three arrays
+    // share a length; row/col index into [0, rows)/[0, cols) respectively.
+    if (row.length !== data.length || col.length !== data.length) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: coo row/col/data lengths must match ` +
+          `(data=${data.length}, row=${row.length}, col=${col.length})`
+      );
+    }
+    assertIndexArrayInRange(row, rows, 'row');
+    assertIndexArrayInRange(col, cols, 'col');
     return {
       format,
       shape,
@@ -554,6 +595,52 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
   if (!Array.isArray(indices) || !Array.isArray(indptr)) {
     throw new Error('Invalid scipy.sparse envelope: csr/csc requires indices and indptr arrays');
   }
+  // CSR/CSC: one column-index (CSR) or row-index (CSC) per stored value, so
+  // indices and data share a length. indptr partitions indices into one segment
+  // per major axis (rows for CSR, cols for CSC), so indptr has majorAxis + 1
+  // entries; the inner indices address the minor axis.
+  if (indices.length !== data.length) {
+    throw new Error(
+      `Invalid scipy.sparse envelope: ${format} indices/data lengths must match ` +
+        `(data=${data.length}, indices=${indices.length})`
+    );
+  }
+  const majorAxis = format === 'csr' ? rows : cols;
+  const minorAxis = format === 'csr' ? cols : rows;
+  if (indptr.length !== majorAxis + 1) {
+    throw new Error(
+      `Invalid scipy.sparse envelope: ${format} indptr length must be ${majorAxis + 1} ` +
+        `(${format === 'csr' ? 'rows' : 'cols'}+1), got ${indptr.length}`
+    );
+  }
+  // indptr must be a valid CSR/CSC pointer array: integers that start at 0, end
+  // at data.length, and never decrease — otherwise a structurally impossible
+  // matrix (e.g. [0, 99, 1]) would slip through the length check above.
+  for (let i = 0; i < indptr.length; i += 1) {
+    const ptr = indptr[i];
+    if (typeof ptr !== 'number' || !Number.isInteger(ptr)) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: indptr[${i}] must be an integer, got ${String(ptr)}`
+      );
+    }
+    if (ptr < 0 || ptr > data.length) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: indptr[${i}]=${ptr} is out of range [0, ${data.length}]`
+      );
+    }
+    if (i > 0 && ptr < (indptr[i - 1] as number)) {
+      throw new Error('Invalid scipy.sparse envelope: indptr must be non-decreasing');
+    }
+  }
+  if (indptr[0] !== 0) {
+    throw new Error('Invalid scipy.sparse envelope: indptr must start at 0');
+  }
+  if (indptr[indptr.length - 1] !== data.length) {
+    throw new Error(
+      `Invalid scipy.sparse envelope: indptr must end at data.length (${data.length})`
+    );
+  }
+  assertIndexArrayInRange(indices, minorAxis, 'indices');
   return {
     format,
     shape,
@@ -564,6 +651,11 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
   } satisfies SparseMatrix;
 };
 
+/** Product of a shape's dimensions (the element count). [] (scalar) -> 1. */
+function shapeProduct(shape: readonly number[]): number {
+  return shape.reduce((acc, dim) => acc * dim, 1);
+}
+
 const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
   value: { [k: string]: unknown },
   _decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
@@ -580,20 +672,96 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
   if (!isObject(nested) || (nested as { __tywrap__?: unknown }).__tywrap__ !== 'ndarray') {
     throw new Error('Invalid torch.tensor envelope: value must be an ndarray envelope');
   }
-  const decoded = recurse(nested);
   const shapeValue = value.shape;
   const shape = isNumberArray(shapeValue) ? shapeValue : undefined;
+  // The tensor shape must be a non-negative-integer dimension list. A negative or
+  // non-integer dim is a corrupt envelope, not a valid tensor.
+  if (shape) {
+    for (let i = 0; i < shape.length; i += 1) {
+      const dim = shape[i] as number;
+      if (!Number.isInteger(dim) || dim < 0) {
+        throw new Error(
+          `Invalid torch.tensor envelope: shape[${i}]=${dim} must be a non-negative integer`
+        );
+      }
+    }
+  }
+  // Cross-check the tensor shape's element count against the nested ndarray's
+  // declared shape (metadata only — no decode needed). A mismatch means the two
+  // shapes disagree about how many elements the payload holds.
+  const nestedShapeValue = (nested as { shape?: unknown }).shape;
+  const nestedShape = isNumberArray(nestedShapeValue) ? nestedShapeValue : undefined;
+  if (shape && nestedShape && shapeProduct(shape) !== shapeProduct(nestedShape)) {
+    throw new Error(
+      `Invalid torch.tensor envelope: shape ${JSON.stringify(shape)} ` +
+        `(product ${shapeProduct(shape)}) disagrees with nested ndarray shape ` +
+        `${JSON.stringify(nestedShape)} (product ${shapeProduct(nestedShape)})`
+    );
+  }
   const dtypeValue = value.dtype;
   const dtype = typeof dtypeValue === 'string' ? dtypeValue : undefined;
   const deviceValue = value.device;
+  if (deviceValue !== undefined && (typeof deviceValue !== 'string' || deviceValue.length === 0)) {
+    throw new Error(
+      'Invalid torch.tensor envelope: device must be a non-empty string when provided'
+    );
+  }
   const device = typeof deviceValue === 'string' ? deviceValue : undefined;
 
+  const decoded = recurse(nested);
   if (isPromiseLike(decoded)) {
     return decoded.then(data => ({ data, shape, dtype, device })) as Promise<T | unknown>;
   }
   return { data: decoded, shape, dtype, device } satisfies TorchTensor;
 };
 
+/**
+ * Recursively assert a value is plain JSON (null | boolean | number | string |
+ * JSON array | plain object of JSON). Rejects functions, symbols, bigints, class
+ * instances, and any non-finite number — the things a metadata-only sklearn
+ * envelope must never carry. This validates; it never reconstructs.
+ */
+function assertPlainJson(value: unknown, path: string): void {
+  if (value === null) {
+    return;
+  }
+  const t = typeof value;
+  if (t === 'string' || t === 'boolean') {
+    return;
+  }
+  if (t === 'number') {
+    if (!Number.isFinite(value as number)) {
+      throw new Error(
+        `Invalid sklearn.estimator envelope: ${path} must be a finite JSON number, got ${String(value)}`
+      );
+    }
+    return;
+  }
+  if (Array.isArray(value)) {
+    value.forEach((item, i) => assertPlainJson(item, `${path}[${i}]`));
+    return;
+  }
+  if (t === 'object') {
+    // Reject exotic objects (class instances, Map/Set, etc.): a JSON object is a
+    // plain object whose prototype is Object.prototype or null.
+    const proto = Object.getPrototypeOf(value as object);
+    if (proto !== Object.prototype && proto !== null) {
+      throw new Error(
+        `Invalid sklearn.estimator envelope: ${path} must be a plain JSON object, ` +
+          `got ${(value as object).constructor?.name ?? 'object'}`
+      );
+    }
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      assertPlainJson(v, `${path}.${k}`);
+    }
+    return;
+  }
+  // function | symbol | bigint | undefined
+  throw new Error(
+    `Invalid sklearn.estimator envelope: ${path} is not JSON-serializable (type ${t})`
+  );
+}
+
 const decodeSklearnEstimatorEnvelope: EnvelopeHandler = value => {
   const encoding = value.encoding;
   if (encoding !== 'json') {
@@ -607,6 +775,19 @@ const decodeSklearnEstimatorEnvelope: EnvelopeHandler = value => {
       'Invalid sklearn.estimator envelope: expected className/module strings + params object'
     );
   }
+  // params must be a PLAIN JSON object end to end — metadata-only estimators never
+  // carry callables, class instances, or nested non-JSON values. Validate (do not
+  // reconstruct) so a corrupt envelope fails clearly instead of leaking a function
+  // or exotic object to a downstream consumer.
+  if (
+    Object.getPrototypeOf(params) !== Object.prototype &&
+    Object.getPrototypeOf(params) !== null
+  ) {
+    throw new Error('Invalid sklearn.estimator envelope: params must be a plain JSON object');
+  }
+  for (const [k, v] of Object.entries(params)) {
+    assertPlainJson(v, `params.${k}`);
+  }
   const versionValue = value.version;
   if (versionValue !== undefined && typeof versionValue !== 'string') {
     throw new Error('Invalid sklearn.estimator envelope: version must be a string when provided');

package/src/version.ts

@@ -9,4 +9,4 @@
  * Regenerate with: node scripts/generate-version.mjs
  */
 
-export const VERSION: string = "0.7.0";
+export const VERSION: string = "0.8.0";