tywrap 0.8.0 → 0.10.0

package/src/utils/codec.ts

@@ -9,6 +9,47 @@
  * Sklearn estimators: { "__tywrap__": "sklearn.estimator", "encoding": "json", ... }
  */
 
+import { tagDecodedShape } from '../runtime/validators.js';
+
+const SCIENTIFIC_MARKERS = [
+  'dataframe',
+  'series',
+  'ndarray',
+  'scipy.sparse',
+  'torch.tensor',
+  'sklearn.estimator',
+] as const;
+
+export type ScientificMarker = (typeof SCIENTIFIC_MARKERS)[number];
+
+export function isScientificMarker(value: unknown): value is ScientificMarker {
+  return SCIENTIFIC_MARKERS.includes(value as ScientificMarker);
+}
+
+type ScientificDecodeErrorKind = 'arrow' | 'envelope';
+type ScientificDecodeErrorMarker = ScientificMarker | 'unknown';
+
+/** @internal Structured failure consumed by BridgeCodec without parsing messages. */
+export class ScientificDecodeError extends Error {
+  constructor(
+    readonly kind: ScientificDecodeErrorKind,
+    readonly marker: ScientificDecodeErrorMarker,
+    override readonly cause: unknown,
+    message = cause instanceof Error ? cause.message : String(cause)
+  ) {
+    super(message, { cause });
+  }
+}
+
+function asScientificDecodeError(
+  error: unknown,
+  marker: ScientificDecodeErrorMarker
+): ScientificDecodeError {
+  return error instanceof ScientificDecodeError
+    ? error
+    : new ScientificDecodeError('envelope', marker, error);
+}
+
 // Avoid hard dependency on apache-arrow types at compile time to keep install optional.
 export type ArrowTable = { readonly numCols?: number; readonly numRows?: number } & Record<
   string,
@@ -31,6 +72,8 @@ export interface TorchTensor {
   shape?: readonly number[];
   dtype?: string;
   device?: string;
+  sourceDtype?: string;
+  sourceDevice?: string;
 }
 
 export interface SklearnEstimator {
@@ -68,6 +111,7 @@ export type ValueEnvelope =
       readonly b64?: string; // when encoding=arrow
       readonly data?: unknown; // when encoding=json
       readonly shape?: readonly number[];
+      readonly dtype?: string | null;
     }
   | {
       readonly __tywrap__: 'scipy.sparse';
@@ -90,6 +134,8 @@ export type ValueEnvelope =
       readonly shape?: readonly number[];
       readonly dtype?: string;
       readonly device?: string;
+      readonly sourceDtype?: string;
+      readonly sourceDevice?: string;
     }
   | {
       readonly __tywrap__: 'sklearn.estimator';
@@ -229,7 +275,15 @@ function isNumberArray(value: unknown): value is number[] {
   return Array.isArray(value) && value.every(item => typeof item === 'number');
 }
 
-function fromBase64(b64: string): Uint8Array {
+const STRICT_BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+
+function fromBase64(b64: string, marker?: string, context?: string): Uint8Array {
+  if (marker && !STRICT_BASE64_PATTERN.test(b64)) {
+    throw new Error(
+      `Invalid ${marker} envelope: b64 at path b64 must be well-formed base64; ` +
+        `${context ? `${context}; ` : ''}actual count unknown, actual type string with length ${b64.length}`
+    );
+  }
   if (typeof Buffer !== 'undefined') {
     const buf = Buffer.from(b64, 'base64');
     return new Uint8Array(buf.buffer, buf.byteOffset, buf.length);
@@ -250,9 +304,11 @@ const ARROW_MISSING_MESSAGE =
   'TYWRAP_CODEC_FALLBACK=json on the Python side to receive JSON instead ' +
   '(lossy for dtype/NA fidelity). tywrap never silently downgrades Arrow payloads.';
 
-function requireArrowDecoder(): (bytes: Uint8Array) => ArrowTable | Uint8Array {
+function requireArrowDecoder(
+  marker: ScientificMarker
+): (bytes: Uint8Array) => ArrowTable | Uint8Array {
   if (!arrowTableFrom) {
-    throw new Error(ARROW_MISSING_MESSAGE);
+    throw new ScientificDecodeError('arrow', marker, new Error(ARROW_MISSING_MESSAGE));
   }
   return arrowTableFrom;
 }
@@ -265,7 +321,9 @@ function requireArrowDecoder(): (bytes: Uint8Array) => ArrowTable | Uint8Array {
  * for the rest of the process. If apache-arrow is absent we throw a clear, actionable
  * error rather than silently producing wrong data.
  */
-async function ensureArrowDecoder(): Promise<(bytes: Uint8Array) => ArrowTable | Uint8Array> {
+async function ensureArrowDecoder(
+  marker: ScientificMarker
+): Promise<(bytes: Uint8Array) => ArrowTable | Uint8Array> {
   if (arrowTableFrom) {
     return arrowTableFrom;
   }
@@ -275,21 +333,30 @@ async function ensureArrowDecoder(): Promise<(bytes: Uint8Array) => ArrowTable |
   );
   await lazyRegistration;
   if (!arrowTableFrom) {
-    throw new Error(ARROW_MISSING_MESSAGE);
+    throw new ScientificDecodeError('arrow', marker, new Error(ARROW_MISSING_MESSAGE));
   }
   return arrowTableFrom;
 }
 
-async function tryDecodeArrowTable(bytes: Uint8Array): Promise<ArrowTable | Uint8Array> {
-  const decoder = await ensureArrowDecoder();
+async function tryDecodeArrowTable(
+  bytes: Uint8Array,
+  marker: ScientificMarker
+): Promise<ArrowTable | Uint8Array> {
+  const decoder = await ensureArrowDecoder(marker);
   try {
     return decoder(bytes);
   } catch (err) {
-    throw new Error(`Arrow decode failed: ${err instanceof Error ? err.message : String(err)}`);
+    throw new ScientificDecodeError(
+      'arrow',
+      marker,
+      err,
+      `Arrow decode failed: ${err instanceof Error ? err.message : String(err)}`
+    );
   }
 }
 
 type MaybePromise<T> = T | Promise<T>;
+type DecodeArrow<T> = (bytes: Uint8Array, marker: ScientificMarker) => MaybePromise<T>;
 
 function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
   return (
@@ -309,7 +376,7 @@ function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
  * @param arr - Typed array or plain array
  * @returns Plain JavaScript array with values converted (BigInt → Number where safe)
  */
-function typedArrayToPlain(arr: unknown): unknown[] {
+function typedArrayToPlain(arr: unknown): unknown[] | null {
   if (Array.isArray(arr)) {
     return arr;
   }
@@ -330,8 +397,7 @@ function typedArrayToPlain(arr: unknown): unknown[] {
   if (arr !== null && arr !== undefined && typeof arr === 'object' && Symbol.iterator in arr) {
     return Array.from(arr as Iterable<unknown>);
   }
-  // Non-iterable: return empty array (shouldn't happen with valid Arrow data)
-  return [];
+  return null;
 }
 
 /**
@@ -355,6 +421,30 @@ function extractArrowValues(data: unknown): unknown[] | null {
   return null;
 }
 
+function extractNdarrayArrowValues(
+  data: unknown,
+  shape: readonly number[] | undefined,
+  dtype: string | undefined
+): unknown[] {
+  try {
+    const values = extractArrowValues(data);
+    if (values) {
+      return values;
+    }
+  } catch (error) {
+    throw new Error(
+      `Invalid ndarray envelope: b64 decoded data extraction failed at path b64 for declared ` +
+        `shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; actual count unknown, ` +
+        `actual type ${actualType(data)} (${error instanceof Error ? error.message : String(error)})`
+    );
+  }
+  throw new Error(
+    `Invalid ndarray envelope: b64 decoded data at path b64 could not extract Arrow values ` +
+      `for declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; ` +
+      `actual count unknown, actual type ${actualType(data)}`
+  );
+}
+
 /**
  * Reshape a flat array into a multi-dimensional nested array.
  *
@@ -394,6 +484,168 @@ function reshapeArray(flat: unknown[], shape: readonly number[]): unknown {
 // Why: decoding needs to reject incompatible envelopes before we attempt to interpret payloads.
 const CODEC_VERSION = 1;
 
+function isStrictV1Envelope(envelope: { codecVersion?: unknown }): boolean {
+  return envelope.codecVersion === CODEC_VERSION;
+}
+
+function actualType(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `array(length=${value.length})`;
+  }
+  if (value === null) {
+    return 'null';
+  }
+  if (typeof value === 'number' && !Number.isFinite(value)) {
+    return `number(${String(value)})`;
+  }
+  return typeof value;
+}
+
+function assertShape(
+  shapeValue: unknown,
+  marker: 'ndarray' | 'torch.tensor' | 'scipy.sparse',
+  field = 'shape',
+  dtype?: unknown
+): number[] {
+  if (!Array.isArray(shapeValue)) {
+    throw new Error(
+      `Invalid ${marker} envelope: ${field} at path ${field} must be a list of ` +
+        `non-negative safe integers; declared shape ${String(shapeValue)} and dtype ` +
+        `${JSON.stringify(dtype)}, actual count unknown, actual type ${actualType(shapeValue)}`
+    );
+  }
+  for (let i = 0; i < shapeValue.length; i += 1) {
+    const dim = shapeValue[i];
+    if (typeof dim !== 'number' || !Number.isSafeInteger(dim) || dim < 0) {
+      throw new Error(
+        `Invalid ${marker} envelope: ${field}[${i}]=${String(dim)} must be a non-negative integer ` +
+          `within the safe range for declared shape ${JSON.stringify(shapeValue)}; ` +
+          `declared dtype ${JSON.stringify(dtype)}, actual count unknown, ` +
+          `actual type ${actualType(dim)} with value ${String(dim)}`
+      );
+    }
+  }
+  return shapeValue as number[];
+}
+
+function assertOptionalNonEmptyDtype(
+  value: unknown,
+  marker: 'ndarray' | 'torch.tensor' | 'scipy.sparse',
+  field = 'dtype',
+  shape?: unknown
+): string | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new Error(
+      `Invalid ${marker} envelope: ${field} at path ${field} must be a non-empty string; ` +
+        `declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(value)}, ` +
+        `actual count unknown, actual type ${actualType(value)}`
+    );
+  }
+  return value;
+}
+
+type ShapedScientificMarker = 'ndarray' | 'torch.tensor' | 'scipy.sparse';
+
+function readEnvelopeShape(
+  envelope: { codecVersion?: unknown },
+  shapeValue: unknown,
+  marker: ShapedScientificMarker,
+  field = 'shape',
+  dtype?: unknown
+): number[] | undefined {
+  return isStrictV1Envelope(envelope)
+    ? assertShape(shapeValue, marker, field, dtype)
+    : isNumberArray(shapeValue)
+      ? shapeValue
+      : undefined;
+}
+
+function readEnvelopeDtype(
+  envelope: { codecVersion?: unknown },
+  dtypeValue: unknown,
+  marker: ShapedScientificMarker,
+  field = 'dtype',
+  shape?: unknown
+): string | undefined {
+  return isStrictV1Envelope(envelope)
+    ? assertOptionalNonEmptyDtype(dtypeValue, marker, field, shape)
+    : typeof dtypeValue === 'string'
+      ? dtypeValue
+      : undefined;
+}
+
+function countJsonLeaves(value: unknown): number {
+  if (!Array.isArray(value)) {
+    return 1;
+  }
+  return value.reduce((count, item) => count + countJsonLeaves(item), 0);
+}
+
+function assertJsonMatchesShape(
+  data: unknown,
+  shape: readonly number[],
+  dtype: string | undefined
+): void {
+  const declaredCount = shapeProduct(shape);
+  const strictLeaves = dtype !== undefined;
+  const actualCount = strictLeaves ? countJsonLeaves(data) : undefined;
+  const visit = (value: unknown, depth: number, path: string): void => {
+    if (depth === shape.length) {
+      if (strictLeaves && Array.isArray(value)) {
+        throw new Error(
+          `Invalid ndarray envelope: data at path ${path} exceeds nesting depth ` +
+            `${shape.length} for declared shape ${JSON.stringify(shape)} and dtype ` +
+            `${JSON.stringify(dtype)}; actual count ${actualCount}, actual type ${actualType(value)}`
+        );
+      }
+      return;
+    }
+    if (!Array.isArray(value)) {
+      throw new Error(
+        `Invalid ndarray envelope: data at path ${path} must be an array at depth ${depth} ` +
+          `for declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; ` +
+          `actual count ${actualCount ?? 'unknown'}, actual type ${actualType(value)}`
+      );
+    }
+    const expectedLength = shape[depth] as number;
+    if (value.length !== expectedLength) {
+      throw new Error(
+        `Invalid ndarray envelope: data at path ${path} has length ${value.length}, expected ` +
+          `${expectedLength} for declared shape ${JSON.stringify(shape)} and dtype ` +
+          `${JSON.stringify(dtype)}; declared count ${declaredCount}, actual count ${actualCount ?? 'unknown'}, ` +
+          `actual type ${actualType(value)}`
+      );
+    }
+    value.forEach((item, index) => visit(item, depth + 1, `${path}[${index}]`));
+  };
+  visit(data, 0, 'data');
+  if (strictLeaves && actualCount !== declaredCount) {
+    throw new Error(
+      `Invalid ndarray envelope: data at path data has leaf count ${actualCount}, expected ` +
+        `${declaredCount} for declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; ` +
+        `actual type ${actualType(data)}`
+    );
+  }
+}
+
+function assertArrowCount(
+  values: readonly unknown[],
+  shape: readonly number[],
+  dtype: string
+): void {
+  const expected = shapeProduct(shape);
+  if (values.length !== expected) {
+    throw new Error(
+      `Invalid ndarray envelope: b64 decoded data at path b64 has element count ${values.length}, ` +
+        `expected ${expected} for declared shape ${JSON.stringify(shape)} and dtype ` +
+        `${JSON.stringify(dtype)}; actual type array(length=${values.length})`
+    );
+  }
+}
+
 function assertCodecVersion(envelope: { codecVersion?: unknown }, typeTag: string): void {
   if (!('codecVersion' in envelope)) {
     return;
@@ -420,7 +672,7 @@ function assertCodecVersion(envelope: { codecVersion?: unknown }, typeTag: strin
  */
 type EnvelopeHandler = <T>(
   value: { [k: string]: unknown },
-  decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
+  decodeArrow: DecodeArrow<T>,
   recurse: (value: unknown) => MaybePromise<T | unknown>
 ) => MaybePromise<T | unknown>;
 
@@ -432,8 +684,8 @@ type EnvelopeHandler = <T>(
  */
 function decodeArrowOrJsonEnvelope<T>(
   value: { [k: string]: unknown },
-  decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
-  typeTag: string
+  decodeArrow: DecodeArrow<T>,
+  typeTag: 'dataframe' | 'series'
 ): MaybePromise<T | unknown> {
   const encoding = value.encoding;
   if (encoding === 'arrow') {
@@ -441,14 +693,17 @@ function decodeArrowOrJsonEnvelope<T>(
     if (typeof b64 !== 'string') {
       throw new Error(`Invalid ${typeTag} envelope: missing b64`);
     }
-    const bytes = fromBase64(b64);
-    return decodeArrow(bytes);
+    const bytes = fromBase64(b64, isStrictV1Envelope(value) ? typeTag : undefined);
+    const decoded = decodeArrow(bytes, typeTag);
+    return isPromiseLike(decoded)
+      ? decoded.then(item => tagDecodedShape(item, { marker: typeTag }))
+      : tagDecodedShape(decoded, { marker: typeTag });
   }
   if (encoding === 'json') {
     if (!('data' in value)) {
       throw new Error(`Invalid ${typeTag} envelope: missing data`);
     }
-    return value.data;
+    return tagDecodedShape(value.data, { marker: typeTag });
   }
   throw new Error(`Invalid ${typeTag} envelope: unsupported encoding ${String(encoding)}`);
 }
@@ -459,51 +714,83 @@ const decodeDataframeEnvelope: EnvelopeHandler = (value, decodeArrow) =>
 const decodeSeriesEnvelope: EnvelopeHandler = (value, decodeArrow) =>
   decodeArrowOrJsonEnvelope(value, decodeArrow, 'series');
 
+function finishNdarrayDecode(
+  data: unknown,
+  strictV1: boolean,
+  shape: readonly number[] | undefined,
+  dtype: string | undefined,
+  metadata: { marker: 'ndarray'; dims: number | undefined; dtype: string | undefined }
+): unknown {
+  const values = strictV1
+    ? extractNdarrayArrowValues(data, shape, dtype)
+    : extractArrowValues(data);
+  if (!values) {
+    return tagDecodedShape(data, metadata);
+  }
+  if (strictV1) {
+    assertArrowCount(values, shape as number[], dtype as string);
+  }
+  return tagDecodedShape(
+    shape && shape.length !== 1 ? reshapeArray(values, shape) : values,
+    metadata
+  );
+}
+
 const decodeNdarrayEnvelope: EnvelopeHandler = (value, decodeArrow) => {
   const encoding = value.encoding;
-  const shapeValue = value.shape;
-  const shape = isNumberArray(shapeValue) ? shapeValue : undefined;
+  const strictV1 = isStrictV1Envelope(value);
+  const shape = readEnvelopeShape(value, value.shape, 'ndarray', 'shape', value.dtype);
+  const dtype = readEnvelopeDtype(value, value.dtype, 'ndarray', 'dtype', shape);
+  const metadata = { marker: 'ndarray' as const, dims: shape?.length, dtype };
 
   if (encoding === 'arrow') {
+    if (strictV1 && dtype === undefined) {
+      throw new Error(
+        `Invalid ndarray envelope: dtype at path dtype is required for Arrow encoding; ` +
+          `declared shape ${JSON.stringify(shape)} and dtype undefined, actual count unknown, actual type undefined`
+      );
+    }
     const b64 = value.b64;
     if (typeof b64 !== 'string') {
-      throw new Error('Invalid ndarray envelope: missing b64');
+      if (!strictV1) {
+        throw new Error('Invalid ndarray envelope: missing b64');
+      }
+      throw new Error(
+        `Invalid ndarray envelope: b64 at path b64 must be a base64 string for declared ` +
+          `shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; actual count unknown, ` +
+          `actual type ${actualType(b64)}`
+      );
     }
-    const bytes = fromBase64(b64);
-    const decoded = decodeArrow(bytes);
+    const bytes = fromBase64(
+      b64,
+      strictV1 ? 'ndarray' : undefined,
+      `declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}`
+    );
+    const decoded = decodeArrow(bytes, 'ndarray');
 
     // Extract values from Arrow table and reshape if needed
     // Arrow only handles 1D arrays, so we flatten on encode and reshape here
     // Reshape for: scalars (shape.length === 0) and multi-dim (shape.length > 1)
     // Skip reshape for: 1D arrays (shape.length === 1) - return as-is
     if (isPromiseLike(decoded)) {
-      return decoded.then(data => {
-        const values = extractArrowValues(data);
-        if (!values) {
-          return data; // Fallback: return raw data if extraction fails
-        }
-        // Reshape scalars and multi-dimensional arrays, but not 1D
-        if (shape && shape.length !== 1) {
-          return reshapeArray(values, shape);
-        }
-        return values;
-      });
-    }
-    const values = extractArrowValues(decoded);
-    if (!values) {
-      return decoded; // Fallback: return raw data if extraction fails
+      return decoded.then(data => finishNdarrayDecode(data, strictV1, shape, dtype, metadata));
     }
-    // Reshape scalars and multi-dimensional arrays, but not 1D
-    if (shape && shape.length !== 1) {
-      return reshapeArray(values, shape);
-    }
-    return values;
+    return finishNdarrayDecode(decoded, strictV1, shape, dtype, metadata);
   }
   if (encoding === 'json') {
     if (!('data' in value)) {
-      throw new Error('Invalid ndarray envelope: missing data');
+      if (!strictV1) {
+        throw new Error('Invalid ndarray envelope: missing data');
+      }
+      throw new Error(
+        `Invalid ndarray envelope: data at path data is required for declared shape ` +
+          `${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; actual count 0, actual type undefined`
+      );
+    }
+    if (strictV1) {
+      assertJsonMatchesShape(value.data, shape as number[], dtype);
     }
-    return value.data;
+    return tagDecodedShape(value.data, metadata);
   }
   throw new Error(`Invalid ndarray envelope: unsupported encoding ${String(encoding)}`);
 };
@@ -533,6 +820,53 @@ function assertIndexArrayInRange(arr: readonly unknown[], bound: number, label:
   }
 }
 
+function assertSparseDataDomain(
+  data: readonly unknown[],
+  dtype: string,
+  shape: readonly number[]
+): void {
+  const expectsBoolean = dtype === 'bool' || dtype === 'bool_';
+  const integerMatch = /^(u?int)(\d*)$/.exec(dtype);
+  const expectsInteger = integerMatch !== null;
+  const expectsFloat = /^float\d*$/.test(dtype);
+  const width = integerMatch?.[2] ? Number(integerMatch[2]) : undefined;
+  const unsignedInteger = integerMatch?.[1] === 'uint';
+  let integerMin: number | undefined;
+  let integerMax: number | undefined;
+  if (width === 8 || width === 16 || width === 32) {
+    integerMin = unsignedInteger ? 0 : -(2 ** (width - 1));
+    integerMax = unsignedInteger ? 2 ** width - 1 : 2 ** (width - 1) - 1;
+  }
+
+  for (let i = 0; i < data.length; i += 1) {
+    const item = data[i];
+    if (typeof item === 'number' && !Number.isFinite(item)) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: data[${i}] at path data[${i}] must be finite for ` +
+          `declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; ` +
+          `actual count ${data.length}, actual type number with value ${String(item)}`
+      );
+    }
+    const valid = expectsBoolean
+      ? typeof item === 'boolean'
+      : expectsInteger
+        ? typeof item === 'number' &&
+          Number.isInteger(item) &&
+          (integerMin === undefined || item >= integerMin) &&
+          (integerMax === undefined || item <= integerMax)
+        : expectsFloat
+          ? typeof item === 'number'
+          : true;
+    if (!valid) {
+      throw new Error(
+        `Invalid scipy.sparse envelope: data[${i}] at path data[${i}] is incompatible with ` +
+          `declared shape ${JSON.stringify(shape)} and dtype ${JSON.stringify(dtype)}; ` +
+          `actual count ${data.length}, actual type ${actualType(item)} with value ${String(item)}`
+      );
+    }
+  }
+}
+
 const decodeScipySparseEnvelope: EnvelopeHandler = value => {
   const encoding = value.encoding;
   if (encoding !== 'json') {
@@ -542,7 +876,8 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
   if (format !== 'csr' && format !== 'csc' && format !== 'coo') {
     throw new Error(`Invalid scipy.sparse envelope: unsupported format ${String(format)}`);
   }
-  const shape = value.shape;
+  const strictV1 = isStrictV1Envelope(value);
+  const shape = readEnvelopeShape(value, value.shape, 'scipy.sparse', 'shape', value.dtype);
   if (
     !Array.isArray(shape) ||
     shape.length !== 2 ||
@@ -561,8 +896,16 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
   if (!Array.isArray(data)) {
     throw new Error('Invalid scipy.sparse envelope: data must be an array');
   }
-  const dtypeValue = value.dtype;
-  const dtype = typeof dtypeValue === 'string' ? dtypeValue : undefined;
+  const dtype = readEnvelopeDtype(value, value.dtype, 'scipy.sparse', 'dtype', shape);
+  if (strictV1 && dtype === undefined) {
+    throw new Error(
+      `Invalid scipy.sparse envelope: dtype at path dtype is required for declared shape ` +
+        `${JSON.stringify(shape)}; actual count ${data.length}, actual type undefined`
+    );
+  }
+  if (strictV1) {
+    assertSparseDataDomain(data, dtype as string, shape);
+  }
 
   if (format === 'coo') {
     const row = value.row;
@@ -580,14 +923,17 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
     }
     assertIndexArrayInRange(row, rows, 'row');
     assertIndexArrayInRange(col, cols, 'col');
-    return {
-      format,
-      shape,
-      data,
-      row,
-      col,
-      dtype,
-    } satisfies SparseMatrix;
+    return tagDecodedShape(
+      {
+        format,
+        shape,
+        data,
+        row,
+        col,
+        dtype,
+      } satisfies SparseMatrix,
+      { marker: 'scipy.sparse', dims: shape.length, dtype }
+    );
   }
 
   const indices = value.indices;
@@ -641,14 +987,17 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
     );
   }
   assertIndexArrayInRange(indices, minorAxis, 'indices');
-  return {
-    format,
-    shape,
-    data,
-    indices,
-    indptr,
-    dtype,
-  } satisfies SparseMatrix;
+  return tagDecodedShape(
+    {
+      format,
+      shape,
+      data,
+      indices,
+      indptr,
+      dtype,
+    } satisfies SparseMatrix,
+    { marker: 'scipy.sparse', dims: shape.length, dtype }
+  );
 };
 
 /** Product of a shape's dimensions (the element count). [] (scalar) -> 1. */
@@ -658,7 +1007,7 @@ function shapeProduct(shape: readonly number[]): number {
 
 const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
   value: { [k: string]: unknown },
-  _decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
+  _decodeArrow: DecodeArrow<T>,
   recurse: (value: unknown) => MaybePromise<T | unknown>
 ): MaybePromise<T | unknown> => {
   const encoding = value.encoding;
@@ -672,34 +1021,49 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
   if (!isObject(nested) || (nested as { __tywrap__?: unknown }).__tywrap__ !== 'ndarray') {
     throw new Error('Invalid torch.tensor envelope: value must be an ndarray envelope');
   }
-  const shapeValue = value.shape;
-  const shape = isNumberArray(shapeValue) ? shapeValue : undefined;
-  // The tensor shape must be a non-negative-integer dimension list. A negative or
-  // non-integer dim is a corrupt envelope, not a valid tensor.
-  if (shape) {
-    for (let i = 0; i < shape.length; i += 1) {
-      const dim = shape[i] as number;
-      if (!Number.isInteger(dim) || dim < 0) {
-        throw new Error(
-          `Invalid torch.tensor envelope: shape[${i}]=${dim} must be a non-negative integer`
-        );
-      }
-    }
-  }
-  // Cross-check the tensor shape's element count against the nested ndarray's
-  // declared shape (metadata only — no decode needed). A mismatch means the two
-  // shapes disagree about how many elements the payload holds.
+  const strictV1 = isStrictV1Envelope(value);
+  const shape = readEnvelopeShape(value, value.shape, 'torch.tensor', 'shape', value.dtype);
   const nestedShapeValue = (nested as { shape?: unknown }).shape;
-  const nestedShape = isNumberArray(nestedShapeValue) ? nestedShapeValue : undefined;
-  if (shape && nestedShape && shapeProduct(shape) !== shapeProduct(nestedShape)) {
+  const nestedShape = readEnvelopeShape(
+    value,
+    nestedShapeValue,
+    'torch.tensor',
+    'value.shape',
+    nested.dtype
+  );
+  const scalarNormalization =
+    shape !== undefined &&
+    nestedShape !== undefined &&
+    ((shape.length === 0 && nestedShape.length === 1 && nestedShape[0] === 1) ||
+      (nestedShape.length === 0 && shape.length === 1 && shape[0] === 1));
+  const shapesEqual =
+    shape !== undefined &&
+    nestedShape !== undefined &&
+    shape.length === nestedShape.length &&
+    shape.every((dim, index) => dim === nestedShape[index]);
+  const shapesDisagree = strictV1
+    ? !shapesEqual && !scalarNormalization
+    : shape !== undefined &&
+      nestedShape !== undefined &&
+      shapeProduct(shape) !== shapeProduct(nestedShape);
+  if (shape && nestedShape && shapesDisagree) {
+    throw new Error(
+      `Invalid torch.tensor envelope: shape at path shape ${JSON.stringify(shape)} disagrees ` +
+        `with nested ndarray shape at path value.shape ${JSON.stringify(nestedShape)}; exact shapes are required ` +
+        `(except scalar []/[1]), declared dtypes ${JSON.stringify(value.dtype)}/${JSON.stringify(nested.dtype)}, ` +
+        `actual counts ${shapeProduct(shape)} and ${shapeProduct(nestedShape)}`
+    );
+  }
+  const dtype = readEnvelopeDtype(value, value.dtype, 'torch.tensor', 'dtype', shape);
+  if (strictV1 && dtype === undefined) {
     throw new Error(
-      `Invalid torch.tensor envelope: shape ${JSON.stringify(shape)} ` +
-        `(product ${shapeProduct(shape)}) disagrees with nested ndarray shape ` +
-        `${JSON.stringify(nestedShape)} (product ${shapeProduct(nestedShape)})`
+      `Invalid torch.tensor envelope: dtype at path dtype is required for declared shape ` +
+        `${JSON.stringify(shape)}; actual count ${shapeProduct(shape as number[])}, actual type undefined`
     );
   }
-  const dtypeValue = value.dtype;
-  const dtype = typeof dtypeValue === 'string' ? dtypeValue : undefined;
+  if (strictV1 && 'dtype' in nested) {
+    assertOptionalNonEmptyDtype(nested.dtype, 'torch.tensor', 'value.dtype', nestedShape);
+  }
   const deviceValue = value.device;
   if (deviceValue !== undefined && (typeof deviceValue !== 'string' || deviceValue.length === 0)) {
     throw new Error(
@@ -707,12 +1071,45 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
     );
   }
   const device = typeof deviceValue === 'string' ? deviceValue : undefined;
+  const sourceDtypeValue = value.sourceDtype;
+  if (
+    sourceDtypeValue !== undefined &&
+    (typeof sourceDtypeValue !== 'string' || sourceDtypeValue.length === 0)
+  ) {
+    throw new Error(
+      'Invalid torch.tensor envelope: sourceDtype must be a non-empty string when provided'
+    );
+  }
+  const sourceDtype = typeof sourceDtypeValue === 'string' ? sourceDtypeValue : undefined;
+  const sourceDeviceValue = value.sourceDevice;
+  if (
+    sourceDeviceValue !== undefined &&
+    (typeof sourceDeviceValue !== 'string' || sourceDeviceValue.length === 0)
+  ) {
+    throw new Error(
+      'Invalid torch.tensor envelope: sourceDevice must be a non-empty string when provided'
+    );
+  }
+  const sourceDevice = typeof sourceDeviceValue === 'string' ? sourceDeviceValue : undefined;
+
+  const finishTensorDecode = (data: unknown): TorchTensor =>
+    tagDecodedShape(
+      {
+        data,
+        shape,
+        dtype,
+        device,
+        ...(sourceDtype === undefined ? {} : { sourceDtype }),
+        ...(sourceDevice === undefined ? {} : { sourceDevice }),
+      },
+      { marker: 'torch.tensor', dims: shape?.length, dtype }
+    );
 
   const decoded = recurse(nested);
   if (isPromiseLike(decoded)) {
-    return decoded.then(data => ({ data, shape, dtype, device })) as Promise<T | unknown>;
+    return decoded.then(finishTensorDecode) as Promise<T | unknown>;
   }
-  return { data: decoded, shape, dtype, device } satisfies TorchTensor;
+  return finishTensorDecode(decoded);
 };
 
 /**
@@ -770,9 +1167,17 @@ const decodeSklearnEstimatorEnvelope: EnvelopeHandler = value => {
   const className = value.className;
   const module = value.module;
   const params = value.params;
-  if (typeof className !== 'string' || typeof module !== 'string' || !isObject(params)) {
+  const strictV1 = isStrictV1Envelope(value);
+  if (
+    typeof className !== 'string' ||
+    typeof module !== 'string' ||
+    (strictV1 && (className.length === 0 || module.length === 0)) ||
+    !isObject(params)
+  ) {
     throw new Error(
-      'Invalid sklearn.estimator envelope: expected className/module strings + params object'
+      `Invalid sklearn.estimator envelope: expected className/module strings + params object; ` +
+        `className at path className and module at path module must be non-empty; declared class ${JSON.stringify(className)}, ` +
+        `declared module ${JSON.stringify(module)}, actual types ${actualType(className)}/${actualType(module)}`
     );
   }
   // params must be a PLAIN JSON object end to end — metadata-only estimators never
@@ -793,19 +1198,22 @@ const decodeSklearnEstimatorEnvelope: EnvelopeHandler = value => {
     throw new Error('Invalid sklearn.estimator envelope: version must be a string when provided');
   }
   const version = typeof versionValue === 'string' ? versionValue : undefined;
-  return {
-    className,
-    module,
-    version,
-    params,
-  } satisfies SklearnEstimator;
+  return tagDecodedShape(
+    {
+      className,
+      module,
+      version,
+      params,
+    } satisfies SklearnEstimator,
+    { marker: 'sklearn.estimator' }
+  );
 };
 
 // Why: dispatch over the __tywrap__ typeTag instead of a long if-chain so each type's
 // decode logic lives in one focused handler. The typeTag strings are the on-the-wire keys
 // emitted by the Python bridge and MUST stay byte-identical. A Map keyed by the typeTag
 // avoids prototype-chain lookups when dispatching on attacker-controlled typeTag strings.
-const ENVELOPE_HANDLERS: ReadonlyMap<string, EnvelopeHandler> = new Map([
+const ENVELOPE_HANDLERS: ReadonlyMap<ScientificMarker, EnvelopeHandler> = new Map([
   ['dataframe', decodeDataframeEnvelope],
   ['series', decodeSeriesEnvelope],
   ['ndarray', decodeNdarrayEnvelope],
@@ -814,16 +1222,38 @@ const ENVELOPE_HANDLERS: ReadonlyMap<string, EnvelopeHandler> = new Map([
   ['sklearn.estimator', decodeSklearnEstimatorEnvelope],
 ]);
 
+const MAX_DECODE_DEPTH = 2048;
+const MAX_DECODE_NODES = 1_000_000;
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  if (!isObject(value)) {
+    return false;
+  }
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+
+function isPlainArray(value: unknown): value is unknown[] {
+  return Array.isArray(value) && Object.getPrototypeOf(value) === Array.prototype;
+}
+
+function decodePath(base: string, key: string | number): string {
+  if (typeof key === 'number') {
+    return `${base}[${key}]`;
+  }
+  return /^[A-Za-z_$][\w$]*$/.test(key) ? `${base}.${key}` : `${base}[${JSON.stringify(key)}]`;
+}
+
 function decodeEnvelopeCore<T>(
   value: unknown,
-  decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
+  decodeArrow: DecodeArrow<T>,
   recurse: (value: unknown) => MaybePromise<T | unknown>
 ): MaybePromise<T | unknown> {
   if (!isObject(value)) {
     return value;
   }
   const typeTag = (value as { __tywrap__?: unknown }).__tywrap__;
-  if (typeof typeTag !== 'string') {
+  if (!isScientificMarker(typeTag)) {
     return value as unknown;
   }
 
@@ -832,11 +1262,23 @@ function decodeEnvelopeCore<T>(
     return value as unknown;
   }
 
-  assertCodecVersion(value as { codecVersion?: unknown }, typeTag);
-  return handler(value, decodeArrow, recurse);
+  try {
+    assertCodecVersion(value as { codecVersion?: unknown }, typeTag);
+    const decoded = handler(value, decodeArrow, recurse);
+    return isPromiseLike(decoded)
+      ? Promise.resolve(decoded).catch(error => {
+          throw asScientificDecodeError(error, typeTag);
+        })
+      : decoded;
+  } catch (error) {
+    throw asScientificDecodeError(error, typeTag);
+  }
 }
 
-function decodeEnvelope<T>(value: unknown, decodeArrow: (bytes: Uint8Array) => T): T | unknown {
+function decodeEnvelope<T>(
+  value: unknown,
+  decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => T
+): T | unknown {
   const recurse: (value: unknown) => MaybePromise<T | unknown> = v =>
     decodeEnvelopeCore(v, decodeArrow, recurse);
   const decoded = decodeEnvelopeCore(value, decodeArrow, recurse);
@@ -848,30 +1290,225 @@ function decodeEnvelope<T>(value: unknown, decodeArrow: (bytes: Uint8Array) => T
 
 async function decodeEnvelopeAsync<T>(
   value: unknown,
-  decodeArrow: (bytes: Uint8Array) => Promise<T>
+  decodeArrow: (bytes: Uint8Array, marker: ScientificMarker) => Promise<T>
 ): Promise<T | unknown> {
-  const recurse: (value: unknown) => MaybePromise<T | unknown> = v =>
-    decodeEnvelopeCore(v, decodeArrow, recurse);
-  return await decodeEnvelopeCore(value, decodeArrow, recurse);
+  let visitedNodes = 0;
+
+  type DecodeTarget =
+    | { container: unknown[]; key: number }
+    | { container: Record<string, unknown>; key: string };
+
+  interface ArrayFrame {
+    kind: 'array';
+    container: unknown[];
+    depth: number;
+    path: string;
+    nextIndex: number;
+  }
+
+  interface ObjectFrame {
+    kind: 'object';
+    container: Record<string, unknown>;
+    depth: number;
+    path: string;
+    entries: [string, unknown][];
+    nextIndex: number;
+  }
+
+  type DecodeFrame = ArrayFrame | ObjectFrame;
+
+  const recordVisit = (depth: number, path: string): void => {
+    visitedNodes += 1;
+    if (visitedNodes > MAX_DECODE_NODES) {
+      throw new Error(
+        `Scientific envelope decode maximum visited nodes ${MAX_DECODE_NODES} exceeded at ${path}`
+      );
+    }
+    if (depth > MAX_DECODE_DEPTH) {
+      throw new Error(
+        `Scientific envelope decode maximum depth ${MAX_DECODE_DEPTH} exceeded at ${path}`
+      );
+    }
+  };
+
+  const prefixHandlerError = (error: unknown, path: string): never => {
+    if (path === 'result') {
+      throw error;
+    }
+    throw new ScientificDecodeError(
+      'envelope',
+      'unknown',
+      error,
+      `Scientific envelope decode failed at ${path}: ${error instanceof Error ? error.message : String(error)}`
+    );
+  };
+
+  const assignDecoded = (target: DecodeTarget, original: unknown, decoded: unknown): void => {
+    if (decoded !== original) {
+      if (Array.isArray(target.container)) {
+        target.container[target.key as number] = decoded;
+      } else {
+        target.container[target.key as string] = decoded;
+      }
+    }
+  };
+
+  const settleDecoded = async (
+    decoded: PromiseLike<T | unknown>,
+    original: unknown,
+    target: DecodeTarget,
+    path: string
+  ): Promise<void> => {
+    let resolved: T | unknown;
+    try {
+      resolved = await decoded;
+    } catch (error) {
+      return prefixHandlerError(error, path);
+    }
+    assignDecoded(target, original, resolved);
+  };
+
+  const decodeSingle = (item: unknown, depth: number, path: string): MaybePromise<T | unknown> => {
+    // Handlers sometimes decode a required child envelope themselves (currently
+    // torch.tensor.value). Keep that single-envelope recursion unchanged.
+    const recurseSingle: (nested: unknown) => MaybePromise<T | unknown> = nested => {
+      const nestedPath = decodePath(path, 'value');
+      recordVisit(depth + 1, nestedPath);
+      return decodeEnvelopeCore(nested, decodeArrow, recurseSingle);
+    };
+    return decodeEnvelopeCore(item, decodeArrow, recurseSingle);
+  };
+
+  const frames: DecodeFrame[] = [];
+  const pending: Promise<void>[] = [];
+
+  const visit = (current: unknown, depth: number, path: string, target?: DecodeTarget): void => {
+    const plainArray = isPlainArray(current);
+    const plainObject = isPlainObject(current);
+
+    // Only containers consume the traversal budget. Primitive leaves are already
+    // bounded by the payload byte cap, and counting them would reject large plain
+    // arrays that decoded fine before recursion existed.
+    if (plainArray || plainObject) {
+      recordVisit(depth, path);
+    }
+
+    if (plainObject && typeof current.__tywrap__ === 'string') {
+      if (!isScientificMarker(current.__tywrap__)) {
+        return;
+      }
+
+      let decoded: MaybePromise<T | unknown>;
+      try {
+        decoded = decodeSingle(current, depth, path);
+      } catch (error) {
+        return prefixHandlerError(error, path);
+      }
+      if (isPromiseLike(decoded)) {
+        if (target) {
+          pending.push(settleDecoded(decoded, current, target, path));
+        } else {
+          pending.push(settleDecoded(decoded, current, { container: root, key: 'value' }, path));
+        }
+      } else if (target) {
+        assignDecoded(target, current, decoded);
+      } else if (decoded !== current) {
+        root.value = decoded;
+      }
+      return;
+    }
+
+    if (plainArray) {
+      frames.push({ kind: 'array', container: current, depth, path, nextIndex: 0 });
+    } else if (plainObject) {
+      frames.push({
+        kind: 'object',
+        container: current,
+        depth,
+        path,
+        entries: Object.entries(current),
+        nextIndex: 0,
+      });
+    }
+  };
+
+  /**
+   * Values have copy semantics across the bridge: repeated Python references are
+   * serialized independently, and this decoder does not restore alias identity.
+   */
+  const root: { value: unknown } = { value };
+  visit(value, 0, 'result');
+
+  const visitChild = (
+    item: unknown,
+    key: string | number,
+    target: DecodeTarget,
+    frame: DecodeFrame
+  ): void => {
+    try {
+      visit(item, frame.depth + 1, decodePath(frame.path, key), target);
+    } catch (error) {
+      pending.push(Promise.reject(error));
+    }
+  };
+
+  while (frames.length > 0) {
+    const frame = frames[frames.length - 1];
+    if (!frame) {
+      break;
+    }
+    if (
+      frame.nextIndex >= (frame.kind === 'array' ? frame.container.length : frame.entries.length)
+    ) {
+      frames.pop();
+      continue;
+    }
+
+    const index = frame.nextIndex;
+    frame.nextIndex += 1;
+    if (frame.kind === 'array') {
+      visitChild(frame.container[index], index, { container: frame.container, key: index }, frame);
+    } else {
+      const entry = frame.entries[index];
+      if (entry) {
+        const [key, item] = entry;
+        visitChild(item, key, { container: frame.container, key }, frame);
+      }
+    }
+  }
+
+  if (pending.length > 0) {
+    await Promise.all(pending);
+  }
+  return root.value;
 }
 
 /**
  * Decode values produced by the Python bridge.
  */
 export async function decodeValueAsync(value: unknown): Promise<DecodedValue> {
-  return await decodeEnvelopeAsync(value, tryDecodeArrowTable);
+  try {
+    return await decodeEnvelopeAsync(value, tryDecodeArrowTable);
+  } catch (error) {
+    throw asScientificDecodeError(error, 'unknown');
+  }
 }
 
 /**
  * Synchronous decode. Arrow decoding requires a registered decoder.
  */
 export function decodeValue(value: unknown): DecodedValue {
-  const decodeArrow = (bytes: Uint8Array): DecodedValue => {
-    const decoder = requireArrowDecoder();
+  const decodeArrow = (bytes: Uint8Array, marker: ScientificMarker): DecodedValue => {
+    const decoder = requireArrowDecoder(marker);
     try {
       return decoder(bytes);
     } catch (err) {
-      throw new Error(`Arrow decode failed: ${err instanceof Error ? err.message : String(err)}`);
+      throw new ScientificDecodeError(
+        'arrow',
+        marker,
+        err,
+        `Arrow decode failed: ${err instanceof Error ? err.message : String(err)}`
+      );
     }
   };
   return decodeEnvelope(value, decodeArrow);