tywrap 0.1.0 → 0.1.2

package/src/tywrap.ts

@@ -72,16 +72,50 @@ expectType<Record<string, unknown>>(mod);
   await writeFile(filePath, content, 'utf-8');
 }
 
+export interface GenerateRunOptions {
+  /**
+   * If true, do not write files; instead compare generated output to what's on disk.
+   * Intended for CI to ensure wrappers are checked in and up to date.
+   */
+  check?: boolean;
+}
+
+export interface GenerateResult {
+  written: string[];
+  warnings: string[];
+  /**
+   * Only set when `GenerateRunOptions.check === true`.
+   * Lists files that are missing or differ from what would be generated.
+   */
+  outOfDate?: string[];
+}
+
+function normalizeForComparison(text: string): string {
+  // Avoid false positives from CRLF vs LF (e.g. Windows checkouts)
+  return text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+}
+
+async function safeReadFile(path: string): Promise<string | null> {
+  try {
+    return await fsUtils.readFile(path);
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Generate TypeScript wrappers for configured Python modules.
  * Minimal MVP implementation: resolves module file, analyzes, generates TS, writes to output.dir
  */
 export async function generate(
-  options: Partial<TywrapOptions>
-): Promise<{ written: string[]; warnings: string[] }> {
+  options: Partial<TywrapOptions>,
+  runOptions: GenerateRunOptions = {}
+): Promise<GenerateResult> {
+  const checkMode = runOptions.check === true;
   const resolvedOptions = createConfig(options);
   const instance = await tywrap(resolvedOptions);
   const written: string[] = [];
+  const outOfDate: string[] = [];
   const warnings: string[] = [];
   const outputDir = resolvedOptions.output.dir;
   const caching = resolvedOptions.performance.caching;
@@ -92,11 +126,13 @@ export async function generate(
   });
 
   // Ensure directory exists (Node-only best-effort)
-  try {
-    const modFs = await import('fs/promises');
-    await modFs.mkdir(outputDir, { recursive: true });
-  } catch {
-    // ignore in non-node or if already exists
+  if (!checkMode) {
+    try {
+      const modFs = await import('fs/promises');
+      await modFs.mkdir(outputDir, { recursive: true });
+    } catch {
+      // ignore in non-node or if already exists
+    }
   }
 
   const modules = resolvedOptions.pythonModules ?? {};
@@ -109,10 +145,6 @@ export async function generate(
     let ir: unknown | null = null;
     let irError: string | undefined;
     if (caching && fsUtils.isAvailable()) {
-      try {
-        const modFs = await import('fs/promises');
-        await modFs.mkdir(cacheDir, { recursive: true });
-      } catch {}
       try {
         const cached = await fsUtils.readFile(pathUtils.join(cacheDir, cacheKey));
         ir = JSON.parse(cached);
@@ -124,7 +156,7 @@ export async function generate(
       const fetchResult = await fetchPythonIr(moduleKey, pythonPath);
       ir = fetchResult.ir;
       irError = fetchResult.error;
-      if (ir && caching && fsUtils.isAvailable()) {
+      if (ir && caching && fsUtils.isAvailable() && !checkMode) {
         try {
           await fsUtils.writeFile(pathUtils.join(cacheDir, cacheKey), JSON.stringify(ir));
         } catch {}
@@ -141,26 +173,43 @@ export async function generate(
     const annotatedJSDoc = Boolean(resolvedOptions.output?.annotatedJSDoc);
     const gen = instance.generator.generateModuleDefinition(moduleModel, annotatedJSDoc);
 
-    // Write file
     const baseName = moduleModel.name || 'module';
-    const outPath = pathUtils.join(outputDir, `${baseName}.generated.ts`);
-    await fsUtils.writeFile(outPath, gen.typescript);
-    written.push(outPath);
+    const filesToEmit: Array<{ path: string; content: string }> = [
+      { path: pathUtils.join(outputDir, `${baseName}.generated.ts`), content: gen.typescript },
+    ];
 
     // Optional .d.ts emission (header-only declarations mirroring exports)
     if (resolvedOptions.output?.declaration) {
-      const dtsPath = pathUtils.join(outputDir, `${baseName}.generated.d.ts`);
-      const dts = renderDts(gen.typescript);
-      await fsUtils.writeFile(dtsPath, dts);
-      written.push(dtsPath);
+      filesToEmit.push({
+        path: pathUtils.join(outputDir, `${baseName}.generated.d.ts`),
+        content: renderDts(gen.typescript),
+      });
     }
 
     // Optional source map emission (placeholder mapping for now)
     if (resolvedOptions.output?.sourceMap) {
-      const mapPath = pathUtils.join(outputDir, `${baseName}.generated.ts.map`);
-      const map = renderSourceMapPlaceholder(moduleModel.name);
-      await fsUtils.writeFile(mapPath, map);
-      written.push(mapPath);
+      filesToEmit.push({
+        path: pathUtils.join(outputDir, `${baseName}.generated.ts.map`),
+        content: renderSourceMapPlaceholder(moduleModel.name),
+      });
+    }
+
+    if (checkMode) {
+      for (const file of filesToEmit) {
+        const existing = await safeReadFile(file.path);
+        if (existing === null) {
+          outOfDate.push(file.path);
+          continue;
+        }
+        if (normalizeForComparison(existing) !== normalizeForComparison(file.content)) {
+          outOfDate.push(file.path);
+        }
+      }
+    } else {
+      for (const file of filesToEmit) {
+        await fsUtils.writeFile(file.path, file.content);
+        written.push(file.path);
+      }
     }
 
     // Emit warning summary of unknown typing constructs (best-effort)
@@ -174,22 +223,28 @@ export async function generate(
         `Module ${baseName}: unknown typing constructs encountered: ${unkList}${entries.length > 25 ? '…' : ''}`
       );
       // Write JSON report
-      try {
-        const reportsDir = pathUtils.join('.tywrap', 'reports');
-        await fsUtils.writeFile(
-          pathUtils.join(reportsDir, `${baseName}.json`),
-          JSON.stringify({
-            module: baseName,
-            unknowns: Object.fromEntries(entries),
-            generatedAt: new Date().toISOString(),
-          })
-        );
-      } catch {
-        // ignore
+      if (!checkMode) {
+        try {
+          const reportsDir = pathUtils.join('.tywrap', 'reports');
+          await fsUtils.writeFile(
+            pathUtils.join(reportsDir, `${baseName}.json`),
+            JSON.stringify({
+              module: baseName,
+              unknowns: Object.fromEntries(entries),
+              generatedAt: new Date().toISOString(),
+            })
+          );
+        } catch {
+          // ignore
+        }
       }
     }
   }
 
+  if (checkMode) {
+    return { written: [], warnings, outOfDate };
+  }
+
   return { written, warnings };
 }
 

package/src/utils/codec.ts

@@ -43,11 +43,19 @@ export interface SklearnEstimator {
 export type CodecEnvelope =
   | {
       readonly __tywrap__: 'dataframe';
+      readonly codecVersion?: number;
       readonly encoding: 'arrow';
       readonly b64: string;
     }
+  | {
+      readonly __tywrap__: 'dataframe';
+      readonly codecVersion?: number;
+      readonly encoding: 'json';
+      readonly data: unknown;
+    }
   | {
       readonly __tywrap__: 'series';
+      readonly codecVersion?: number;
       readonly encoding: 'arrow' | 'json';
       readonly b64?: string;
       readonly data?: unknown;
@@ -55,6 +63,7 @@ export type CodecEnvelope =
     }
   | {
       readonly __tywrap__: 'ndarray';
+      readonly codecVersion?: number;
       readonly encoding: 'arrow' | 'json';
       readonly b64?: string; // when encoding=arrow
       readonly data?: unknown; // when encoding=json
@@ -62,6 +71,7 @@ export type CodecEnvelope =
     }
   | {
       readonly __tywrap__: 'scipy.sparse';
+      readonly codecVersion?: number;
       readonly encoding: 'json';
       readonly format: 'csr' | 'csc' | 'coo';
       readonly shape: readonly number[];
@@ -74,6 +84,7 @@ export type CodecEnvelope =
     }
   | {
       readonly __tywrap__: 'torch.tensor';
+      readonly codecVersion?: number;
       readonly encoding: 'ndarray';
       readonly value: unknown;
       readonly shape?: readonly number[];
@@ -82,6 +93,7 @@ export type CodecEnvelope =
     }
   | {
       readonly __tywrap__: 'sklearn.estimator';
+      readonly codecVersion?: number;
       readonly encoding: 'json';
       readonly className: string;
       readonly module: string;
@@ -89,7 +101,13 @@ export type CodecEnvelope =
       readonly params: Record<string, unknown>;
     };
 
-export type DecodedValue = ArrowTable | Uint8Array | SparseMatrix | TorchTensor | SklearnEstimator | unknown;
+export type DecodedValue =
+  | ArrowTable
+  | Uint8Array
+  | SparseMatrix
+  | TorchTensor
+  | SklearnEstimator
+  | unknown;
 
 let arrowTableFrom: ((bytes: Uint8Array) => ArrowTable | Uint8Array) | undefined;
 
@@ -110,10 +128,79 @@ export function hasArrowDecoder(): boolean {
   return typeof arrowTableFrom === 'function';
 }
 
+type ArrowModuleLoader = () => unknown | Promise<unknown>;
+
+/**
+ * Detect Node.js runtime capabilities without hard dependencies.
+ *
+ * Why: keep browser/bundler builds safe while still enabling Node-only paths.
+ */
+function isNodeRuntime(): boolean {
+  return (
+    typeof process !== 'undefined' &&
+    typeof (process as { versions?: { node?: string } }).versions?.node === 'string'
+  );
+}
+
+/**
+ * Validate the Arrow module shape and register its IPC decoder.
+ *
+ * Why: centralize tableFromIPC checks so callers get consistent errors and can
+ * rely on a single registration path.
+ */
+function registerArrowDecoderFromModule(module: { tableFromIPC?: unknown }): void {
+  const tableFromIPC = module.tableFromIPC;
+  if (typeof tableFromIPC !== 'function') {
+    throw new Error('apache-arrow does not export tableFromIPC');
+  }
+  registerArrowDecoder((bytes: Uint8Array) => tableFromIPC(bytes));
+}
+
+/**
+ * Attempt to lazily register an Arrow decoder at runtime.
+ *
+ * Why: keep apache-arrow optional while letting NodeBridge (or callers) enable
+ * Arrow decoding when the module is present.
+ */
+export async function autoRegisterArrowDecoder(
+  options: { loader?: ArrowModuleLoader } = {}
+): Promise<boolean> {
+  if (hasArrowDecoder()) {
+    return true;
+  }
+  const loader: ArrowModuleLoader | undefined =
+    options.loader ??
+    (isNodeRuntime()
+      ? async (): Promise<unknown> => {
+          try {
+            const nodeModule = await import('node:module');
+            const require = nodeModule.createRequire(import.meta.url);
+            return require('apache-arrow') as unknown;
+          } catch {
+            return await import('apache-arrow');
+          }
+        }
+      : undefined);
+  if (!loader) {
+    return false;
+  }
+  try {
+    const arrowModule = await loader();
+    registerArrowDecoderFromModule(arrowModule as { tableFromIPC?: unknown });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function isObject(value: unknown): value is { [k: string]: unknown } {
   return typeof value === 'object' && value !== null;
 }
 
+function isNumberArray(value: unknown): value is number[] {
+  return Array.isArray(value) && value.every(item => typeof item === 'number');
+}
+
 function fromBase64(b64: string): Uint8Array {
   if (typeof Buffer !== 'undefined') {
     const buf = Buffer.from(b64, 'base64');
@@ -145,118 +232,254 @@ async function tryDecodeArrowTable(bytes: Uint8Array): Promise<ArrowTable | Uint
   }
 }
 
-function decodeEnvelope<T>(value: unknown, decodeArrow: (bytes: Uint8Array) => T): T | unknown {
+type MaybePromise<T> = T | Promise<T>;
+
+function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'then' in value &&
+    typeof (value as { then?: unknown }).then === 'function'
+  );
+}
+
+// Why: decoding needs to reject incompatible envelopes before we attempt to interpret payloads.
+const CODEC_VERSION = 1;
+
+function assertCodecVersion(envelope: { codecVersion?: unknown }, marker: string): void {
+  if (!('codecVersion' in envelope)) {
+    return;
+  }
+  const version = envelope.codecVersion;
+  if (version === undefined) {
+    return;
+  }
+  if (typeof version !== 'number' || !Number.isFinite(version)) {
+    throw new Error(`Invalid ${marker} envelope: codecVersion must be a number`);
+  }
+  if (version !== CODEC_VERSION) {
+    throw new Error(`Unsupported ${marker} envelope codecVersion: ${version}`);
+  }
+}
+
+function decodeEnvelopeCore<T>(
+  value: unknown,
+  decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
+  recurse: (value: unknown) => MaybePromise<T | unknown>
+): MaybePromise<T | unknown> {
   if (!isObject(value)) {
     return value;
   }
   const marker = (value as { __tywrap__?: unknown }).__tywrap__;
-  if (
-    (marker === 'dataframe' || marker === 'series') &&
-    (value as { encoding?: unknown }).encoding === 'arrow' &&
-    typeof (value as { b64?: unknown }).b64 === 'string'
-  ) {
-    const bytes = fromBase64(String((value as { b64: string }).b64));
-    return decodeArrow(bytes);
+  if (typeof marker !== 'string') {
+    return value as unknown;
   }
+
   if (
-    marker === 'dataframe' &&
-    (value as { encoding?: unknown }).encoding === 'json' &&
-    'data' in (value as object)
+    marker === 'dataframe' ||
+    marker === 'series' ||
+    marker === 'ndarray' ||
+    marker === 'scipy.sparse' ||
+    marker === 'torch.tensor' ||
+    marker === 'sklearn.estimator'
   ) {
-    return (value as { data: unknown }).data;
+    assertCodecVersion(value as { codecVersion?: unknown }, marker);
   }
-  if (
-    marker === 'series' &&
-    (value as { encoding?: unknown }).encoding === 'json' &&
-    'data' in (value as object)
-  ) {
-    return (value as { data: unknown }).data;
+
+  if (marker === 'dataframe') {
+    const encoding = (value as { encoding?: unknown }).encoding;
+    if (encoding === 'arrow') {
+      const b64 = (value as { b64?: unknown }).b64;
+      if (typeof b64 !== 'string') {
+        throw new Error('Invalid dataframe envelope: missing b64');
+      }
+      const bytes = fromBase64(b64);
+      return decodeArrow(bytes);
+    }
+    if (encoding === 'json') {
+      if (!('data' in (value as object))) {
+        throw new Error('Invalid dataframe envelope: missing data');
+      }
+      return (value as { data: unknown }).data;
+    }
+    throw new Error(`Invalid dataframe envelope: unsupported encoding ${String(encoding)}`);
+  }
+
+  if (marker === 'series') {
+    const encoding = (value as { encoding?: unknown }).encoding;
+    if (encoding === 'arrow') {
+      const b64 = (value as { b64?: unknown }).b64;
+      if (typeof b64 !== 'string') {
+        throw new Error('Invalid series envelope: missing b64');
+      }
+      const bytes = fromBase64(b64);
+      return decodeArrow(bytes);
+    }
+    if (encoding === 'json') {
+      if (!('data' in (value as object))) {
+        throw new Error('Invalid series envelope: missing data');
+      }
+      return (value as { data: unknown }).data;
+    }
+    throw new Error(`Invalid series envelope: unsupported encoding ${String(encoding)}`);
   }
+
   if (marker === 'ndarray') {
-    if (
-      (value as { encoding?: unknown }).encoding === 'arrow' &&
-      typeof (value as { b64?: unknown }).b64 === 'string'
-    ) {
-      const bytes = fromBase64(String((value as { b64: string }).b64));
+    const encoding = (value as { encoding?: unknown }).encoding;
+    if (encoding === 'arrow') {
+      const b64 = (value as { b64?: unknown }).b64;
+      if (typeof b64 !== 'string') {
+        throw new Error('Invalid ndarray envelope: missing b64');
+      }
+      const bytes = fromBase64(b64);
       return decodeArrow(bytes);
     }
-    if ((value as { encoding?: unknown }).encoding === 'json' && 'data' in (value as object)) {
+    if (encoding === 'json') {
+      if (!('data' in (value as object))) {
+        throw new Error('Invalid ndarray envelope: missing data');
+      }
       return (value as { data: unknown }).data;
     }
+    throw new Error(`Invalid ndarray envelope: unsupported encoding ${String(encoding)}`);
   }
-  if (
-    marker === 'scipy.sparse' &&
-    (value as { encoding?: unknown }).encoding === 'json' &&
-    typeof (value as { format?: unknown }).format === 'string' &&
-    Array.isArray((value as { shape?: unknown }).shape) &&
-    Array.isArray((value as { data?: unknown }).data)
-  ) {
-    const sparse = value as {
-      format: 'csr' | 'csc' | 'coo';
-      shape: readonly number[];
-      data: readonly unknown[];
-      indices?: readonly number[];
-      indptr?: readonly number[];
-      row?: readonly number[];
-      col?: readonly number[];
-      dtype?: string;
-    };
+
+  if (marker === 'scipy.sparse') {
+    const encoding = (value as { encoding?: unknown }).encoding;
+    if (encoding !== 'json') {
+      throw new Error(`Invalid scipy.sparse envelope: unsupported encoding ${String(encoding)}`);
+    }
+    const format = (value as { format?: unknown }).format;
+    if (format !== 'csr' && format !== 'csc' && format !== 'coo') {
+      throw new Error(`Invalid scipy.sparse envelope: unsupported format ${String(format)}`);
+    }
+    const shape = (value as { shape?: unknown }).shape;
+    if (
+      !Array.isArray(shape) ||
+      shape.length !== 2 ||
+      typeof shape[0] !== 'number' ||
+      typeof shape[1] !== 'number'
+    ) {
+      throw new Error('Invalid scipy.sparse envelope: shape must be a 2-item number[]');
+    }
+    const data = (value as { data?: unknown }).data;
+    if (!Array.isArray(data)) {
+      throw new Error('Invalid scipy.sparse envelope: data must be an array');
+    }
+    const dtypeValue = (value as { dtype?: unknown }).dtype;
+    const dtype = typeof dtypeValue === 'string' ? dtypeValue : undefined;
+
+    if (format === 'coo') {
+      const row = (value as { row?: unknown }).row;
+      const col = (value as { col?: unknown }).col;
+      if (!Array.isArray(row) || !Array.isArray(col)) {
+        throw new Error('Invalid scipy.sparse envelope: coo requires row and col arrays');
+      }
+      return {
+        format,
+        shape,
+        data,
+        row,
+        col,
+        dtype,
+      } satisfies SparseMatrix;
+    }
+
+    const indices = (value as { indices?: unknown }).indices;
+    const indptr = (value as { indptr?: unknown }).indptr;
+    if (!Array.isArray(indices) || !Array.isArray(indptr)) {
+      throw new Error('Invalid scipy.sparse envelope: csr/csc requires indices and indptr arrays');
+    }
     return {
-      format: sparse.format,
-      shape: sparse.shape,
-      data: sparse.data,
-      indices: sparse.indices,
-      indptr: sparse.indptr,
-      row: sparse.row,
-      col: sparse.col,
-      dtype: sparse.dtype,
+      format,
+      shape,
+      data,
+      indices,
+      indptr,
+      dtype,
     } satisfies SparseMatrix;
   }
-  if (marker === 'torch.tensor' && (value as { encoding?: unknown }).encoding === 'ndarray') {
-    const torchValue = value as {
-      value?: unknown;
-      shape?: readonly number[];
-      dtype?: string;
-      device?: string;
-    };
-    if ('value' in (torchValue as object)) {
-      const decoded = decodeEnvelope(torchValue.value, decodeArrow);
-      return {
-        data: decoded,
-        shape: torchValue.shape,
-        dtype: torchValue.dtype,
-        device: torchValue.device,
-      } satisfies TorchTensor;
+
+  if (marker === 'torch.tensor') {
+    const encoding = (value as { encoding?: unknown }).encoding;
+    if (encoding !== 'ndarray') {
+      throw new Error(`Invalid torch.tensor envelope: unsupported encoding ${String(encoding)}`);
+    }
+    if (!('value' in (value as object))) {
+      throw new Error('Invalid torch.tensor envelope: missing value');
     }
+    const nested = (value as { value: unknown }).value;
+    if (!isObject(nested) || (nested as { __tywrap__?: unknown }).__tywrap__ !== 'ndarray') {
+      throw new Error('Invalid torch.tensor envelope: value must be an ndarray envelope');
+    }
+    const decoded = recurse(nested);
+    const shapeValue = (value as { shape?: unknown }).shape;
+    const shape = isNumberArray(shapeValue) ? shapeValue : undefined;
+    const dtypeValue = (value as { dtype?: unknown }).dtype;
+    const dtype = typeof dtypeValue === 'string' ? dtypeValue : undefined;
+    const deviceValue = (value as { device?: unknown }).device;
+    const device = typeof deviceValue === 'string' ? deviceValue : undefined;
+
+    if (isPromiseLike(decoded)) {
+      return decoded.then(data => ({ data, shape, dtype, device })) as Promise<T | unknown>;
+    }
+    return { data: decoded, shape, dtype, device } satisfies TorchTensor;
   }
-  if (
-    marker === 'sklearn.estimator' &&
-    (value as { encoding?: unknown }).encoding === 'json' &&
-    typeof (value as { className?: unknown }).className === 'string' &&
-    typeof (value as { module?: unknown }).module === 'string' &&
-    isObject((value as { params?: unknown }).params)
-  ) {
-    const estimator = value as {
-      className: string;
-      module: string;
-      version?: string;
-      params: Record<string, unknown>;
-    };
+
+  if (marker === 'sklearn.estimator') {
+    const encoding = (value as { encoding?: unknown }).encoding;
+    if (encoding !== 'json') {
+      throw new Error(
+        `Invalid sklearn.estimator envelope: unsupported encoding ${String(encoding)}`
+      );
+    }
+    const className = (value as { className?: unknown }).className;
+    const module = (value as { module?: unknown }).module;
+    const params = (value as { params?: unknown }).params;
+    if (typeof className !== 'string' || typeof module !== 'string' || !isObject(params)) {
+      throw new Error(
+        'Invalid sklearn.estimator envelope: expected className/module strings + params object'
+      );
+    }
+    const versionValue = (value as { version?: unknown }).version;
+    if (versionValue !== undefined && typeof versionValue !== 'string') {
+      throw new Error('Invalid sklearn.estimator envelope: version must be a string when provided');
+    }
+    const version = typeof versionValue === 'string' ? versionValue : undefined;
     return {
-      className: estimator.className,
-      module: estimator.module,
-      version: estimator.version,
-      params: estimator.params,
+      className,
+      module,
+      version,
+      params,
     } satisfies SklearnEstimator;
   }
+
   return value as unknown;
 }
 
+function decodeEnvelope<T>(value: unknown, decodeArrow: (bytes: Uint8Array) => T): T | unknown {
+  const recurse: (value: unknown) => MaybePromise<T | unknown> = v =>
+    decodeEnvelopeCore(v, decodeArrow, recurse);
+  const decoded = decodeEnvelopeCore(value, decodeArrow, recurse);
+  if (isPromiseLike(decoded)) {
+    throw new Error('Unexpected Promise return from decodeValue; use decodeValueAsync instead.');
+  }
+  return decoded;
+}
+
+async function decodeEnvelopeAsync<T>(
+  value: unknown,
+  decodeArrow: (bytes: Uint8Array) => Promise<T>
+): Promise<T | unknown> {
+  const recurse: (value: unknown) => MaybePromise<T | unknown> = v =>
+    decodeEnvelopeCore(v, decodeArrow, recurse);
+  return await decodeEnvelopeCore(value, decodeArrow, recurse);
+}
+
 /**
  * Decode values produced by the Python bridge.
  */
 export async function decodeValueAsync(value: unknown): Promise<DecodedValue> {
-  return await decodeEnvelope(value, tryDecodeArrowTable);
+  return await decodeEnvelopeAsync(value, tryDecodeArrowTable);
 }
 
 /**