space-data-module-sdk 0.2.5 → 0.2.6

package/README.md

@@ -294,8 +294,11 @@ npm run check:compliance
 ```
 
 Node.js `>=20` is required. The compiler uses `sdn-emception` and `flatc-wasm`
-by default, with a system Emscripten fallback only if the embedded toolchain
-cannot load.
+by default.
+
+If another repo needs the same compiler runtime, the package also exposes a
+shared emception session at `space-data-module-sdk/compiler/emception` with
+helpers for serialized command execution and virtual filesystem access.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "space-data-module-sdk",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "Module SDK for building, validating, signing, and deploying WebAssembly modules on the Space Data Network.",
   "type": "module",
   "types": "./src/index.d.ts",
@@ -9,7 +9,7 @@
     "url": "git+https://github.com/DigitalArsenal/space-data-module-sdk.git"
   },
   "bin": {
-    "space-data-module": "./bin/space-data-module.js"
+    "space-data-module": "bin/space-data-module.js"
   },
   "exports": {
     ".": "./src/index.js",
@@ -18,8 +18,10 @@
     "./auth": "./src/auth/index.js",
     "./transport": "./src/transport/index.js",
     "./compiler": "./src/compiler/index.js",
+    "./compiler/emception": "./src/compiler/emception.js",
     "./bundle": "./src/bundle/index.js",
     "./invoke": "./src/invoke/index.js",
+    "./runtime": "./src/runtime/index.js",
     "./standards": "./src/standards/index.js",
     "./schemas/*": "./schemas/*"
   },

package/src/compiler/compileModule.js

@@ -1,14 +1,10 @@
 import os from "node:os";
 import path from "node:path";
 import {
-  mkdir,
   mkdtemp,
-  readFile,
   rm,
   writeFile,
 } from "node:fs/promises";
-import { execFile as execFileCallback } from "node:child_process";
-import { promisify } from "node:util";
 
 import {
   createDeploymentAuthorization,
@@ -43,7 +39,6 @@ import {
 import { sha256Bytes } from "../utils/crypto.js";
 import { getWasmWallet } from "../utils/wasmCrypto.js";
 
-const execFile = promisify(execFileCallback);
 const C_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
 
 function selectCompiler(language) {
@@ -89,14 +84,6 @@ async function getInvokeCppSupportFiles() {
   return { runtimeHeaders, schemaHeaders };
 }
 
-async function writeFilesToDirectory(rootDir, files) {
-  for (const [relativePath, content] of Object.entries(files)) {
-    const filePath = path.join(rootDir, relativePath);
-    await mkdir(path.dirname(filePath), { recursive: true });
-    await writeFile(filePath, content, "utf8");
-  }
-}
-
 async function writeFilesToEmception(emception, rootDir, files) {
   for (const [relativePath, content] of Object.entries(files)) {
     const filePath = path.posix.join(rootDir, relativePath);
@@ -235,101 +222,6 @@ async function compileWithEmception(options = {}) {
   }
 }
 
-// ---------------------------------------------------------------------------
-// System Emscripten — fallback to emcc/em++ on PATH
-// ---------------------------------------------------------------------------
-
-async function compileWithSystemToolchain(options = {}) {
-  const {
-    compilerCommand,
-    sourceCompilerCommand,
-    sourceExtension,
-    sourceCode,
-    manifestSource,
-    invokeHeaderSource,
-    invokeSource,
-    exportedSymbols,
-    outputPath,
-    compileOptions,
-  } = options;
-  const tempDir = await mkdtemp(
-    path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
-  );
-  const sourcePath = path.join(tempDir, `module.${sourceExtension}`);
-  const manifestSourcePath = path.join(tempDir, "plugin-manifest-exports.cpp");
-  const invokeHeaderPath = path.join(tempDir, "space_data_module_invoke.h");
-  const invokeSourcePath = path.join(tempDir, "plugin-invoke-bridge.cpp");
-  const sourceObjectPath = path.join(tempDir, "module.o");
-  const manifestObjectPath = path.join(tempDir, "plugin-manifest-exports.o");
-  const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
-  const resolvedOutputPath = path.resolve(
-    outputPath ?? path.join(tempDir, "module.wasm"),
-  );
-  const runtimeIncludeDir = path.join(tempDir, "flatbuffers-runtime");
-
-  const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
-
-  await writeFile(sourcePath, sourceCode, "utf8");
-  await writeFile(manifestSourcePath, manifestSource, "utf8");
-  await writeFile(invokeHeaderPath, invokeHeaderSource, "utf8");
-  await writeFile(invokeSourcePath, invokeSource, "utf8");
-  await writeFilesToDirectory(tempDir, schemaHeaders);
-  await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
-
-  const args = buildCompilerArgs(exportedSymbols, compileOptions);
-
-  try {
-    await execFile(sourceCompilerCommand, [
-      "-c",
-      sourcePath,
-      `-I${tempDir}`,
-      "-o",
-      sourceObjectPath,
-    ], { timeout: 120_000 });
-
-    await execFile(compilerCommand, [
-      "-c",
-      manifestSourcePath,
-      "-std=c++17",
-      `-I${tempDir}`,
-      `-I${runtimeIncludeDir}`,
-      "-o",
-      manifestObjectPath,
-    ], { timeout: 120_000 });
-
-    await execFile(compilerCommand, [
-      "-c",
-      invokeSourcePath,
-      "-std=c++17",
-      `-I${tempDir}`,
-      `-I${runtimeIncludeDir}`,
-      "-o",
-      invokeObjectPath,
-    ], { timeout: 120_000 });
-
-    await execFile(compilerCommand, [
-      sourceObjectPath,
-      manifestObjectPath,
-      invokeObjectPath,
-      ...args,
-      "-o",
-      resolvedOutputPath,
-    ], { timeout: 120_000 });
-  } catch (error) {
-    error.message =
-      `Compilation failed with ${compilerCommand}: ` +
-      (error.stderr || error.message);
-    throw error;
-  }
-
-  const wasmBytes = await readFile(resolvedOutputPath);
-  return { wasmBytes, outputPath: resolvedOutputPath, tempDir };
-}
-
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
 export async function compileModuleFromSource(options = {}) {
   const manifest = options.manifest ?? {};
   const sourceCode = String(options.sourceCode ?? "");
@@ -382,38 +274,17 @@ export async function compileModuleFromSource(options = {}) {
     ...options,
     noEntry: includeCommandMain !== true,
   };
-  let compilerBackend = "em++ (emception)";
-  let result;
-  try {
-    result = await compileWithEmception({
-      sourceCompilerCommand: compiler.command,
-      sourceExtension: compiler.extension,
-      sourceCode,
-      manifestSource,
-      invokeHeaderSource,
-      invokeSource,
-      exportedSymbols,
-      outputPath: options.outputPath,
-      compileOptions,
-    });
-  } catch (error) {
-    if (error?.code !== "EMCEPTION_LOAD_FAILED") {
-      throw error;
-    }
-    result = await compileWithSystemToolchain({
-      compilerCommand: "em++",
-      sourceCompilerCommand: compiler.command,
-      sourceExtension: compiler.extension,
-      sourceCode,
-      manifestSource,
-      invokeHeaderSource,
-      invokeSource,
-      exportedSymbols,
-      outputPath: options.outputPath,
-      compileOptions,
-    });
-    compilerBackend = "em++ (system)";
-  }
+  const result = await compileWithEmception({
+    sourceCompilerCommand: compiler.command,
+    sourceExtension: compiler.extension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
+    outputPath: options.outputPath,
+    compileOptions,
+  });
   wasmBytes = result.wasmBytes;
   resolvedOutputPath = result.outputPath;
   tempDir = result.tempDir;
@@ -425,7 +296,7 @@ export async function compileModuleFromSource(options = {}) {
   });
 
   return {
-    compiler: compilerBackend,
+    compiler: "em++ (emception)",
     language: compiler.language,
     outputPath: resolvedOutputPath,
     tempDir,

package/src/compiler/emception.d.ts

@@ -0,0 +1,60 @@
+export interface EmceptionCommandResult {
+  command: string;
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+export type SharedEmceptionFileContent =
+  | string
+  | Uint8Array
+  | ArrayBuffer
+  | ArrayBufferView;
+
+export interface SharedEmceptionHandle {
+  getRaw(): unknown;
+  exists(targetPath: string): boolean;
+  mkdirTree(directoryPath: string): void;
+  writeFile(filePath: string, content: SharedEmceptionFileContent): void;
+  writeFiles(
+    rootDir: string,
+    files: Record<string, SharedEmceptionFileContent>,
+  ): void;
+  readFile(filePath: string): Uint8Array;
+  readFile(filePath: string, options: { encoding: "utf8" }): string;
+  removeTree(targetPath: string): void;
+  run(
+    command: string,
+    options?: { throwOnNonZero?: boolean },
+  ): EmceptionCommandResult;
+}
+
+export interface SharedEmceptionSession {
+  load(): Promise<unknown>;
+  withLock<T>(
+    task: (handle: SharedEmceptionHandle) => T | Promise<T>,
+  ): Promise<T>;
+  exists(targetPath: string): Promise<boolean>;
+  mkdirTree(directoryPath: string): Promise<void>;
+  writeFile(
+    filePath: string,
+    content: SharedEmceptionFileContent,
+  ): Promise<void>;
+  writeFiles(
+    rootDir: string,
+    files: Record<string, SharedEmceptionFileContent>,
+  ): Promise<void>;
+  readFile(filePath: string): Promise<Uint8Array>;
+  readFile(filePath: string, options: { encoding: "utf8" }): Promise<string>;
+  removeTree(targetPath: string): Promise<void>;
+  run(
+    command: string,
+    options?: { throwOnNonZero?: boolean },
+  ): Promise<EmceptionCommandResult>;
+}
+
+export function createSharedEmceptionSession(): SharedEmceptionSession;
+export function loadSharedEmception(): Promise<unknown>;
+export function withSharedEmception<T>(
+  task: (handle: SharedEmceptionHandle) => T | Promise<T>,
+): Promise<T>;

package/src/compiler/emception.js

@@ -0,0 +1,191 @@
+import path from "node:path";
+
+import {
+  getSharedEmceptionController,
+  loadEmception,
+  runWithEmceptionLock,
+} from "./emceptionNode.js";
+
+const TEXT_DECODER = new TextDecoder();
+const TEXT_ENCODER = new TextEncoder();
+
+function normalizeFileContent(content) {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (content instanceof Uint8Array) {
+    return content;
+  }
+  if (content instanceof ArrayBuffer) {
+    return new Uint8Array(content);
+  }
+  if (ArrayBuffer.isView(content)) {
+    return new Uint8Array(
+      content.buffer,
+      content.byteOffset,
+      content.byteLength,
+    );
+  }
+  throw new TypeError(
+    "Emception file content must be a string, Uint8Array, ArrayBuffer, or ArrayBufferView.",
+  );
+}
+
+function cloneReadBytes(value) {
+  if (typeof value === "string") {
+    return TEXT_ENCODER.encode(value);
+  }
+  const bytes = normalizeFileContent(value);
+  return new Uint8Array(bytes);
+}
+
+function removeTree(emception, targetPath) {
+  const analysis = emception.FS.analyzePath(targetPath);
+  if (!analysis.exists) {
+    return;
+  }
+  const stat = emception.FS.stat(targetPath);
+  if (!emception.FS.isDir(stat.mode)) {
+    emception.FS.unlink(targetPath);
+    return;
+  }
+  const entries = emception.FS.readdir(targetPath).filter(
+    (entry) => entry !== "." && entry !== "..",
+  );
+  for (const entry of entries) {
+    removeTree(emception, path.posix.join(targetPath, entry));
+  }
+  emception.FS.rmdir(targetPath);
+}
+
+function normalizeRunResult(command, result) {
+  const normalized = {
+    command,
+    exitCode: Number(result?.returncode ?? 0) >>> 0,
+    stdout: String(result?.stdout ?? ""),
+    stderr: String(result?.stderr ?? ""),
+  };
+  return normalized;
+}
+
+function maybeThrowRunFailure(result, options = {}) {
+  if (options.throwOnNonZero === false || result.exitCode === 0) {
+    return result;
+  }
+  const detail = result.stderr || result.stdout || "unknown emception failure";
+  throw new Error(
+    `Emception command failed with exit code ${result.exitCode}: ${result.command}\n${detail}`,
+  );
+}
+
+class SharedEmceptionHandle {
+  constructor(emception) {
+    this.emception = emception;
+  }
+
+  getRaw() {
+    return this.emception;
+  }
+
+  exists(targetPath) {
+    return this.emception.FS.analyzePath(targetPath).exists;
+  }
+
+  mkdirTree(directoryPath) {
+    this.emception.FS.mkdirTree(directoryPath);
+  }
+
+  writeFile(filePath, content) {
+    this.emception.FS.mkdirTree(path.posix.dirname(filePath));
+    this.emception.writeFile(filePath, normalizeFileContent(content));
+  }
+
+  writeFiles(rootDir, files) {
+    for (const [relativePath, content] of Object.entries(files ?? {})) {
+      this.writeFile(path.posix.join(rootDir, relativePath), content);
+    }
+  }
+
+  readFile(filePath, options = {}) {
+    const bytes = cloneReadBytes(this.emception.readFile(filePath));
+    if (options.encoding === "utf8") {
+      return TEXT_DECODER.decode(bytes);
+    }
+    return bytes;
+  }
+
+  removeTree(targetPath) {
+    removeTree(this.emception, targetPath);
+  }
+
+  run(command, options = {}) {
+    const result = normalizeRunResult(command, this.emception.run(command));
+    return maybeThrowRunFailure(result, options);
+  }
+}
+
+class SharedEmceptionSession {
+  constructor(controller = getSharedEmceptionController()) {
+    this.controller = controller;
+  }
+
+  async load() {
+    return this.controller.load();
+  }
+
+  async withLock(task) {
+    return this.controller.withLock(
+      (emception) => task(new SharedEmceptionHandle(emception)),
+    );
+  }
+
+  async exists(targetPath) {
+    return this.withLock((handle) => handle.exists(targetPath));
+  }
+
+  async mkdirTree(directoryPath) {
+    await this.withLock((handle) => {
+      handle.mkdirTree(directoryPath);
+    });
+  }
+
+  async writeFile(filePath, content) {
+    await this.withLock((handle) => {
+      handle.writeFile(filePath, content);
+    });
+  }
+
+  async writeFiles(rootDir, files) {
+    await this.withLock((handle) => {
+      handle.writeFiles(rootDir, files);
+    });
+  }
+
+  async readFile(filePath, options = {}) {
+    return this.withLock((handle) => handle.readFile(filePath, options));
+  }
+
+  async removeTree(targetPath) {
+    await this.withLock((handle) => {
+      handle.removeTree(targetPath);
+    });
+  }
+
+  async run(command, options = {}) {
+    return this.withLock((handle) => handle.run(command, options));
+  }
+}
+
+export function createSharedEmceptionSession() {
+  return new SharedEmceptionSession();
+}
+
+export async function loadSharedEmception() {
+  return loadEmception();
+}
+
+export async function withSharedEmception(task) {
+  return runWithEmceptionLock(
+    (emception) => task(new SharedEmceptionHandle(emception)),
+  );
+}

package/src/compiler/emceptionNode.js

@@ -16,8 +16,6 @@ const FILE_URL_FETCH_PATCH_FLAG =
   "__spaceDataModuleSdkFileUrlFetchPatched";
 
 let patchedEmceptionRootPromise = null;
-let emceptionInstancePromise = null;
-let emceptionExecutionQueue = Promise.resolve();
 
 function installNodeRuntimeShims() {
   if (typeof globalThis.require !== "function") {
@@ -175,43 +173,62 @@ async function preparePatchedEmceptionRoot() {
   return patchedEmceptionRootPromise;
 }
 
-export async function loadEmception() {
-  if (!emceptionInstancePromise) {
-    emceptionInstancePromise = (async () => {
-      installNodeRuntimeShims();
-      const patchedRoot = await preparePatchedEmceptionRoot();
-      const moduleUrl = pathToFileURL(path.join(patchedRoot, "emception.mjs")).href;
-      const { default: Emception } = await import(moduleUrl);
-      const emception = new Emception({
-        baseUrl: pathToFileURL(`${patchedRoot}${path.sep}`).href,
+class EmceptionController {
+  #instancePromise = null;
+  #executionQueue = Promise.resolve();
+
+  async load() {
+    if (!this.#instancePromise) {
+      this.#instancePromise = (async () => {
+        installNodeRuntimeShims();
+        const patchedRoot = await preparePatchedEmceptionRoot();
+        const moduleUrl = pathToFileURL(path.join(patchedRoot, "emception.mjs")).href;
+        const { default: Emception } = await import(moduleUrl);
+        const emception = new Emception({
+          baseUrl: pathToFileURL(`${patchedRoot}${path.sep}`).href,
+        });
+        await emception.init();
+        return emception;
+      })().catch((error) => {
+        this.#instancePromise = null;
+        throw error;
       });
-      await emception.init();
-      return emception;
-    })().catch((error) => {
-      emceptionInstancePromise = null;
-      throw error;
+    }
+
+    return this.#instancePromise;
+  }
+
+  async withLock(task) {
+    const previous = this.#executionQueue;
+    let release = () => {};
+    this.#executionQueue = new Promise((resolve) => {
+      release = resolve;
     });
+    await previous.catch(() => {});
+    try {
+      const emception = await this.load().catch((error) => {
+        if (!error.code) {
+          error.code = "EMCEPTION_LOAD_FAILED";
+        }
+        throw error;
+      });
+      return await task(emception);
+    } finally {
+      release();
+    }
   }
+}
+
+const sharedEmceptionController = new EmceptionController();
 
-  return emceptionInstancePromise;
+export function getSharedEmceptionController() {
+  return sharedEmceptionController;
+}
+
+export async function loadEmception() {
+  return sharedEmceptionController.load();
 }
 
 export async function runWithEmceptionLock(task) {
-  const previous = emceptionExecutionQueue;
-  let release = () => {};
-  emceptionExecutionQueue = new Promise((resolve) => {
-    release = resolve;
-  });
-  await previous.catch(() => {});
-  try {
-    const emception = await loadEmception().catch((error) => {
-      if (!error.code) {
-        error.code = "EMCEPTION_LOAD_FAILED";
-      }
-      throw error;
-    });
-    return await task(emception);
-  } finally {
-    release();
-  }
+  return sharedEmceptionController.withLock(task);
 }

package/src/compiler/index.d.ts

@@ -0,0 +1,24 @@
+export type {
+  CompilationResult,
+  ProtectedArtifact,
+} from "../index.js";
+
+export {
+  cleanupCompilation,
+  compileModuleFromSource,
+  createRecipientKeypairHex,
+  protectModuleArtifact,
+} from "../index.js";
+
+export type {
+  EmceptionCommandResult,
+  SharedEmceptionFileContent,
+  SharedEmceptionHandle,
+  SharedEmceptionSession,
+} from "./emception.js";
+
+export {
+  createSharedEmceptionSession,
+  loadSharedEmception,
+  withSharedEmception,
+} from "./emception.js";

package/src/compiler/index.js

@@ -5,3 +5,8 @@ export {
   protectModuleArtifact,
 } from "./compileModule.js";
 
+export {
+  createSharedEmceptionSession,
+  loadSharedEmception,
+  withSharedEmception,
+} from "./emception.js";

package/src/index.d.ts

@@ -408,6 +408,19 @@ export function createRecipientKeypairHex(): Promise<{
   privateKeyHex: string;
 }>;
 
+export type {
+  EmceptionCommandResult,
+  SharedEmceptionFileContent,
+  SharedEmceptionHandle,
+  SharedEmceptionSession,
+} from "./compiler/emception.js";
+
+export {
+  createSharedEmceptionSession,
+  loadSharedEmception,
+  withSharedEmception,
+} from "./compiler/emception.js";
+
 // --- Standards ---
 
 export interface StandardsEntry {

package/src/runtime/index.d.ts

@@ -0,0 +1,13 @@
+export {
+  DefaultInvokeExports,
+  DefaultManifestExports,
+  DrainPolicy,
+  ExternalInterfaceDirection,
+  ExternalInterfaceKind,
+  InvokeSurface,
+  RuntimeTarget,
+} from "../index.js";
+
+export function isArrayBufferLike(value: unknown): boolean;
+export function hasByteAddressableBuffer(value: unknown): boolean;
+export function toUint8Array(value: unknown): Uint8Array | null;

package/src/runtime/index.js

@@ -0,0 +1,2 @@
+export * from "./bufferLike.js";
+export * from "./constants.js";