space-data-module-sdk 0.2.0 → 0.2.6

package/src/compiler/compileModule.js

@@ -1,8 +1,10 @@
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { execFile as execFileCallback } from "node:child_process";
-import { promisify } from "node:util";
+import {
+  mkdtemp,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 
 import {
   createDeploymentAuthorization,
@@ -11,7 +13,18 @@ import {
 } from "../auth/index.js";
 import { validateArtifactWithStandards } from "../compliance/index.js";
 import { generateEmbeddedManifestSource } from "../embeddedManifest.js";
+import {
+  generateInvokeSupportHeader,
+  generateInvokeSupportSource,
+  resolveInvokeSurfaces,
+} from "./invokeGlue.js";
+import {
+  getFlatbuffersCppRuntimeHeaders,
+  getInvokeCppSchemaHeaders,
+} from "./flatcSupport.js";
+import { runWithEmceptionLock } from "./emceptionNode.js";
 import { encodePluginManifest, toEmbeddedPluginManifest } from "../manifest/index.js";
+import { DefaultInvokeExports, InvokeSurface } from "../runtime/constants.js";
 import {
   encryptJsonForRecipient,
   generateX25519Keypair,
@@ -26,7 +39,6 @@ import {
 import { sha256Bytes } from "../utils/crypto.js";
 import { getWasmWallet } from "../utils/wasmCrypto.js";
 
-const execFile = promisify(execFileCallback);
 const C_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
 
 function selectCompiler(language) {
@@ -49,7 +61,7 @@ function ensureExportableMethodIds(manifest) {
   }
 }
 
-function buildCompilerArgs(compiler, exportedSymbols, options = {}) {
+function buildCompilerArgs(exportedSymbols, options = {}) {
   const linkerExports = exportedSymbols.map(
     (symbol) => "-Wl,--export=" + symbol,
   );
@@ -57,125 +69,159 @@ function buildCompilerArgs(compiler, exportedSymbols, options = {}) {
   if (options.allowUndefinedImports === true) {
     extraArgs.push("-s", "ERROR_ON_UNDEFINED_SYMBOLS=0", "-Wl,--allow-undefined");
   }
-  return [
-    "-O2",
-    "--no-entry",
-    "-s",
-    "STANDALONE_WASM=1",
-    ...extraArgs,
-    ...linkerExports,
-  ];
+  const args = ["-O2", "-s", "STANDALONE_WASM=1", ...extraArgs, ...linkerExports];
+  if (options.noEntry === true) {
+    args.splice(1, 0, "--no-entry");
+  }
+  return args;
 }
 
-// ---------------------------------------------------------------------------
-// Emception — in-process WASM-based Emscripten (preferred)
-// ---------------------------------------------------------------------------
-
-let emceptionInstance = null;
-let emceptionLoadAttempted = false;
+async function getInvokeCppSupportFiles() {
+  const [runtimeHeaders, schemaHeaders] = await Promise.all([
+    getFlatbuffersCppRuntimeHeaders(),
+    getInvokeCppSchemaHeaders(),
+  ]);
+  return { runtimeHeaders, schemaHeaders };
+}
 
-async function loadEmception() {
-  if (emceptionLoadAttempted) return emceptionInstance;
-  emceptionLoadAttempted = true;
-  try {
-    const { default: Emception } = await import(
-      "sdn-emception"
-    );
-    emceptionInstance = new Emception();
-    await emceptionInstance.init();
-    return emceptionInstance;
-  } catch {
-    // emception not available or init failed — fall back to system emcc
+async function writeFilesToEmception(emception, rootDir, files) {
+  for (const [relativePath, content] of Object.entries(files)) {
+    const filePath = path.posix.join(rootDir, relativePath);
+    emception.FS.mkdirTree(path.posix.dirname(filePath));
+    emception.writeFile(filePath, content);
   }
-  return null;
 }
 
-async function compileWithEmception(
-  emception,
-  compiler,
-  sourceCode,
-  manifestSource,
-  exportedSymbols,
-  compileOptions,
-) {
-  const ext = compiler.extension;
-  const inputPath = `/working/module.${ext}`;
-  const manifestPath = "/working/plugin-manifest-exports.c";
-  const outputPath = "/working/module.wasm";
-
-  emception.writeFile(inputPath, sourceCode);
-  emception.writeFile(manifestPath, manifestSource);
-
-  const args = buildCompilerArgs(compiler, exportedSymbols, compileOptions);
-  const cmd = [
-    compiler.command,
-    inputPath,
-    manifestPath,
-    ...args,
-    "-o",
-    outputPath,
-  ].join(" ");
-
-  const result = emception.run(cmd);
-  if (result.returncode !== 0) {
-    throw new Error(
-      `Compilation failed with ${compiler.command} (emception): ${result.stderr || result.stdout}`,
-    );
+function removeEmceptionDirectory(emception, directoryPath) {
+  if (!emception.FS.analyzePath(directoryPath).exists) {
+    return;
   }
-
-  const wasmBytes = emception.readFile(outputPath);
-  return new Uint8Array(wasmBytes);
+  const entries = emception.FS.readdir(directoryPath).filter(
+    (entry) => entry !== "." && entry !== "..",
+  );
+  for (const entry of entries) {
+    const entryPath = path.posix.join(directoryPath, entry);
+    const stat = emception.FS.stat(entryPath);
+    if (emception.FS.isDir(stat.mode)) {
+      removeEmceptionDirectory(emception, entryPath);
+      emception.FS.rmdir(entryPath);
+    } else {
+      emception.FS.unlink(entryPath);
+    }
+  }
+  emception.FS.rmdir(directoryPath);
 }
 
-// ---------------------------------------------------------------------------
-// System Emscripten — fallback to emcc/em++ on PATH
-// ---------------------------------------------------------------------------
-
-async function compileWithSystemEmcc(
-  compiler,
-  sourceCode,
-  manifestSource,
-  exportedSymbols,
-  outputPath,
-  compileOptions,
-) {
+async function compileWithEmception(options = {}) {
+  const {
+    sourceCompilerCommand,
+    sourceExtension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
+    outputPath,
+    compileOptions,
+  } = options;
   const tempDir = await mkdtemp(
     path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
   );
-  const sourcePath = path.join(tempDir, `module.${compiler.extension}`);
-  const manifestSourcePath = path.join(tempDir, "plugin-manifest-exports.c");
   const resolvedOutputPath = path.resolve(
     outputPath ?? path.join(tempDir, "module.wasm"),
   );
 
-  await writeFile(sourcePath, sourceCode, "utf8");
-  await writeFile(manifestSourcePath, manifestSource, "utf8");
-
-  const args = buildCompilerArgs(compiler, exportedSymbols, compileOptions);
-
   try {
-    await execFile(compiler.command, [
-      sourcePath,
-      manifestSourcePath,
-      ...args,
-      "-o",
-      resolvedOutputPath,
-    ], { timeout: 120_000 });
+    return await runWithEmceptionLock(async (emception) => {
+      const workDir = "/working/space-data-module-sdk-compile";
+      const runtimeIncludeDir = path.posix.join(workDir, "flatbuffers-runtime");
+      const sourcePath = path.posix.join(workDir, `module.${sourceExtension}`);
+      const manifestSourcePath = path.posix.join(workDir, "plugin-manifest-exports.cpp");
+      const invokeHeaderPath = path.posix.join(workDir, "space_data_module_invoke.h");
+      const invokeSourcePath = path.posix.join(workDir, "plugin-invoke-bridge.cpp");
+      const sourceObjectPath = path.posix.join(workDir, "module.o");
+      const manifestObjectPath = path.posix.join(workDir, "plugin-manifest-exports.o");
+      const invokeObjectPath = path.posix.join(workDir, "plugin-invoke-bridge.o");
+      const wasmOutputPath = path.posix.join(workDir, "module.wasm");
+
+      const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
+      const args = buildCompilerArgs(exportedSymbols, compileOptions);
+
+      try {
+        emception.FS.mkdirTree(workDir);
+        await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
+        await writeFilesToEmception(emception, workDir, schemaHeaders);
+        emception.writeFile(sourcePath, sourceCode);
+        emception.writeFile(manifestSourcePath, manifestSource);
+        emception.writeFile(invokeHeaderPath, invokeHeaderSource);
+        emception.writeFile(invokeSourcePath, invokeSource);
+
+        const commands = [
+          [
+            sourceCompilerCommand,
+            "-c",
+            sourcePath,
+            `-I${workDir}`,
+            "-o",
+            sourceObjectPath,
+          ],
+          [
+            "em++",
+            "-c",
+            manifestSourcePath,
+            "-std=c++17",
+            `-I${workDir}`,
+            `-I${runtimeIncludeDir}`,
+            "-o",
+            manifestObjectPath,
+          ],
+          [
+            "em++",
+            "-c",
+            invokeSourcePath,
+            "-std=c++17",
+            `-I${workDir}`,
+            `-I${runtimeIncludeDir}`,
+            "-o",
+            invokeObjectPath,
+          ],
+          [
+            "em++",
+            sourceObjectPath,
+            manifestObjectPath,
+            invokeObjectPath,
+            ...args,
+            "-o",
+            wasmOutputPath,
+          ],
+        ];
+
+        for (const command of commands) {
+          const result = emception.run(command.join(" "));
+          if (result.returncode !== 0) {
+            throw new Error(
+              `Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
+            );
+          }
+        }
+
+        const wasmBytes = new Uint8Array(emception.readFile(wasmOutputPath));
+        await writeFile(resolvedOutputPath, wasmBytes);
+        return { wasmBytes, outputPath: resolvedOutputPath, tempDir };
+      } finally {
+        try {
+          removeEmceptionDirectory(emception, workDir);
+        } catch {
+          // Best-effort cleanup only; the shared emception instance remains usable.
+        }
+      }
+    });
   } catch (error) {
-    error.message =
-      `Compilation failed with ${compiler.command}: ` +
-      (error.stderr || error.message);
+    await rm(tempDir, { recursive: true, force: true });
     throw error;
   }
-
-  const wasmBytes = await readFile(resolvedOutputPath);
-  return { wasmBytes, outputPath: resolvedOutputPath, tempDir };
 }
 
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
 export async function compileModuleFromSource(options = {}) {
   const manifest = options.manifest ?? {};
   const sourceCode = String(options.sourceCode ?? "");
@@ -193,16 +239,27 @@ export async function compileModuleFromSource(options = {}) {
   }
 
   const compiler = selectCompiler(options.language);
+  const invokeSurfaces = resolveInvokeSurfaces(manifest);
+  const includeCommandMain = invokeSurfaces.includes(InvokeSurface.COMMAND);
   const { manifest: embeddedManifest, warnings } = toEmbeddedPluginManifest(
     manifest,
   );
   const manifestSource = generateEmbeddedManifestSource({
     manifest: embeddedManifest,
   });
+  const invokeHeaderSource = generateInvokeSupportHeader();
+  const invokeSource = generateInvokeSupportSource({
+    manifest,
+    includeCommandMain,
+  });
 
   const exportedSymbols = [
     "plugin_get_manifest_flatbuffer",
     "plugin_get_manifest_flatbuffer_size",
+    DefaultInvokeExports.invokeSymbol,
+    DefaultInvokeExports.allocSymbol,
+    DefaultInvokeExports.freeSymbol,
+    ...(includeCommandMain ? [DefaultInvokeExports.commandSymbol] : []),
     ...new Set(
       (Array.isArray(manifest.methods) ? manifest.methods : [])
         .map((method) => String(method?.methodId ?? "").trim())
@@ -210,54 +267,36 @@ export async function compileModuleFromSource(options = {}) {
     ),
   ];
 
-  // Try emception first, fall back to system emcc/em++
-  const emception = await loadEmception();
-
   let wasmBytes;
   let resolvedOutputPath = null;
   let tempDir = null;
-  let compilerBackend;
-
-  if (emception) {
-    wasmBytes = await compileWithEmception(
-      emception,
-      compiler,
-      sourceCode,
-      manifestSource,
-      exportedSymbols,
-      options,
-    );
-    compilerBackend = `${compiler.command} (emception)`;
-  } else {
-    const result = await compileWithSystemEmcc(
-      compiler,
-      sourceCode,
-      manifestSource,
-      exportedSymbols,
-      options.outputPath,
-      options,
-    );
-    wasmBytes = result.wasmBytes;
-    resolvedOutputPath = result.outputPath;
-    tempDir = result.tempDir;
-    compilerBackend = `${compiler.command} (system)`;
-  }
+  const compileOptions = {
+    ...options,
+    noEntry: includeCommandMain !== true,
+  };
+  const result = await compileWithEmception({
+    sourceCompilerCommand: compiler.command,
+    sourceExtension: compiler.extension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
+    outputPath: options.outputPath,
+    compileOptions,
+  });
+  wasmBytes = result.wasmBytes;
+  resolvedOutputPath = result.outputPath;
+  tempDir = result.tempDir;
 
   // Validate the compiled artifact
-  const report = emception
-    ? await validateArtifactWithStandards({
-        manifest,
-        exportNames: WebAssembly.Module.exports(
-          new WebAssembly.Module(wasmBytes),
-        ).map((e) => e.name),
-      })
-    : await validateArtifactWithStandards({
-        manifest,
-        wasmPath: resolvedOutputPath,
-      });
+  const report = await validateArtifactWithStandards({
+    manifest,
+    wasmPath: resolvedOutputPath,
+  });
 
   return {
-    compiler: compilerBackend,
+    compiler: "em++ (emception)",
     language: compiler.language,
     outputPath: resolvedOutputPath,
     tempDir,

package/src/compiler/emception.d.ts

@@ -0,0 +1,60 @@
+export interface EmceptionCommandResult {
+  command: string;
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+export type SharedEmceptionFileContent =
+  | string
+  | Uint8Array
+  | ArrayBuffer
+  | ArrayBufferView;
+
+export interface SharedEmceptionHandle {
+  getRaw(): unknown;
+  exists(targetPath: string): boolean;
+  mkdirTree(directoryPath: string): void;
+  writeFile(filePath: string, content: SharedEmceptionFileContent): void;
+  writeFiles(
+    rootDir: string,
+    files: Record<string, SharedEmceptionFileContent>,
+  ): void;
+  readFile(filePath: string): Uint8Array;
+  readFile(filePath: string, options: { encoding: "utf8" }): string;
+  removeTree(targetPath: string): void;
+  run(
+    command: string,
+    options?: { throwOnNonZero?: boolean },
+  ): EmceptionCommandResult;
+}
+
+export interface SharedEmceptionSession {
+  load(): Promise<unknown>;
+  withLock<T>(
+    task: (handle: SharedEmceptionHandle) => T | Promise<T>,
+  ): Promise<T>;
+  exists(targetPath: string): Promise<boolean>;
+  mkdirTree(directoryPath: string): Promise<void>;
+  writeFile(
+    filePath: string,
+    content: SharedEmceptionFileContent,
+  ): Promise<void>;
+  writeFiles(
+    rootDir: string,
+    files: Record<string, SharedEmceptionFileContent>,
+  ): Promise<void>;
+  readFile(filePath: string): Promise<Uint8Array>;
+  readFile(filePath: string, options: { encoding: "utf8" }): Promise<string>;
+  removeTree(targetPath: string): Promise<void>;
+  run(
+    command: string,
+    options?: { throwOnNonZero?: boolean },
+  ): Promise<EmceptionCommandResult>;
+}
+
+export function createSharedEmceptionSession(): SharedEmceptionSession;
+export function loadSharedEmception(): Promise<unknown>;
+export function withSharedEmception<T>(
+  task: (handle: SharedEmceptionHandle) => T | Promise<T>,
+): Promise<T>;

package/src/compiler/emception.js

@@ -0,0 +1,191 @@
+import path from "node:path";
+
+import {
+  getSharedEmceptionController,
+  loadEmception,
+  runWithEmceptionLock,
+} from "./emceptionNode.js";
+
+const TEXT_DECODER = new TextDecoder();
+const TEXT_ENCODER = new TextEncoder();
+
+function normalizeFileContent(content) {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (content instanceof Uint8Array) {
+    return content;
+  }
+  if (content instanceof ArrayBuffer) {
+    return new Uint8Array(content);
+  }
+  if (ArrayBuffer.isView(content)) {
+    return new Uint8Array(
+      content.buffer,
+      content.byteOffset,
+      content.byteLength,
+    );
+  }
+  throw new TypeError(
+    "Emception file content must be a string, Uint8Array, ArrayBuffer, or ArrayBufferView.",
+  );
+}
+
+function cloneReadBytes(value) {
+  if (typeof value === "string") {
+    return TEXT_ENCODER.encode(value);
+  }
+  const bytes = normalizeFileContent(value);
+  return new Uint8Array(bytes);
+}
+
+function removeTree(emception, targetPath) {
+  const analysis = emception.FS.analyzePath(targetPath);
+  if (!analysis.exists) {
+    return;
+  }
+  const stat = emception.FS.stat(targetPath);
+  if (!emception.FS.isDir(stat.mode)) {
+    emception.FS.unlink(targetPath);
+    return;
+  }
+  const entries = emception.FS.readdir(targetPath).filter(
+    (entry) => entry !== "." && entry !== "..",
+  );
+  for (const entry of entries) {
+    removeTree(emception, path.posix.join(targetPath, entry));
+  }
+  emception.FS.rmdir(targetPath);
+}
+
+function normalizeRunResult(command, result) {
+  const normalized = {
+    command,
+    exitCode: Number(result?.returncode ?? 0) >>> 0,
+    stdout: String(result?.stdout ?? ""),
+    stderr: String(result?.stderr ?? ""),
+  };
+  return normalized;
+}
+
+function maybeThrowRunFailure(result, options = {}) {
+  if (options.throwOnNonZero === false || result.exitCode === 0) {
+    return result;
+  }
+  const detail = result.stderr || result.stdout || "unknown emception failure";
+  throw new Error(
+    `Emception command failed with exit code ${result.exitCode}: ${result.command}\n${detail}`,
+  );
+}
+
+class SharedEmceptionHandle {
+  constructor(emception) {
+    this.emception = emception;
+  }
+
+  getRaw() {
+    return this.emception;
+  }
+
+  exists(targetPath) {
+    return this.emception.FS.analyzePath(targetPath).exists;
+  }
+
+  mkdirTree(directoryPath) {
+    this.emception.FS.mkdirTree(directoryPath);
+  }
+
+  writeFile(filePath, content) {
+    this.emception.FS.mkdirTree(path.posix.dirname(filePath));
+    this.emception.writeFile(filePath, normalizeFileContent(content));
+  }
+
+  writeFiles(rootDir, files) {
+    for (const [relativePath, content] of Object.entries(files ?? {})) {
+      this.writeFile(path.posix.join(rootDir, relativePath), content);
+    }
+  }
+
+  readFile(filePath, options = {}) {
+    const bytes = cloneReadBytes(this.emception.readFile(filePath));
+    if (options.encoding === "utf8") {
+      return TEXT_DECODER.decode(bytes);
+    }
+    return bytes;
+  }
+
+  removeTree(targetPath) {
+    removeTree(this.emception, targetPath);
+  }
+
+  run(command, options = {}) {
+    const result = normalizeRunResult(command, this.emception.run(command));
+    return maybeThrowRunFailure(result, options);
+  }
+}
+
+class SharedEmceptionSession {
+  constructor(controller = getSharedEmceptionController()) {
+    this.controller = controller;
+  }
+
+  async load() {
+    return this.controller.load();
+  }
+
+  async withLock(task) {
+    return this.controller.withLock(
+      (emception) => task(new SharedEmceptionHandle(emception)),
+    );
+  }
+
+  async exists(targetPath) {
+    return this.withLock((handle) => handle.exists(targetPath));
+  }
+
+  async mkdirTree(directoryPath) {
+    await this.withLock((handle) => {
+      handle.mkdirTree(directoryPath);
+    });
+  }
+
+  async writeFile(filePath, content) {
+    await this.withLock((handle) => {
+      handle.writeFile(filePath, content);
+    });
+  }
+
+  async writeFiles(rootDir, files) {
+    await this.withLock((handle) => {
+      handle.writeFiles(rootDir, files);
+    });
+  }
+
+  async readFile(filePath, options = {}) {
+    return this.withLock((handle) => handle.readFile(filePath, options));
+  }
+
+  async removeTree(targetPath) {
+    await this.withLock((handle) => {
+      handle.removeTree(targetPath);
+    });
+  }
+
+  async run(command, options = {}) {
+    return this.withLock((handle) => handle.run(command, options));
+  }
+}
+
+export function createSharedEmceptionSession() {
+  return new SharedEmceptionSession();
+}
+
+export async function loadSharedEmception() {
+  return loadEmception();
+}
+
+export async function withSharedEmception(task) {
+  return runWithEmceptionLock(
+    (emception) => task(new SharedEmceptionHandle(emception)),
+  );
+}