space-data-module-sdk 0.2.0 → 0.2.5

package/src/compiler/compileModule.js

@@ -1,6 +1,12 @@
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import {
+  mkdir,
+  mkdtemp,
+  readFile,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 import { execFile as execFileCallback } from "node:child_process";
 import { promisify } from "node:util";
 
@@ -11,7 +17,18 @@ import {
 } from "../auth/index.js";
 import { validateArtifactWithStandards } from "../compliance/index.js";
 import { generateEmbeddedManifestSource } from "../embeddedManifest.js";
+import {
+  generateInvokeSupportHeader,
+  generateInvokeSupportSource,
+  resolveInvokeSurfaces,
+} from "./invokeGlue.js";
+import {
+  getFlatbuffersCppRuntimeHeaders,
+  getInvokeCppSchemaHeaders,
+} from "./flatcSupport.js";
+import { runWithEmceptionLock } from "./emceptionNode.js";
 import { encodePluginManifest, toEmbeddedPluginManifest } from "../manifest/index.js";
+import { DefaultInvokeExports, InvokeSurface } from "../runtime/constants.js";
 import {
   encryptJsonForRecipient,
   generateX25519Keypair,
@@ -49,7 +66,7 @@ function ensureExportableMethodIds(manifest) {
   }
 }
 
-function buildCompilerArgs(compiler, exportedSymbols, options = {}) {
+function buildCompilerArgs(exportedSymbols, options = {}) {
   const linkerExports = exportedSymbols.map(
     (symbol) => "-Wl,--export=" + symbol,
   );
@@ -57,113 +74,250 @@ function buildCompilerArgs(compiler, exportedSymbols, options = {}) {
   if (options.allowUndefinedImports === true) {
     extraArgs.push("-s", "ERROR_ON_UNDEFINED_SYMBOLS=0", "-Wl,--allow-undefined");
   }
-  return [
-    "-O2",
-    "--no-entry",
-    "-s",
-    "STANDALONE_WASM=1",
-    ...extraArgs,
-    ...linkerExports,
-  ];
+  const args = ["-O2", "-s", "STANDALONE_WASM=1", ...extraArgs, ...linkerExports];
+  if (options.noEntry === true) {
+    args.splice(1, 0, "--no-entry");
+  }
+  return args;
 }
 
-// ---------------------------------------------------------------------------
-// Emception — in-process WASM-based Emscripten (preferred)
-// ---------------------------------------------------------------------------
+async function getInvokeCppSupportFiles() {
+  const [runtimeHeaders, schemaHeaders] = await Promise.all([
+    getFlatbuffersCppRuntimeHeaders(),
+    getInvokeCppSchemaHeaders(),
+  ]);
+  return { runtimeHeaders, schemaHeaders };
+}
 
-let emceptionInstance = null;
-let emceptionLoadAttempted = false;
+async function writeFilesToDirectory(rootDir, files) {
+  for (const [relativePath, content] of Object.entries(files)) {
+    const filePath = path.join(rootDir, relativePath);
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, content, "utf8");
+  }
+}
 
-async function loadEmception() {
-  if (emceptionLoadAttempted) return emceptionInstance;
-  emceptionLoadAttempted = true;
-  try {
-    const { default: Emception } = await import(
-      "sdn-emception"
-    );
-    emceptionInstance = new Emception();
-    await emceptionInstance.init();
-    return emceptionInstance;
-  } catch {
-    // emception not available or init failed — fall back to system emcc
+async function writeFilesToEmception(emception, rootDir, files) {
+  for (const [relativePath, content] of Object.entries(files)) {
+    const filePath = path.posix.join(rootDir, relativePath);
+    emception.FS.mkdirTree(path.posix.dirname(filePath));
+    emception.writeFile(filePath, content);
   }
-  return null;
 }
 
-async function compileWithEmception(
-  emception,
-  compiler,
-  sourceCode,
-  manifestSource,
-  exportedSymbols,
-  compileOptions,
-) {
-  const ext = compiler.extension;
-  const inputPath = `/working/module.${ext}`;
-  const manifestPath = "/working/plugin-manifest-exports.c";
-  const outputPath = "/working/module.wasm";
-
-  emception.writeFile(inputPath, sourceCode);
-  emception.writeFile(manifestPath, manifestSource);
-
-  const args = buildCompilerArgs(compiler, exportedSymbols, compileOptions);
-  const cmd = [
-    compiler.command,
-    inputPath,
-    manifestPath,
-    ...args,
-    "-o",
+function removeEmceptionDirectory(emception, directoryPath) {
+  if (!emception.FS.analyzePath(directoryPath).exists) {
+    return;
+  }
+  const entries = emception.FS.readdir(directoryPath).filter(
+    (entry) => entry !== "." && entry !== "..",
+  );
+  for (const entry of entries) {
+    const entryPath = path.posix.join(directoryPath, entry);
+    const stat = emception.FS.stat(entryPath);
+    if (emception.FS.isDir(stat.mode)) {
+      removeEmceptionDirectory(emception, entryPath);
+      emception.FS.rmdir(entryPath);
+    } else {
+      emception.FS.unlink(entryPath);
+    }
+  }
+  emception.FS.rmdir(directoryPath);
+}
+
+async function compileWithEmception(options = {}) {
+  const {
+    sourceCompilerCommand,
+    sourceExtension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
     outputPath,
-  ].join(" ");
+    compileOptions,
+  } = options;
+  const tempDir = await mkdtemp(
+    path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
+  );
+  const resolvedOutputPath = path.resolve(
+    outputPath ?? path.join(tempDir, "module.wasm"),
+  );
 
-  const result = emception.run(cmd);
-  if (result.returncode !== 0) {
-    throw new Error(
-      `Compilation failed with ${compiler.command} (emception): ${result.stderr || result.stdout}`,
-    );
+  try {
+    return await runWithEmceptionLock(async (emception) => {
+      const workDir = "/working/space-data-module-sdk-compile";
+      const runtimeIncludeDir = path.posix.join(workDir, "flatbuffers-runtime");
+      const sourcePath = path.posix.join(workDir, `module.${sourceExtension}`);
+      const manifestSourcePath = path.posix.join(workDir, "plugin-manifest-exports.cpp");
+      const invokeHeaderPath = path.posix.join(workDir, "space_data_module_invoke.h");
+      const invokeSourcePath = path.posix.join(workDir, "plugin-invoke-bridge.cpp");
+      const sourceObjectPath = path.posix.join(workDir, "module.o");
+      const manifestObjectPath = path.posix.join(workDir, "plugin-manifest-exports.o");
+      const invokeObjectPath = path.posix.join(workDir, "plugin-invoke-bridge.o");
+      const wasmOutputPath = path.posix.join(workDir, "module.wasm");
+
+      const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
+      const args = buildCompilerArgs(exportedSymbols, compileOptions);
+
+      try {
+        emception.FS.mkdirTree(workDir);
+        await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
+        await writeFilesToEmception(emception, workDir, schemaHeaders);
+        emception.writeFile(sourcePath, sourceCode);
+        emception.writeFile(manifestSourcePath, manifestSource);
+        emception.writeFile(invokeHeaderPath, invokeHeaderSource);
+        emception.writeFile(invokeSourcePath, invokeSource);
+
+        const commands = [
+          [
+            sourceCompilerCommand,
+            "-c",
+            sourcePath,
+            `-I${workDir}`,
+            "-o",
+            sourceObjectPath,
+          ],
+          [
+            "em++",
+            "-c",
+            manifestSourcePath,
+            "-std=c++17",
+            `-I${workDir}`,
+            `-I${runtimeIncludeDir}`,
+            "-o",
+            manifestObjectPath,
+          ],
+          [
+            "em++",
+            "-c",
+            invokeSourcePath,
+            "-std=c++17",
+            `-I${workDir}`,
+            `-I${runtimeIncludeDir}`,
+            "-o",
+            invokeObjectPath,
+          ],
+          [
+            "em++",
+            sourceObjectPath,
+            manifestObjectPath,
+            invokeObjectPath,
+            ...args,
+            "-o",
+            wasmOutputPath,
+          ],
+        ];
+
+        for (const command of commands) {
+          const result = emception.run(command.join(" "));
+          if (result.returncode !== 0) {
+            throw new Error(
+              `Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
+            );
+          }
+        }
+
+        const wasmBytes = new Uint8Array(emception.readFile(wasmOutputPath));
+        await writeFile(resolvedOutputPath, wasmBytes);
+        return { wasmBytes, outputPath: resolvedOutputPath, tempDir };
+      } finally {
+        try {
+          removeEmceptionDirectory(emception, workDir);
+        } catch {
+          // Best-effort cleanup only; the shared emception instance remains usable.
+        }
+      }
+    });
+  } catch (error) {
+    await rm(tempDir, { recursive: true, force: true });
+    throw error;
   }
-
-  const wasmBytes = emception.readFile(outputPath);
-  return new Uint8Array(wasmBytes);
 }
 
 // ---------------------------------------------------------------------------
 // System Emscripten — fallback to emcc/em++ on PATH
 // ---------------------------------------------------------------------------
 
-async function compileWithSystemEmcc(
-  compiler,
-  sourceCode,
-  manifestSource,
-  exportedSymbols,
-  outputPath,
-  compileOptions,
-) {
+async function compileWithSystemToolchain(options = {}) {
+  const {
+    compilerCommand,
+    sourceCompilerCommand,
+    sourceExtension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
+    outputPath,
+    compileOptions,
+  } = options;
   const tempDir = await mkdtemp(
     path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
   );
-  const sourcePath = path.join(tempDir, `module.${compiler.extension}`);
-  const manifestSourcePath = path.join(tempDir, "plugin-manifest-exports.c");
+  const sourcePath = path.join(tempDir, `module.${sourceExtension}`);
+  const manifestSourcePath = path.join(tempDir, "plugin-manifest-exports.cpp");
+  const invokeHeaderPath = path.join(tempDir, "space_data_module_invoke.h");
+  const invokeSourcePath = path.join(tempDir, "plugin-invoke-bridge.cpp");
+  const sourceObjectPath = path.join(tempDir, "module.o");
+  const manifestObjectPath = path.join(tempDir, "plugin-manifest-exports.o");
+  const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
   const resolvedOutputPath = path.resolve(
     outputPath ?? path.join(tempDir, "module.wasm"),
   );
+  const runtimeIncludeDir = path.join(tempDir, "flatbuffers-runtime");
+
+  const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
 
   await writeFile(sourcePath, sourceCode, "utf8");
   await writeFile(manifestSourcePath, manifestSource, "utf8");
+  await writeFile(invokeHeaderPath, invokeHeaderSource, "utf8");
+  await writeFile(invokeSourcePath, invokeSource, "utf8");
+  await writeFilesToDirectory(tempDir, schemaHeaders);
+  await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
 
-  const args = buildCompilerArgs(compiler, exportedSymbols, compileOptions);
+  const args = buildCompilerArgs(exportedSymbols, compileOptions);
 
   try {
-    await execFile(compiler.command, [
+    await execFile(sourceCompilerCommand, [
+      "-c",
       sourcePath,
+      `-I${tempDir}`,
+      "-o",
+      sourceObjectPath,
+    ], { timeout: 120_000 });
+
+    await execFile(compilerCommand, [
+      "-c",
       manifestSourcePath,
+      "-std=c++17",
+      `-I${tempDir}`,
+      `-I${runtimeIncludeDir}`,
+      "-o",
+      manifestObjectPath,
+    ], { timeout: 120_000 });
+
+    await execFile(compilerCommand, [
+      "-c",
+      invokeSourcePath,
+      "-std=c++17",
+      `-I${tempDir}`,
+      `-I${runtimeIncludeDir}`,
+      "-o",
+      invokeObjectPath,
+    ], { timeout: 120_000 });
+
+    await execFile(compilerCommand, [
+      sourceObjectPath,
+      manifestObjectPath,
+      invokeObjectPath,
       ...args,
       "-o",
       resolvedOutputPath,
     ], { timeout: 120_000 });
   } catch (error) {
     error.message =
-      `Compilation failed with ${compiler.command}: ` +
+      `Compilation failed with ${compilerCommand}: ` +
       (error.stderr || error.message);
     throw error;
   }
@@ -193,16 +347,27 @@ export async function compileModuleFromSource(options = {}) {
   }
 
   const compiler = selectCompiler(options.language);
+  const invokeSurfaces = resolveInvokeSurfaces(manifest);
+  const includeCommandMain = invokeSurfaces.includes(InvokeSurface.COMMAND);
   const { manifest: embeddedManifest, warnings } = toEmbeddedPluginManifest(
     manifest,
   );
   const manifestSource = generateEmbeddedManifestSource({
     manifest: embeddedManifest,
   });
+  const invokeHeaderSource = generateInvokeSupportHeader();
+  const invokeSource = generateInvokeSupportSource({
+    manifest,
+    includeCommandMain,
+  });
 
   const exportedSymbols = [
     "plugin_get_manifest_flatbuffer",
     "plugin_get_manifest_flatbuffer_size",
+    DefaultInvokeExports.invokeSymbol,
+    DefaultInvokeExports.allocSymbol,
+    DefaultInvokeExports.freeSymbol,
+    ...(includeCommandMain ? [DefaultInvokeExports.commandSymbol] : []),
     ...new Set(
       (Array.isArray(manifest.methods) ? manifest.methods : [])
         .map((method) => String(method?.methodId ?? "").trim())
@@ -210,51 +375,54 @@ export async function compileModuleFromSource(options = {}) {
     ),
   ];
 
-  // Try emception first, fall back to system emcc/em++
-  const emception = await loadEmception();
-
   let wasmBytes;
   let resolvedOutputPath = null;
   let tempDir = null;
-  let compilerBackend;
-
-  if (emception) {
-    wasmBytes = await compileWithEmception(
-      emception,
-      compiler,
+  const compileOptions = {
+    ...options,
+    noEntry: includeCommandMain !== true,
+  };
+  let compilerBackend = "em++ (emception)";
+  let result;
+  try {
+    result = await compileWithEmception({
+      sourceCompilerCommand: compiler.command,
+      sourceExtension: compiler.extension,
       sourceCode,
       manifestSource,
+      invokeHeaderSource,
+      invokeSource,
       exportedSymbols,
-      options,
-    );
-    compilerBackend = `${compiler.command} (emception)`;
-  } else {
-    const result = await compileWithSystemEmcc(
-      compiler,
+      outputPath: options.outputPath,
+      compileOptions,
+    });
+  } catch (error) {
+    if (error?.code !== "EMCEPTION_LOAD_FAILED") {
+      throw error;
+    }
+    result = await compileWithSystemToolchain({
+      compilerCommand: "em++",
+      sourceCompilerCommand: compiler.command,
+      sourceExtension: compiler.extension,
       sourceCode,
       manifestSource,
+      invokeHeaderSource,
+      invokeSource,
       exportedSymbols,
-      options.outputPath,
-      options,
-    );
-    wasmBytes = result.wasmBytes;
-    resolvedOutputPath = result.outputPath;
-    tempDir = result.tempDir;
-    compilerBackend = `${compiler.command} (system)`;
+      outputPath: options.outputPath,
+      compileOptions,
+    });
+    compilerBackend = "em++ (system)";
   }
+  wasmBytes = result.wasmBytes;
+  resolvedOutputPath = result.outputPath;
+  tempDir = result.tempDir;
 
   // Validate the compiled artifact
-  const report = emception
-    ? await validateArtifactWithStandards({
-        manifest,
-        exportNames: WebAssembly.Module.exports(
-          new WebAssembly.Module(wasmBytes),
-        ).map((e) => e.name),
-      })
-    : await validateArtifactWithStandards({
-        manifest,
-        wasmPath: resolvedOutputPath,
-      });
+  const report = await validateArtifactWithStandards({
+    manifest,
+    wasmPath: resolvedOutputPath,
+  });
 
   return {
     compiler: compilerBackend,

package/src/compiler/emceptionNode.js

@@ -0,0 +1,217 @@
+import os from "node:os";
+import path from "node:path";
+import { cp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+
+const require = createRequire(import.meta.url);
+const PATCH_VERSION = "space-data-module-sdk-emception-node-v1";
+const PATCH_MARKER_FILENAME = ".space-data-module-sdk-emception-patch";
+const EMCEPTION_PATCH_ROOT = path.join(
+  os.tmpdir(),
+  `space-data-module-sdk-emception-node-${process.pid}`,
+);
+const FILE_URL_FETCH_PATCH_FLAG =
+  "__spaceDataModuleSdkFileUrlFetchPatched";
+
+let patchedEmceptionRootPromise = null;
+let emceptionInstancePromise = null;
+let emceptionExecutionQueue = Promise.resolve();
+
+function installNodeRuntimeShims() {
+  if (typeof globalThis.require !== "function") {
+    globalThis.require = require;
+  }
+
+  if (!globalThis.XMLHttpRequest) {
+    class FileUrlXMLHttpRequest {
+      open(method, url, async = true) {
+        this.method = method;
+        this.url = url;
+        this.async = async;
+        this.status = 0;
+        this.response = null;
+      }
+
+      overrideMimeType() {}
+
+      send() {
+        if (this.async) {
+          throw new Error("Async file XMLHttpRequest is not supported.");
+        }
+        const target =
+          typeof this.url === "string" && this.url.startsWith("file:")
+            ? new URL(this.url)
+            : this.url;
+        const data = readFileSync(target);
+        this.status = 200;
+        this.response = data.buffer.slice(
+          data.byteOffset,
+          data.byteOffset + data.byteLength,
+        );
+      }
+    }
+
+    globalThis.XMLHttpRequest = FileUrlXMLHttpRequest;
+  }
+
+  if (!globalThis[FILE_URL_FETCH_PATCH_FLAG]) {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async (input, init) => {
+      const url =
+        typeof input === "string"
+          ? input
+          : input?.url ?? String(input);
+      if (!url.startsWith("file:")) {
+        if (typeof originalFetch !== "function") {
+          throw new Error("fetch is not available for non-file URLs.");
+        }
+        return originalFetch(input, init);
+      }
+      const bytes = await readFile(new URL(url));
+      return {
+        ok: true,
+        status: 200,
+        url,
+        async arrayBuffer() {
+          return bytes.buffer.slice(
+            bytes.byteOffset,
+            bytes.byteOffset + bytes.byteLength,
+          );
+        },
+      };
+    };
+    globalThis[FILE_URL_FETCH_PATCH_FLAG] = true;
+  }
+}
+
+async function patchEmceptionModuleTree(rootDir) {
+  const entries = await readdir(rootDir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      await patchEmceptionModuleTree(fullPath);
+      continue;
+    }
+    if (!entry.name.endsWith(".mjs")) {
+      continue;
+    }
+
+    let source = await readFile(fullPath, "utf8");
+    source = source.replaceAll(
+      'scriptDirectory=__dirname+"/"',
+      'scriptDirectory=(new URL(".", import.meta.url)).pathname',
+    );
+
+    if (entry.name === "emception.mjs") {
+      source = source.replace(
+        "this.#fs = await new FileSystem();",
+        [
+          "this.#fs = await new FileSystem({",
+          "      locateFile: (file, scriptDirectory) => scriptDirectory + file,",
+          '      cache: "/tmp/emception-cache",',
+          "    });",
+        ].join("\n"),
+      );
+      source = source.replace(
+        "const config = {",
+        [
+          "const config = {",
+          "      locateFile: (file, scriptDirectory) => scriptDirectory + file,",
+        ].join("\n"),
+      );
+    }
+
+    if (entry.name === "FileSystem.mjs") {
+      source = source.replace(
+        [
+          "        if (!this.exists(cache)) {",
+          "            this.persist(cache);",
+          "        }",
+          "        await this.pull();",
+        ].join("\n"),
+        [
+          '        if (typeof indexedDB !== "undefined") {',
+          "            if (!this.exists(cache)) {",
+          "                this.persist(cache);",
+          "            }",
+          "            await this.pull();",
+          "        }",
+        ].join("\n"),
+      );
+    }
+
+    await writeFile(fullPath, source, "utf8");
+  }
+}
+
+async function preparePatchedEmceptionRoot() {
+  if (!patchedEmceptionRootPromise) {
+    patchedEmceptionRootPromise = (async () => {
+      const sourceRoot = path.dirname(require.resolve("sdn-emception"));
+      const markerPath = path.join(EMCEPTION_PATCH_ROOT, PATCH_MARKER_FILENAME);
+      let marker = null;
+      try {
+        marker = await readFile(markerPath, "utf8");
+      } catch {
+        marker = null;
+      }
+
+      if (marker?.trim() !== PATCH_VERSION) {
+        await rm(EMCEPTION_PATCH_ROOT, { recursive: true, force: true });
+        await cp(sourceRoot, EMCEPTION_PATCH_ROOT, { recursive: true });
+        await patchEmceptionModuleTree(EMCEPTION_PATCH_ROOT);
+        await writeFile(markerPath, `${PATCH_VERSION}\n`, "utf8");
+      }
+
+      return EMCEPTION_PATCH_ROOT;
+    })().catch((error) => {
+      patchedEmceptionRootPromise = null;
+      throw error;
+    });
+  }
+
+  return patchedEmceptionRootPromise;
+}
+
+export async function loadEmception() {
+  if (!emceptionInstancePromise) {
+    emceptionInstancePromise = (async () => {
+      installNodeRuntimeShims();
+      const patchedRoot = await preparePatchedEmceptionRoot();
+      const moduleUrl = pathToFileURL(path.join(patchedRoot, "emception.mjs")).href;
+      const { default: Emception } = await import(moduleUrl);
+      const emception = new Emception({
+        baseUrl: pathToFileURL(`${patchedRoot}${path.sep}`).href,
+      });
+      await emception.init();
+      return emception;
+    })().catch((error) => {
+      emceptionInstancePromise = null;
+      throw error;
+    });
+  }
+
+  return emceptionInstancePromise;
+}
+
+export async function runWithEmceptionLock(task) {
+  const previous = emceptionExecutionQueue;
+  let release = () => {};
+  emceptionExecutionQueue = new Promise((resolve) => {
+    release = resolve;
+  });
+  await previous.catch(() => {});
+  try {
+    const emception = await loadEmception().catch((error) => {
+      if (!error.code) {
+        error.code = "EMCEPTION_LOAD_FAILED";
+      }
+      throw error;
+    });
+    return await task(emception);
+  } finally {
+    release();
+  }
+}