space-data-module-sdk 0.1.0 → 0.2.5

package/src/compiler/compileModule.js

@@ -1,6 +1,12 @@
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import {
+  mkdir,
+  mkdtemp,
+  readFile,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 import { execFile as execFileCallback } from "node:child_process";
 import { promisify } from "node:util";
 
@@ -11,11 +17,23 @@ import {
 } from "../auth/index.js";
 import { validateArtifactWithStandards } from "../compliance/index.js";
 import { generateEmbeddedManifestSource } from "../embeddedManifest.js";
+import {
+  generateInvokeSupportHeader,
+  generateInvokeSupportSource,
+  resolveInvokeSurfaces,
+} from "./invokeGlue.js";
+import {
+  getFlatbuffersCppRuntimeHeaders,
+  getInvokeCppSchemaHeaders,
+} from "./flatcSupport.js";
+import { runWithEmceptionLock } from "./emceptionNode.js";
 import { encodePluginManifest, toEmbeddedPluginManifest } from "../manifest/index.js";
+import { DefaultInvokeExports, InvokeSurface } from "../runtime/constants.js";
 import {
   encryptJsonForRecipient,
   generateX25519Keypair,
 } from "../transport/index.js";
+import { createSingleFileBundle } from "../bundle/index.js";
 import {
   base64ToBytes,
   bytesToBase64,
@@ -48,6 +66,270 @@ function ensureExportableMethodIds(manifest) {
   }
 }
 
+function buildCompilerArgs(exportedSymbols, options = {}) {
+  const linkerExports = exportedSymbols.map(
+    (symbol) => "-Wl,--export=" + symbol,
+  );
+  const extraArgs = [];
+  if (options.allowUndefinedImports === true) {
+    extraArgs.push("-s", "ERROR_ON_UNDEFINED_SYMBOLS=0", "-Wl,--allow-undefined");
+  }
+  const args = ["-O2", "-s", "STANDALONE_WASM=1", ...extraArgs, ...linkerExports];
+  if (options.noEntry === true) {
+    args.splice(1, 0, "--no-entry");
+  }
+  return args;
+}
+
+async function getInvokeCppSupportFiles() {
+  const [runtimeHeaders, schemaHeaders] = await Promise.all([
+    getFlatbuffersCppRuntimeHeaders(),
+    getInvokeCppSchemaHeaders(),
+  ]);
+  return { runtimeHeaders, schemaHeaders };
+}
+
+async function writeFilesToDirectory(rootDir, files) {
+  for (const [relativePath, content] of Object.entries(files)) {
+    const filePath = path.join(rootDir, relativePath);
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, content, "utf8");
+  }
+}
+
+async function writeFilesToEmception(emception, rootDir, files) {
+  for (const [relativePath, content] of Object.entries(files)) {
+    const filePath = path.posix.join(rootDir, relativePath);
+    emception.FS.mkdirTree(path.posix.dirname(filePath));
+    emception.writeFile(filePath, content);
+  }
+}
+
+function removeEmceptionDirectory(emception, directoryPath) {
+  if (!emception.FS.analyzePath(directoryPath).exists) {
+    return;
+  }
+  const entries = emception.FS.readdir(directoryPath).filter(
+    (entry) => entry !== "." && entry !== "..",
+  );
+  for (const entry of entries) {
+    const entryPath = path.posix.join(directoryPath, entry);
+    const stat = emception.FS.stat(entryPath);
+    if (emception.FS.isDir(stat.mode)) {
+      removeEmceptionDirectory(emception, entryPath);
+      emception.FS.rmdir(entryPath);
+    } else {
+      emception.FS.unlink(entryPath);
+    }
+  }
+  emception.FS.rmdir(directoryPath);
+}
+
+async function compileWithEmception(options = {}) {
+  const {
+    sourceCompilerCommand,
+    sourceExtension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
+    outputPath,
+    compileOptions,
+  } = options;
+  const tempDir = await mkdtemp(
+    path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
+  );
+  const resolvedOutputPath = path.resolve(
+    outputPath ?? path.join(tempDir, "module.wasm"),
+  );
+
+  try {
+    return await runWithEmceptionLock(async (emception) => {
+      const workDir = "/working/space-data-module-sdk-compile";
+      const runtimeIncludeDir = path.posix.join(workDir, "flatbuffers-runtime");
+      const sourcePath = path.posix.join(workDir, `module.${sourceExtension}`);
+      const manifestSourcePath = path.posix.join(workDir, "plugin-manifest-exports.cpp");
+      const invokeHeaderPath = path.posix.join(workDir, "space_data_module_invoke.h");
+      const invokeSourcePath = path.posix.join(workDir, "plugin-invoke-bridge.cpp");
+      const sourceObjectPath = path.posix.join(workDir, "module.o");
+      const manifestObjectPath = path.posix.join(workDir, "plugin-manifest-exports.o");
+      const invokeObjectPath = path.posix.join(workDir, "plugin-invoke-bridge.o");
+      const wasmOutputPath = path.posix.join(workDir, "module.wasm");
+
+      const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
+      const args = buildCompilerArgs(exportedSymbols, compileOptions);
+
+      try {
+        emception.FS.mkdirTree(workDir);
+        await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
+        await writeFilesToEmception(emception, workDir, schemaHeaders);
+        emception.writeFile(sourcePath, sourceCode);
+        emception.writeFile(manifestSourcePath, manifestSource);
+        emception.writeFile(invokeHeaderPath, invokeHeaderSource);
+        emception.writeFile(invokeSourcePath, invokeSource);
+
+        const commands = [
+          [
+            sourceCompilerCommand,
+            "-c",
+            sourcePath,
+            `-I${workDir}`,
+            "-o",
+            sourceObjectPath,
+          ],
+          [
+            "em++",
+            "-c",
+            manifestSourcePath,
+            "-std=c++17",
+            `-I${workDir}`,
+            `-I${runtimeIncludeDir}`,
+            "-o",
+            manifestObjectPath,
+          ],
+          [
+            "em++",
+            "-c",
+            invokeSourcePath,
+            "-std=c++17",
+            `-I${workDir}`,
+            `-I${runtimeIncludeDir}`,
+            "-o",
+            invokeObjectPath,
+          ],
+          [
+            "em++",
+            sourceObjectPath,
+            manifestObjectPath,
+            invokeObjectPath,
+            ...args,
+            "-o",
+            wasmOutputPath,
+          ],
+        ];
+
+        for (const command of commands) {
+          const result = emception.run(command.join(" "));
+          if (result.returncode !== 0) {
+            throw new Error(
+              `Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
+            );
+          }
+        }
+
+        const wasmBytes = new Uint8Array(emception.readFile(wasmOutputPath));
+        await writeFile(resolvedOutputPath, wasmBytes);
+        return { wasmBytes, outputPath: resolvedOutputPath, tempDir };
+      } finally {
+        try {
+          removeEmceptionDirectory(emception, workDir);
+        } catch {
+          // Best-effort cleanup only; the shared emception instance remains usable.
+        }
+      }
+    });
+  } catch (error) {
+    await rm(tempDir, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// System Emscripten — fallback to emcc/em++ on PATH
+// ---------------------------------------------------------------------------
+
+async function compileWithSystemToolchain(options = {}) {
+  const {
+    compilerCommand,
+    sourceCompilerCommand,
+    sourceExtension,
+    sourceCode,
+    manifestSource,
+    invokeHeaderSource,
+    invokeSource,
+    exportedSymbols,
+    outputPath,
+    compileOptions,
+  } = options;
+  const tempDir = await mkdtemp(
+    path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
+  );
+  const sourcePath = path.join(tempDir, `module.${sourceExtension}`);
+  const manifestSourcePath = path.join(tempDir, "plugin-manifest-exports.cpp");
+  const invokeHeaderPath = path.join(tempDir, "space_data_module_invoke.h");
+  const invokeSourcePath = path.join(tempDir, "plugin-invoke-bridge.cpp");
+  const sourceObjectPath = path.join(tempDir, "module.o");
+  const manifestObjectPath = path.join(tempDir, "plugin-manifest-exports.o");
+  const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
+  const resolvedOutputPath = path.resolve(
+    outputPath ?? path.join(tempDir, "module.wasm"),
+  );
+  const runtimeIncludeDir = path.join(tempDir, "flatbuffers-runtime");
+
+  const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
+
+  await writeFile(sourcePath, sourceCode, "utf8");
+  await writeFile(manifestSourcePath, manifestSource, "utf8");
+  await writeFile(invokeHeaderPath, invokeHeaderSource, "utf8");
+  await writeFile(invokeSourcePath, invokeSource, "utf8");
+  await writeFilesToDirectory(tempDir, schemaHeaders);
+  await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
+
+  const args = buildCompilerArgs(exportedSymbols, compileOptions);
+
+  try {
+    await execFile(sourceCompilerCommand, [
+      "-c",
+      sourcePath,
+      `-I${tempDir}`,
+      "-o",
+      sourceObjectPath,
+    ], { timeout: 120_000 });
+
+    await execFile(compilerCommand, [
+      "-c",
+      manifestSourcePath,
+      "-std=c++17",
+      `-I${tempDir}`,
+      `-I${runtimeIncludeDir}`,
+      "-o",
+      manifestObjectPath,
+    ], { timeout: 120_000 });
+
+    await execFile(compilerCommand, [
+      "-c",
+      invokeSourcePath,
+      "-std=c++17",
+      `-I${tempDir}`,
+      `-I${runtimeIncludeDir}`,
+      "-o",
+      invokeObjectPath,
+    ], { timeout: 120_000 });
+
+    await execFile(compilerCommand, [
+      sourceObjectPath,
+      manifestObjectPath,
+      invokeObjectPath,
+      ...args,
+      "-o",
+      resolvedOutputPath,
+    ], { timeout: 120_000 });
+  } catch (error) {
+    error.message =
+      `Compilation failed with ${compilerCommand}: ` +
+      (error.stderr || error.message);
+    throw error;
+  }
+
+  const wasmBytes = await readFile(resolvedOutputPath);
+  return { wasmBytes, outputPath: resolvedOutputPath, tempDir };
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
 export async function compileModuleFromSource(options = {}) {
   const manifest = options.manifest ?? {};
   const sourceCode = String(options.sourceCode ?? "");
@@ -65,65 +347,87 @@ export async function compileModuleFromSource(options = {}) {
   }
 
   const compiler = selectCompiler(options.language);
+  const invokeSurfaces = resolveInvokeSurfaces(manifest);
+  const includeCommandMain = invokeSurfaces.includes(InvokeSurface.COMMAND);
   const { manifest: embeddedManifest, warnings } = toEmbeddedPluginManifest(
     manifest,
   );
-  const tempDir = await mkdtemp(
-    path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
-  );
-  const sourcePath = path.join(tempDir, `module.${compiler.extension}`);
-  const manifestSourcePath = path.join(tempDir, "plugin-manifest-exports.c");
-  const outputPath = path.resolve(options.outputPath ?? path.join(tempDir, "module.wasm"));
-
-  await writeFile(sourcePath, sourceCode, "utf8");
-  await writeFile(
-    manifestSourcePath,
-    generateEmbeddedManifestSource({ manifest: embeddedManifest }),
-    "utf8",
-  );
+  const manifestSource = generateEmbeddedManifestSource({
+    manifest: embeddedManifest,
+  });
+  const invokeHeaderSource = generateInvokeSupportHeader();
+  const invokeSource = generateInvokeSupportSource({
+    manifest,
+    includeCommandMain,
+  });
 
   const exportedSymbols = [
     "plugin_get_manifest_flatbuffer",
     "plugin_get_manifest_flatbuffer_size",
+    DefaultInvokeExports.invokeSymbol,
+    DefaultInvokeExports.allocSymbol,
+    DefaultInvokeExports.freeSymbol,
+    ...(includeCommandMain ? [DefaultInvokeExports.commandSymbol] : []),
     ...new Set(
       (Array.isArray(manifest.methods) ? manifest.methods : [])
         .map((method) => String(method?.methodId ?? "").trim())
         .filter(Boolean),
     ),
   ];
-  const linkerExports = exportedSymbols.flatMap((symbol) => [
-    "-Wl,--export=" + symbol,
-  ]);
 
+  let wasmBytes;
+  let resolvedOutputPath = null;
+  let tempDir = null;
+  const compileOptions = {
+    ...options,
+    noEntry: includeCommandMain !== true,
+  };
+  let compilerBackend = "em++ (emception)";
+  let result;
   try {
-    await execFile(compiler.command, [
-      sourcePath,
-      manifestSourcePath,
-      "-O2",
-      "--no-entry",
-      "-s",
-      "STANDALONE_WASM=1",
-      ...linkerExports,
-      "-o",
-      outputPath,
-    ]);
+    result = await compileWithEmception({
+      sourceCompilerCommand: compiler.command,
+      sourceExtension: compiler.extension,
+      sourceCode,
+      manifestSource,
+      invokeHeaderSource,
+      invokeSource,
+      exportedSymbols,
+      outputPath: options.outputPath,
+      compileOptions,
+    });
   } catch (error) {
-    error.message =
-      `Compilation failed with ${compiler.command}: ` +
-      (error.stderr || error.message);
-    throw error;
+    if (error?.code !== "EMCEPTION_LOAD_FAILED") {
+      throw error;
+    }
+    result = await compileWithSystemToolchain({
+      compilerCommand: "em++",
+      sourceCompilerCommand: compiler.command,
+      sourceExtension: compiler.extension,
+      sourceCode,
+      manifestSource,
+      invokeHeaderSource,
+      invokeSource,
+      exportedSymbols,
+      outputPath: options.outputPath,
+      compileOptions,
+    });
+    compilerBackend = "em++ (system)";
   }
+  wasmBytes = result.wasmBytes;
+  resolvedOutputPath = result.outputPath;
+  tempDir = result.tempDir;
 
-  const wasmBytes = await readFile(outputPath);
+  // Validate the compiled artifact
   const report = await validateArtifactWithStandards({
     manifest,
-    wasmPath: outputPath,
+    wasmPath: resolvedOutputPath,
   });
 
   return {
-    compiler: compiler.command,
+    compiler: compilerBackend,
     language: compiler.language,
-    outputPath,
+    outputPath: resolvedOutputPath,
     tempDir,
     wasmBytes,
     manifestWarnings: warnings,
@@ -217,6 +521,17 @@ export async function protectModuleArtifact(options = {}) {
     });
   }
 
+  let singleFileBundle = null;
+  if (options.singleFileBundle === true) {
+    singleFileBundle = await createSingleFileBundle({
+      wasmBytes,
+      manifest,
+      authorization: signedAuthorization,
+      transportEnvelope: encryptedEnvelope,
+      entries: options.bundleEntries,
+    });
+  }
+
   return {
     mnemonic: identity.mnemonic,
     signingPublicKeyHex: bytesToHex(identity.signingKey.publicKey),
@@ -224,6 +539,8 @@ export async function protectModuleArtifact(options = {}) {
     payload,
     encrypted: Boolean(encryptedEnvelope),
     encryptedEnvelope,
+    singleFileBundle,
+    bundledWasmBytes: singleFileBundle?.wasmBytes ?? null,
   };
 }
 
@@ -234,4 +551,3 @@ export async function createRecipientKeypairHex() {
     privateKeyHex: bytesToHex(keypair.privateKey),
   };
 }
-

package/src/compiler/emceptionNode.js

@@ -0,0 +1,217 @@
+import os from "node:os";
+import path from "node:path";
+import { cp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+
+const require = createRequire(import.meta.url);
+const PATCH_VERSION = "space-data-module-sdk-emception-node-v1";
+const PATCH_MARKER_FILENAME = ".space-data-module-sdk-emception-patch";
+const EMCEPTION_PATCH_ROOT = path.join(
+  os.tmpdir(),
+  `space-data-module-sdk-emception-node-${process.pid}`,
+);
+const FILE_URL_FETCH_PATCH_FLAG =
+  "__spaceDataModuleSdkFileUrlFetchPatched";
+
+let patchedEmceptionRootPromise = null;
+let emceptionInstancePromise = null;
+let emceptionExecutionQueue = Promise.resolve();
+
+function installNodeRuntimeShims() {
+  if (typeof globalThis.require !== "function") {
+    globalThis.require = require;
+  }
+
+  if (!globalThis.XMLHttpRequest) {
+    class FileUrlXMLHttpRequest {
+      open(method, url, async = true) {
+        this.method = method;
+        this.url = url;
+        this.async = async;
+        this.status = 0;
+        this.response = null;
+      }
+
+      overrideMimeType() {}
+
+      send() {
+        if (this.async) {
+          throw new Error("Async file XMLHttpRequest is not supported.");
+        }
+        const target =
+          typeof this.url === "string" && this.url.startsWith("file:")
+            ? new URL(this.url)
+            : this.url;
+        const data = readFileSync(target);
+        this.status = 200;
+        this.response = data.buffer.slice(
+          data.byteOffset,
+          data.byteOffset + data.byteLength,
+        );
+      }
+    }
+
+    globalThis.XMLHttpRequest = FileUrlXMLHttpRequest;
+  }
+
+  if (!globalThis[FILE_URL_FETCH_PATCH_FLAG]) {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async (input, init) => {
+      const url =
+        typeof input === "string"
+          ? input
+          : input?.url ?? String(input);
+      if (!url.startsWith("file:")) {
+        if (typeof originalFetch !== "function") {
+          throw new Error("fetch is not available for non-file URLs.");
+        }
+        return originalFetch(input, init);
+      }
+      const bytes = await readFile(new URL(url));
+      return {
+        ok: true,
+        status: 200,
+        url,
+        async arrayBuffer() {
+          return bytes.buffer.slice(
+            bytes.byteOffset,
+            bytes.byteOffset + bytes.byteLength,
+          );
+        },
+      };
+    };
+    globalThis[FILE_URL_FETCH_PATCH_FLAG] = true;
+  }
+}
+
+async function patchEmceptionModuleTree(rootDir) {
+  const entries = await readdir(rootDir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      await patchEmceptionModuleTree(fullPath);
+      continue;
+    }
+    if (!entry.name.endsWith(".mjs")) {
+      continue;
+    }
+
+    let source = await readFile(fullPath, "utf8");
+    source = source.replaceAll(
+      'scriptDirectory=__dirname+"/"',
+      'scriptDirectory=(new URL(".", import.meta.url)).pathname',
+    );
+
+    if (entry.name === "emception.mjs") {
+      source = source.replace(
+        "this.#fs = await new FileSystem();",
+        [
+          "this.#fs = await new FileSystem({",
+          "      locateFile: (file, scriptDirectory) => scriptDirectory + file,",
+          '      cache: "/tmp/emception-cache",',
+          "    });",
+        ].join("\n"),
+      );
+      source = source.replace(
+        "const config = {",
+        [
+          "const config = {",
+          "      locateFile: (file, scriptDirectory) => scriptDirectory + file,",
+        ].join("\n"),
+      );
+    }
+
+    if (entry.name === "FileSystem.mjs") {
+      source = source.replace(
+        [
+          "        if (!this.exists(cache)) {",
+          "            this.persist(cache);",
+          "        }",
+          "        await this.pull();",
+        ].join("\n"),
+        [
+          '        if (typeof indexedDB !== "undefined") {',
+          "            if (!this.exists(cache)) {",
+          "                this.persist(cache);",
+          "            }",
+          "            await this.pull();",
+          "        }",
+        ].join("\n"),
+      );
+    }
+
+    await writeFile(fullPath, source, "utf8");
+  }
+}
+
+async function preparePatchedEmceptionRoot() {
+  if (!patchedEmceptionRootPromise) {
+    patchedEmceptionRootPromise = (async () => {
+      const sourceRoot = path.dirname(require.resolve("sdn-emception"));
+      const markerPath = path.join(EMCEPTION_PATCH_ROOT, PATCH_MARKER_FILENAME);
+      let marker = null;
+      try {
+        marker = await readFile(markerPath, "utf8");
+      } catch {
+        marker = null;
+      }
+
+      if (marker?.trim() !== PATCH_VERSION) {
+        await rm(EMCEPTION_PATCH_ROOT, { recursive: true, force: true });
+        await cp(sourceRoot, EMCEPTION_PATCH_ROOT, { recursive: true });
+        await patchEmceptionModuleTree(EMCEPTION_PATCH_ROOT);
+        await writeFile(markerPath, `${PATCH_VERSION}\n`, "utf8");
+      }
+
+      return EMCEPTION_PATCH_ROOT;
+    })().catch((error) => {
+      patchedEmceptionRootPromise = null;
+      throw error;
+    });
+  }
+
+  return patchedEmceptionRootPromise;
+}
+
+export async function loadEmception() {
+  if (!emceptionInstancePromise) {
+    emceptionInstancePromise = (async () => {
+      installNodeRuntimeShims();
+      const patchedRoot = await preparePatchedEmceptionRoot();
+      const moduleUrl = pathToFileURL(path.join(patchedRoot, "emception.mjs")).href;
+      const { default: Emception } = await import(moduleUrl);
+      const emception = new Emception({
+        baseUrl: pathToFileURL(`${patchedRoot}${path.sep}`).href,
+      });
+      await emception.init();
+      return emception;
+    })().catch((error) => {
+      emceptionInstancePromise = null;
+      throw error;
+    });
+  }
+
+  return emceptionInstancePromise;
+}
+
+export async function runWithEmceptionLock(task) {
+  const previous = emceptionExecutionQueue;
+  let release = () => {};
+  emceptionExecutionQueue = new Promise((resolve) => {
+    release = resolve;
+  });
+  await previous.catch(() => {});
+  try {
+    const emception = await loadEmception().catch((error) => {
+      if (!error.code) {
+        error.code = "EMCEPTION_LOAD_FAILED";
+      }
+      throw error;
+    });
+    return await task(emception);
+  } finally {
+    release();
+  }
+}