space-data-module-sdk 0.8.19 → 0.8.20

package/src/testing/buildWasmEdgeRunner.js

@@ -5,6 +5,8 @@ import path from "node:path";
 import process from "node:process";
 import { promisify } from "node:util";
 import { fileURLToPath } from "node:url";
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile, copyFile, rename, rm } from "node:fs/promises";
 
 const execFileAsync = promisify(execFile);
 const __filename = fileURLToPath(import.meta.url);
@@ -56,11 +58,13 @@ function resolveGeneratedIncludeDir(requestedIncludeDir) {
   );
 }
 
-export function resolveWasmEdgeRunnerSourcePath() {
+export function resolveWasmEdgeRunnerSourcePath(options = {}) {
   return path.resolve(
     __dirname,
     "native",
-    "wasmedge_emscripten_pthread_runner.c",
+    options.runnerKind === "wasi-threads"
+      ? "wasmedge_wasi_threads_runner.c"
+      : "wasmedge_emscripten_pthread_runner.c",
   );
 }
 
@@ -127,7 +131,7 @@ export function resolveWasmEdgeRunnerBuildPlan(options = {}) {
     throw new Error("Missing runner outputPath.");
   }
 
-  const runnerSourcePath = resolveWasmEdgeRunnerSourcePath();
+  const runnerSourcePath = resolveWasmEdgeRunnerSourcePath(options);
   const wasmedgeSharedLibraryPath = path.join(
     wasmedgeLibDir,
     resolveWasmEdgeSharedLibraryFilename(),
@@ -146,6 +150,7 @@ export function resolveWasmEdgeRunnerBuildPlan(options = {}) {
       process.platform === "darwin"
         ? [
             "clang",
+            "-Wl,-headerpad_max_install_names",
             runnerSourcePath,
             "-std=c11",
             "-O2",
@@ -191,6 +196,7 @@ export async function buildWasmEdgeEmscriptenPthreadRunner(options = {}) {
     process.platform === "darwin"
       ? [
           "clang",
+          "-Wl,-headerpad_max_install_names",
           basePlan.runnerSourcePath,
           "-std=c11",
           "-O2",
@@ -242,12 +248,149 @@ export async function buildWasmEdgeEmscriptenPthreadRunner(options = {}) {
     cwd: options.cwd ?? process.cwd(),
   });
   if (process.platform === "darwin") {
+    let linkedLibrary = plan.wasmedgeSharedLibraryPath;
+    if (options.runnerKind === "wasi-threads") {
+      // DYLD_LIBRARY_PATH can override even an absolute install name by its
+      // basename. Give this exact runtime a unique name so ambient SDK paths
+      // cannot silently replace the corrected atomic-wait implementation.
+      const sha = createHash("sha256").update(await readFile(linkedLibrary)).digest("hex");
+      linkedLibrary = path.join(path.dirname(plan.outputPath), `libsdm-wasmedge-${sha}.dylib`);
+      if (!existsSync(linkedLibrary)) {
+        const stagingLibrary = `${linkedLibrary}.${process.pid}`;
+        try {
+          await copyFile(plan.wasmedgeSharedLibraryPath, stagingLibrary);
+          await rename(stagingLibrary, linkedLibrary);
+        } finally { await rm(stagingLibrary, { force: true }); }
+      }
+    }
     await execFileAsync("install_name_tool", [
       "-change",
       "@rpath/libwasmedge.0.dylib",
-      plan.wasmedgeSharedLibraryPath,
+      linkedLibrary,
       plan.outputPath,
     ]);
   }
   return plan.outputPath;
 }
+
+const WASMEDGE_THREADS_REVISION = "be85c2fbba68318f103b4a766728f6946e65abf8";
+const runtimeBuilds = new Map();
+
+// Upstream 0.16.4 can lose atomic notifications between comparison and sleep.
+// Keep its established SDN fix isolated from the owner's installed SDK/CLI.
+// An explicit include/lib pair remains supported for operator-built runtimes.
+async function prepareWasmEdgeThreadsRuntime(options) {
+  if ((options.wasmedgeIncludeDir ?? process.env.WASMEDGE_INCLUDE_DIR) &&
+      (options.wasmedgeLibDir ?? process.env.WASMEDGE_LIB_DIR)) return {};
+  const patchPath = path.join(__dirname, "native/wasmedge-0.16.4-atomic-wait.patch");
+  const digest = createHash("sha256").update(WASMEDGE_THREADS_REVISION)
+    .update(await readFile(patchPath)).update(`${process.platform}-${process.arch}`).digest("hex");
+  if (!runtimeBuilds.has(digest)) {
+    const build = (async () => {
+      const root = path.join(os.tmpdir(), "sdm-wasmedge-runtimes", digest);
+      const prefix = path.join(root, "sdk");
+      const marker = path.join(root, "complete.json");
+      const lock = `${root}.lock`;
+      await mkdir(path.dirname(root), { recursive: true, mode: 0o700 });
+      const started = Date.now();
+      let held = false;
+      while (!existsSync(marker)) {
+        try {
+          await mkdir(lock);
+          held = true;
+          await writeFile(path.join(lock, "pid"), String(process.pid));
+          break;
+        } catch (error) {
+          if (error.code !== "EEXIST") throw error;
+          try {
+            const owner = Number(await readFile(path.join(lock, "pid"), "utf8"));
+            if (owner > 0) {
+              try { process.kill(owner, 0); }
+              catch (error) { if (error.code === "ESRCH") await rm(lock, { recursive: true, force: true }); }
+            }
+          } catch { /* another builder is still acquiring its lock */ }
+          if (Date.now() - started > 900000) throw new Error("Timed out waiting for the isolated WasmEdge runtime build.");
+          await new Promise((resolve) => setTimeout(resolve, 250));
+        }
+      }
+      try {
+        if (!existsSync(marker)) {
+          await mkdir(root, { recursive: true });
+          const source = path.join(root, "source");
+          const buildDir = path.join(root, "build");
+          const run = (command, args) => execFileAsync(command, args, { timeout: 900000, maxBuffer: 16 * 1024 * 1024 });
+          if (!existsSync(path.join(source, ".git"))) {
+            await run("git", ["clone", "--depth", "1", "--branch", "0.16.4", "https://github.com/WasmEdge/WasmEdge.git", source]);
+          }
+          const revision = (await run("git", ["-C", source, "rev-parse", "HEAD"])).stdout.trim();
+          if (revision !== WASMEDGE_THREADS_REVISION) throw new Error(`Unexpected WasmEdge source revision: ${revision}`);
+          try { await run("git", ["-C", source, "apply", "--reverse", "--check", patchPath]); }
+          catch { await run("git", ["-C", source, "apply", patchPath]); }
+          await run("cmake", ["-S", source, "-B", buildDir, "-G", "Ninja",
+            "-DCMAKE_BUILD_TYPE=Release", `-DCMAKE_INSTALL_PREFIX=${prefix}`,
+            process.platform === "darwin" ? "-DCMAKE_CXX_FLAGS=-Wno-invalid-specialization" : "-DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized -Wno-error=array-bounds",
+            "-DWASMEDGE_USE_LLVM=OFF",
+            "-DWASMEDGE_BUILD_PLUGINS=OFF", "-DWASMEDGE_BUILD_TOOLS=OFF", "-DWASMEDGE_FORCE_DISABLE_LTO=ON"]);
+          await run("cmake", ["--build", buildDir, "-j", "4"]);
+          await run("cmake", ["--install", buildDir]);
+          await writeFile(marker, JSON.stringify({ revision, patchSha256: createHash("sha256").update(await readFile(patchPath)).digest("hex") }));
+        }
+      } finally {
+        if (held) await rm(lock, { recursive: true, force: true });
+      }
+      const libDir = existsSync(path.join(prefix, "lib", resolveWasmEdgeSharedLibraryFilename())) ? "lib" : "lib64";
+      return { wasmedgeIncludeDir: path.join(prefix, "include"), wasmedgeLibDir: path.join(prefix, libDir) };
+    })();
+    runtimeBuilds.set(digest, build);
+    build.catch(() => runtimeBuilds.delete(digest));
+  }
+  return runtimeBuilds.get(digest);
+}
+
+export async function buildWasmEdgeWasiThreadsRunner(options = {}) {
+  const runtime = await prepareWasmEdgeThreadsRuntime(options);
+  return buildWasmEdgeEmscriptenPthreadRunner({ ...options, ...runtime, runnerKind: "wasi-threads" });
+}
+
+const builds = new Map();
+
+// The command runner is separate from the legacy resident Emscripten host.
+// Invalidate cached binaries when either the source or linked library changes.
+export async function resolveWasmEdgeWasiThreadsRunner(options = {}) {
+  const explicit = options.wasmEdgeRunnerBinary ?? options.wasmedgeRunnerBinary ??
+    process.env.SDM_WASMEDGE_RUNNER_BINARY;
+  if (explicit) return path.resolve(String(explicit));
+  options = { ...options, ...await prepareWasmEdgeThreadsRuntime(options) };
+  const plan = resolveWasmEdgeRunnerBuildPlan({
+    ...options, runnerKind: "wasi-threads", outputPath: path.join(os.tmpdir(), "sdm-runner"),
+  });
+  const digest = createHash("sha256")
+    .update(await readFile(__filename))
+    .update(await readFile(plan.runnerSourcePath))
+    .update(await readFile(plan.wasmedgeSharedLibraryPath))
+    .update(JSON.stringify([process.platform, process.arch, plan.wasmedgeIncludeDir,
+      plan.wasmedgeSharedLibraryPath, plan.compilerCommand, plan.compilerArgs]))
+    .digest("hex");
+  if (!builds.has(digest)) {
+    const build = (async () => {
+      const dir = path.join(os.tmpdir(), "sdm-wasmedge-runners");
+      await mkdir(dir, { recursive: true, mode: 0o700 });
+      const outputPath = path.join(dir, `wasi-threads-${digest}`);
+      if (!existsSync(outputPath)) {
+        const staging = `${outputPath}.${process.pid}`;
+        try {
+          await buildWasmEdgeWasiThreadsRunner({ ...options, outputPath: staging });
+          await rename(staging, outputPath);
+        } finally {
+          await rm(staging, { force: true });
+        }
+      }
+      // This executes the linked library's version check, including on cache hits.
+      await execFileAsync(outputPath, ["--version"]);
+      return outputPath;
+    })();
+    builds.set(digest, build);
+    build.catch(() => builds.delete(digest));
+  }
+  return builds.get(digest);
+}

package/src/testing/docker/wasmedge-parity.Dockerfile

@@ -11,7 +11,7 @@ ARG WASMEDGE_VERSION
 RUN test -n "${WASMEDGE_VERSION}" || (echo "WASMEDGE_VERSION build-arg is required (injected from src/testing/wasmedgePin.json)" && exit 1)
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl ca-certificates \
+    && apt-get install -y --no-install-recommends curl ca-certificates build-essential git cmake ninja-build \
     && rm -rf /var/lib/apt/lists/*
 
 RUN set -eux; \
@@ -34,6 +34,28 @@ RUN set -eux; \
 ENV PATH="/opt/wasmedge/bin:${PATH}" \
     LD_LIBRARY_PATH="/opt/wasmedge/lib64:/opt/wasmedge/lib"
 
+COPY native/wasmedge_wasi_threads_runner.c /tmp/sdm-wasi-threads-runner.c
+COPY native/wasmedge-0.16.4-atomic-wait.patch /tmp/atomic-wait.patch
+# The ordinary CLI keeps its release library. Only the SDK thread runner uses
+# the isolated 0.16.4 atomic-wait correction, matching the native build.
+RUN git clone --depth 1 --branch 0.16.4 https://github.com/WasmEdge/WasmEdge.git /tmp/wasmedge-source \
+    && test "$(git -C /tmp/wasmedge-source rev-parse HEAD)" = be85c2fbba68318f103b4a766728f6946e65abf8 \
+    && git -C /tmp/wasmedge-source apply /tmp/atomic-wait.patch \
+    && cmake -S /tmp/wasmedge-source -B /tmp/wasmedge-build -G Ninja \
+       -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sdm-wasmedge \
+       "-DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized -Wno-error=array-bounds" \
+       -DWASMEDGE_USE_LLVM=OFF -DWASMEDGE_BUILD_PLUGINS=OFF \
+       -DWASMEDGE_BUILD_TOOLS=OFF -DWASMEDGE_FORCE_DISABLE_LTO=ON \
+    && cmake --build /tmp/wasmedge-build -j 4 \
+    && cmake --install /tmp/wasmedge-build \
+    && rm -rf /tmp/wasmedge-source /tmp/wasmedge-build /tmp/atomic-wait.patch
+RUN cc /tmp/sdm-wasi-threads-runner.c -std=c11 -O2 -pthread -Wall -Wextra -Werror \
+    -I/opt/sdm-wasmedge/include -L/opt/sdm-wasmedge/lib64 -L/opt/sdm-wasmedge/lib -lwasmedge \
+    -Wl,--disable-new-dtags,-rpath,/opt/sdm-wasmedge/lib64,-rpath,/opt/sdm-wasmedge/lib \
+    -o /opt/wasmedge/bin/sdm-wasi-threads-runner \
+    && /opt/wasmedge/bin/sdm-wasi-threads-runner --version \
+    && rm /tmp/sdm-wasi-threads-runner.c
+
 # The parity harness always runs this image with an explicit wasmedge argv
 # (flags + guest module + guest args) so the container is a pure runtime shell.
 ENTRYPOINT ["wasmedge"]

package/src/testing/isomorphicHarness.js

@@ -58,6 +58,10 @@ export async function createStandaloneHarness(runtimeKind, wasmPath, options = {
       allowRawInvoke: options.allowRawInvoke,
       initialMemoryBytes: options.initialMemoryBytes,
       maximumMemoryBytes: options.maximumMemoryBytes,
+      maxThreads: options.maxThreads,
+      enableBrowserWasiThreads: options.enableBrowserWasiThreads,
+      wasiThreadWorkerUrl: options.wasiThreadWorkerUrl,
+      wasiThreadWorkerBaseUrl: options.wasiThreadWorkerBaseUrl,
     });
   }
 
@@ -65,6 +69,7 @@ export async function createStandaloneHarness(runtimeKind, wasmPath, options = {
     return loadModule({
       wasmSource: resolvedWasmPath,
       runtimeKind: "wasmedge",
+      surface: options.surface ?? "command",
       enableThreads: options.enableThreads ?? false,
       wasmEdgeBinary: options.wasmEdgeBinary,
       wasmEdgeRunnerBinary: options.wasmEdgeRunnerBinary,

package/src/testing/native/wasmedge-0.16.4-atomic-wait.patch

@@ -0,0 +1,111 @@
+diff --git a/include/executor/engine/atomic.ipp b/include/executor/engine/atomic.ipp
+index 8568e24..8414705 100644
+--- a/include/executor/engine/atomic.ipp
++++ b/include/executor/engine/atomic.ipp
+@@ -478,40 +478,32 @@ Executor::atomicWait(Runtime::Instance::MemoryInstance &MemInst,
+   auto *AtomicObj = MemInst.getPointer<std::atomic<T> *>(Address);
+   assuming(AtomicObj);
+ 
++  // Compare, register, and sleep under the notifier's mutex. Otherwise a
++  // store+notify between the comparison and wait can be lost forever.
++  std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);
+   if (AtomicObj->load() != Expected.le()) {
+     return UINT32_C(1); // NotEqual
+   }
+-
+-  decltype(WaiterMap)::iterator WaiterIterator;
+-  {
+-    std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);
+-    WaiterIterator = WaiterMap.emplace(Address, &MemInst);
+-  }
+-
++  auto WaiterIterator = WaiterMap.emplace(Address, &MemInst);
+   cxx20::scope_exit ScopeExitHolder([&]() noexcept {
+-    std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);
+     WaiterMap.erase(WaiterIterator);
+   });
+-
+-  while (true) {
+-    std::unique_lock<decltype(WaiterIterator->second.Mutex)> Locker(
+-        WaiterIterator->second.Mutex);
+-    std::cv_status WaitResult = std::cv_status::no_timeout;
+-    if (!Until) {
+-      WaiterIterator->second.Cond.wait(Locker);
+-    } else {
+-      WaitResult = WaiterIterator->second.Cond.wait_until(Locker, *Until);
+-    }
+-    if (unlikely(StopToken.load(std::memory_order_relaxed) != 0)) {
+-      return Unexpect(ErrCode::Value::Interrupted);
+-    }
+-    if (likely(AtomicObj->load() != Expected.le())) {
+-      return UINT32_C(0); // ok
+-    }
+-    if (WaitResult == std::cv_status::timeout) {
+-      return UINT32_C(2); // Timed-out
+-    }
++  auto Ready = [&]() {
++    return WaiterIterator->second.Notified ||
++           StopToken.load(std::memory_order_relaxed) != 0;
++  };
++  bool Woken = true;
++  if (!Until) {
++    WaiterIterator->second.Cond.wait(Locker, Ready);
++  } else {
++    Woken = WaiterIterator->second.Cond.wait_until(Locker, *Until, Ready);
++  }
++  if (unlikely(StopToken.load(std::memory_order_relaxed) != 0)) {
++    return Unexpect(ErrCode::Value::Interrupted);
+   }
++  // A notification succeeds even when the memory value has not changed.
++  return Woken ? UINT32_C(0) : UINT32_C(2);
++
+ }
+ 
+ } // namespace Executor
+diff --git a/include/executor/executor.h b/include/executor/executor.h
+index fbcc6ac..6683ee9 100644
+--- a/include/executor/executor.h
++++ b/include/executor/executor.h
+@@ -28,6 +28,7 @@
+ #include "runtime/stackmgr.h"
+ #include "runtime/storemgr.h"
+ 
++#include <map>
+ #include <atomic>
+ #include <condition_variable>
+ #include <csignal>
+@@ -1103,7 +1104,7 @@ private:
+ 
+   /// Waiter struct for atomic instructions
+   struct Waiter {
+-    std::mutex Mutex;
++    bool Notified = false;
+     std::condition_variable Cond;
+     Runtime::Instance::MemoryInstance *MemInst;
+     Waiter(Runtime::Instance::MemoryInstance *Inst) noexcept : MemInst(Inst) {}
+@@ -1111,7 +1112,7 @@ private:
+   /// Waiter map mutex
+   std::mutex WaiterMapMutex;
+   /// Waiter multimap
+-  std::unordered_multimap<uint32_t, Waiter> WaiterMap;
++  std::multimap<uint32_t, Waiter> WaiterMap;
+ 
+   /// WasmEdge configuration
+   const Configure Conf;
+diff --git a/lib/executor/engine/threadInstr.cpp b/lib/executor/engine/threadInstr.cpp
+index 0d1ca70..5331e95 100644
+--- a/lib/executor/engine/threadInstr.cpp
++++ b/lib/executor/engine/threadInstr.cpp
+@@ -67,8 +67,10 @@ Executor::atomicNotify(Runtime::Instance::MemoryInstance &MemInst,
+   auto Range = WaiterMap.equal_range(Address);
+   for (auto Iterator = Range.first; Total < Count && Iterator != Range.second;
+        ++Iterator) {
+-    if (likely(&MemInst == Iterator->second.MemInst)) {
+-      Iterator->second.Cond.notify_all();
++    if (likely(&MemInst == Iterator->second.MemInst) &&
++        !Iterator->second.Notified) {
++      Iterator->second.Notified = true;
++      Iterator->second.Cond.notify_one();
+       ++Total;
+     }
+   }