space-data-module-sdk 0.8.4 → 0.8.5

package/src/testing/streamInvokeCodec.js

@@ -1,175 +1,10 @@
-import * as flatbuffers from "flatbuffers/mjs/flatbuffers.js";
-
-import { DrainPolicy } from "../generated/orbpro/manifest/drain-policy.js";
-import { TypedArenaBuffer } from "../generated/orbpro/stream/typed-arena-buffer.js";
-
-function toByteBuffer(data) {
-  if (data instanceof flatbuffers.ByteBuffer) {
-    return data;
-  }
-  return new flatbuffers.ByteBuffer(data);
-}
-
-class StreamInvokeRequest {
-  constructor() {
-    this.bb = null;
-    this.bb_pos = 0;
-  }
-
-  __init(i, bb) {
-    this.bb_pos = i;
-    this.bb = bb;
-    return this;
-  }
-
-  static getRootAsStreamInvokeRequest(bb, obj) {
-    return (obj || new StreamInvokeRequest()).__init(
-      bb.readInt32(bb.position()) + bb.position(),
-      bb,
-    );
-  }
-
-  static startStreamInvokeRequest(builder) {
-    builder.startObject(4);
-  }
-
-  static addMethodId(builder, methodIdOffset) {
-    builder.addFieldOffset(0, methodIdOffset, 0);
-  }
-
-  static addInputs(builder, inputsOffset) {
-    builder.addFieldOffset(1, inputsOffset, 0);
-  }
-
-  static createInputsVector(builder, data) {
-    builder.startVector(4, data.length, 4);
-    for (let i = data.length - 1; i >= 0; i -= 1) {
-      builder.addOffset(data[i]);
-    }
-    return builder.endVector();
-  }
-
-  static addOutputStreamCap(builder, outputStreamCap) {
-    builder.addFieldInt32(2, outputStreamCap, 0);
-  }
-
-  static addDrainPolicy(builder, drainPolicy) {
-    builder.addFieldInt8(3, drainPolicy, DrainPolicy.DRAIN_UNTIL_YIELD);
-  }
-
-  static endStreamInvokeRequest(builder) {
-    const offset = builder.endObject();
-    builder.requiredField(offset, 4);
-    return offset;
-  }
-}
-
-class StreamInvokeRequestT {
-  constructor(
-    methodId = null,
-    inputs = [],
-    outputStreamCap = 0,
-    drainPolicy = DrainPolicy.DRAIN_UNTIL_YIELD,
-  ) {
-    this.methodId = methodId;
-    this.inputs = inputs;
-    this.outputStreamCap = outputStreamCap;
-    this.drainPolicy = drainPolicy;
-  }
-
-  pack(builder) {
-    const methodId =
-      this.methodId !== null ? builder.createString(this.methodId) : 0;
-    const inputs = StreamInvokeRequest.createInputsVector(
-      builder,
-      builder.createObjectOffsetList(this.inputs),
-    );
-    StreamInvokeRequest.startStreamInvokeRequest(builder);
-    StreamInvokeRequest.addMethodId(builder, methodId);
-    StreamInvokeRequest.addInputs(builder, inputs);
-    StreamInvokeRequest.addOutputStreamCap(builder, this.outputStreamCap);
-    StreamInvokeRequest.addDrainPolicy(builder, this.drainPolicy);
-    return StreamInvokeRequest.endStreamInvokeRequest(builder);
-  }
-}
-
-class StreamInvokeResponse {
-  constructor() {
-    this.bb = null;
-    this.bb_pos = 0;
-  }
-
-  __init(i, bb) {
-    this.bb_pos = i;
-    this.bb = bb;
-    return this;
-  }
-
-  static getRootAsStreamInvokeResponse(bb, obj) {
-    return (obj || new StreamInvokeResponse()).__init(
-      bb.readInt32(bb.position()) + bb.position(),
-      bb,
-    );
-  }
-
-  outputs(index, obj) {
-    const offset = this.bb.__offset(this.bb_pos, 4);
-    return offset
-      ? (obj || new TypedArenaBuffer()).__init(
-          this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4),
-          this.bb,
-        )
-      : null;
-  }
-
-  outputsLength() {
-    const offset = this.bb.__offset(this.bb_pos, 4);
-    return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
-  }
-
-  backlogRemaining() {
-    const offset = this.bb.__offset(this.bb_pos, 6);
-    return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
-  }
-
-  yielded() {
-    const offset = this.bb.__offset(this.bb_pos, 8);
-    return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false;
-  }
-
-  errorCode() {
-    const offset = this.bb.__offset(this.bb_pos, 10);
-    return offset ? this.bb.readInt32(this.bb_pos + offset) : 0;
-  }
-
-  errorMessage(optionalEncoding) {
-    const offset = this.bb.__offset(this.bb_pos, 12);
-    return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
-  }
-
-  unpack() {
-    return {
-      outputs: this.bb.createObjList(this.outputs.bind(this), this.outputsLength()),
-      backlogRemaining: this.backlogRemaining(),
-      yielded: this.yielded(),
-      errorCode: this.errorCode(),
-      errorMessage: this.errorMessage(),
-    };
-  }
-}
-
-export function encodeStreamInvokeRequest(request) {
-  const normalized =
-    request instanceof StreamInvokeRequestT
-      ? request
-      : Object.assign(new StreamInvokeRequestT(), request);
-  const builder = new flatbuffers.Builder(1024);
-  builder.finish(normalized.pack(builder));
-  return builder.asUint8Array();
-}
-
-export function decodeStreamInvokeResponse(data) {
-  return StreamInvokeResponse.getRootAsStreamInvokeResponse(
-    toByteBuffer(data),
-  ).unpack();
-}
+// Subsumed by the generated-schema flow codecs (WS12.2): the hand-rolled
+// StreamInvoke flatbuffer classes that used to live here duplicated the
+// generated orbpro/plugin StreamInvokeRequest/Response tables. Kept as a thin
+// re-export for any deep importers.
+export {
+  encodeStreamInvokeRequest,
+  decodeStreamInvokeResponse,
+  decodeStreamInvokeRequest,
+  encodeStreamInvokeResponse,
+} from "../flow/flowCodec.js";

package/src/testing/wasmedgePin.json

@@ -0,0 +1,6 @@
+{
+  "//": "THE single source of truth for the WasmEdge runtime pin. The native lane (host-embedded / local wasmedge binary) and the Docker parity lane both read THIS version and must match it exactly; the parity harness fails loudly on drift. The Docker image tag is DERIVED from this version (never written anywhere else), so the host/container pin pair bumps together by editing one string. Keep in sync with the node host's verified runtime (space-data-network scripts/install-wasmedge.sh + build-static-wasmedge.sh, currently 0.16.4 per loop C.9: fixes libwasmedge atomic-wait defects proven broken on 0.14.x).",
+  "wasmedgeVersion": "0.16.4",
+  "dockerImageRepository": "space-data-module-sdk/parity-wasmedge",
+  "dockerfile": "docker/wasmedge-parity.Dockerfile"
+}

package/src/testing/workerModuleHarness.js

@@ -0,0 +1,242 @@
+/**
+ * Worker module harness — the async in-WASM host bridge (WS6.1).
+ *
+ * Runs a guest WASM module inside a Worker so its synchronous
+ * `space_data_module_host.call` hostcalls can block (Atomics.wait on a
+ * SharedArrayBuffer channel) while async host capabilities — http, ipfs,
+ * storage, pubsub, timers, filesystem — resolve on this (controlling)
+ * thread. The plain browser harness runs guests on the main thread, where
+ * blocking is impossible, so async operations throw
+ * `... is not available in the synchronous hostcall ABI`.
+ *
+ *   const harness = await createWorkerModuleHarness({ wasmSource, host });
+ *   await harness.callExport("guest_fn");     // guest may hostcall http.request
+ *   await harness.invoke({ methodId, ... });  // full invoke surface, proxied
+ *   await harness.destroy();
+ *
+ * Requirements: SharedArrayBuffer + Worker. In browsers the page must be
+ * cross-origin isolated (COOP+COEP). In Node, worker_threads is used.
+ */
+
+import { createBrowserHost } from "../host/browserHost.js";
+import { createAsyncHostDispatcher } from "../host/abi.js";
+import {
+  createSabHostcallBuffer,
+  createSabHostcallServer,
+  DEFAULT_SAB_HOSTCALL_RESPONSE_BYTES,
+} from "../host/sabHostcallChannel.js";
+import { WASI_THREAD_HOSTCALL_MESSAGE } from "../host/wasiThreadWorkerRuntime.js";
+
+const WORKER_URL = new URL("./workerModuleHarnessWorker.js", import.meta.url);
+
+function randomChannelToken() {
+  const bytes = new Uint8Array(16);
+  if (typeof globalThis.crypto?.getRandomValues === "function") {
+    globalThis.crypto.getRandomValues(bytes);
+  } else {
+    for (let index = 0; index < bytes.length; index += 1) {
+      bytes[index] = Math.floor(Math.random() * 256);
+    }
+  }
+  return [...bytes].map((value) => value.toString(16).padStart(2, "0")).join("");
+}
+
+function isNodeRuntime() {
+  return (
+    typeof process !== "undefined" &&
+    typeof process.versions?.node === "string" &&
+    typeof window === "undefined"
+  );
+}
+
+async function spawnWorker(workerUrl) {
+  if (isNodeRuntime()) {
+    const { Worker: NodeWorker } = await import("node:worker_threads");
+    const worker = new NodeWorker(workerUrl);
+    return {
+      post: (message) => worker.postMessage(message),
+      on: (handler) => worker.on("message", handler),
+      terminate: () => worker.terminate(),
+      // Keep the process free to exit if the caller forgets destroy().
+      unref: () => worker.unref?.(),
+    };
+  }
+  if (typeof Worker !== "function") {
+    throw new Error("Worker is unavailable; the worker module harness requires it.");
+  }
+  const worker = new Worker(workerUrl, { type: "module" });
+  return {
+    post: (message) => worker.postMessage(message),
+    on: (handler) => {
+      worker.onmessage = (event) => handler(event.data);
+    },
+    terminate: () => worker.terminate(),
+    unref: () => {},
+  };
+}
+
+function toBytes(source, label) {
+  if (source instanceof Uint8Array) return new Uint8Array(source);
+  if (source instanceof ArrayBuffer) return new Uint8Array(source.slice(0));
+  throw new TypeError(
+    `${label} must be a Uint8Array or ArrayBuffer (the worker harness ships bytes to the worker).`,
+  );
+}
+
+/**
+ * @param {object} options
+ * @param {Uint8Array|ArrayBuffer} options.wasmSource module bytes
+ * @param {object} [options.host] host servicing guest hostcalls (default createBrowserHost(hostOptions))
+ * @param {object} [options.hostOptions] options for the default host
+ * @param {(operation: string, params: any) => Promise<any>} [options.dispatchHost] dispatch override
+ * @param {number} [options.maxHostcallResponseBytes] SAB channel capacity (default 4 MiB)
+ * @param {number} [options.hostcallTimeoutMs] worker-side Atomics.wait timeout (default 120s)
+ * @param {URL|string} [options.workerUrl] worker entry override (bundlers / served pages)
+ * @param {object} [options.harnessOptions] structured-clonable options forwarded to the in-worker
+ *   createBrowserModuleHarness (surface, args, env, allowRawInvoke, logOutput, ...)
+ */
+export async function createWorkerModuleHarness(options = {}) {
+  const wasmBytes = toBytes(options.wasmSource, "wasmSource");
+  const host = options.host ?? createBrowserHost(options.hostOptions);
+  const dispatch = options.dispatchHost ?? createAsyncHostDispatcher(host);
+  const buffer = createSabHostcallBuffer({
+    maxResponseBytes: options.maxHostcallResponseBytes,
+  });
+  const server = createSabHostcallServer({ buffer, dispatch });
+  if (typeof BroadcastChannel !== "function") {
+    throw new Error(
+      "BroadcastChannel is unavailable; nested pthread hostcalls require it.",
+    );
+  }
+  const threadHostcallToken = randomChannelToken();
+  const threadHostcallChannelName =
+    `sdm.thread-hostcall.${threadHostcallToken}.${randomChannelToken()}`;
+  const threadHostcallChannel = new BroadcastChannel(threadHostcallChannelName);
+  const threadHostcallServers = new WeakMap();
+  threadHostcallChannel.onmessage = (event) => {
+    const message = event.data ?? {};
+    if (
+      message.type !== WASI_THREAD_HOSTCALL_MESSAGE ||
+      message.token !== threadHostcallToken ||
+      !(message.buffer instanceof SharedArrayBuffer)
+    ) {
+      return;
+    }
+    let nestedServer = threadHostcallServers.get(message.buffer);
+    if (!nestedServer) {
+      nestedServer = createSabHostcallServer({
+        buffer: message.buffer,
+        dispatch,
+      });
+      threadHostcallServers.set(message.buffer, nestedServer);
+    }
+    void nestedServer.handleRequest(message);
+  };
+  const port = await spawnWorker(options.workerUrl ?? WORKER_URL);
+
+  let nextId = 1;
+  const pending = new Map();
+  let readyResolve;
+  let readyReject;
+  const ready = new Promise((resolve, reject) => {
+    readyResolve = resolve;
+    readyReject = reject;
+  });
+  let destroyed = false;
+
+  port.on((message) => {
+    if (!message || typeof message !== "object") return;
+    switch (message.type) {
+      case "worker-online":
+        port.post({
+          type: "init",
+          wasmBytes,
+          buffer,
+          hostcallTimeoutMs: options.hostcallTimeoutMs,
+          harnessOptions: {
+            ...(options.harnessOptions ?? {}),
+            enableBrowserWasiThreads:
+              options.harnessOptions?.enableBrowserWasiThreads !== false,
+            threadHostcallChannel: {
+              channelName: threadHostcallChannelName,
+              token: threadHostcallToken,
+              maxResponseBytes: Number.isInteger(
+                options.maxHostcallResponseBytes,
+              )
+                ? options.maxHostcallResponseBytes
+                : DEFAULT_SAB_HOSTCALL_RESPONSE_BYTES,
+              timeoutMs: options.hostcallTimeoutMs,
+            },
+          },
+        });
+        return;
+      case "ready":
+        readyResolve({ exports: message.exports, runtime: message.runtime });
+        return;
+      case "init-error": {
+        const error = new Error(message.error?.message ?? "Worker init failed");
+        error.name = message.error?.name ?? "Error";
+        readyReject(error);
+        return;
+      }
+      case "hostcall":
+        // Serviced asynchronously; the worker blocks on the SAB until done.
+        server.handleRequest(message);
+        return;
+      case "result": {
+        const entry = pending.get(message.id);
+        if (!entry) return;
+        pending.delete(message.id);
+        if (message.ok) {
+          entry.resolve(message.value);
+        } else {
+          const error = new Error(message.error?.message ?? "Worker command failed");
+          error.name = message.error?.name ?? "Error";
+          error.code = message.error?.code ?? null;
+          entry.reject(error);
+        }
+        return;
+      }
+      default:
+    }
+  });
+
+  function send(command) {
+    if (destroyed) {
+      return Promise.reject(new Error("Worker harness is destroyed."));
+    }
+    const id = nextId++;
+    return new Promise((resolve, reject) => {
+      pending.set(id, { resolve, reject });
+      port.post({ ...command, id });
+    });
+  }
+
+  const { exports, runtime } = await ready;
+
+  return {
+    runtime: { kind: "worker", inner: runtime },
+    host,
+    buffer,
+    exports,
+    callExport: (name, ...args) => send({ type: "callExport", name, args }),
+    invoke: (request) => send({ type: "invoke", request }),
+    invokeRaw: (requestBytes) => send({ type: "invokeRaw", requestBytes }),
+    readMemory: (ptr, length) => send({ type: "readMemory", ptr, length }),
+    readManifest: () => send({ type: "readManifest" }),
+    async destroy() {
+      if (destroyed) return;
+      destroyed = true;
+      for (const entry of pending.values()) {
+        entry.reject(new Error("Worker harness destroyed."));
+      }
+      pending.clear();
+      try {
+        await port.terminate();
+      } catch {
+        // termination is best-effort
+      }
+      threadHostcallChannel.close();
+    },
+  };
+}

package/src/testing/workerModuleHarnessWorker.js

@@ -0,0 +1,176 @@
+/**
+ * Worker entry for the async-hostcall module harness.
+ *
+ * Runs the guest WASM inside this worker via createBrowserModuleHarness with
+ * a SAB-channel hostcall dispatch: guest `space_data_module_host.call`
+ * invocations block this thread on Atomics.wait while the controlling thread
+ * services the async host operation (http/ipfs/storage/pubsub/...).
+ *
+ * Runs under both browser module Workers and Node worker_threads.
+ */
+
+import { createBrowserModuleHarness } from "./browserModuleHarness.js";
+import { createSabHostcallClientDispatch } from "../host/sabHostcallChannel.js";
+
+async function resolvePort() {
+  if (
+    typeof self !== "undefined" &&
+    typeof self.postMessage === "function" &&
+    typeof window === "undefined" &&
+    !(typeof process !== "undefined" && process?.versions?.node)
+  ) {
+    return {
+      post: (message) => self.postMessage(message),
+      on: (handler) => {
+        self.onmessage = (event) => handler(event.data);
+      },
+    };
+  }
+  const { parentPort } = await import("node:worker_threads");
+  if (!parentPort) {
+    throw new Error("workerModuleHarnessWorker must run inside a Worker.");
+  }
+  return {
+    post: (message) => parentPort.postMessage(message),
+    on: (handler) => parentPort.on("message", handler),
+  };
+}
+
+function serializeWorkerError(error) {
+  return {
+    name: error?.name ?? "Error",
+    message: error?.message ?? String(error),
+    stack: error?.stack ?? null,
+    code: error?.code ?? null,
+  };
+}
+
+function detachInvokeFrame(frame = {}) {
+  const {
+    arenaLease: _arenaLease,
+    arenaGeneration: _arenaGeneration,
+    generation: _generation,
+    ...detached
+  } = frame;
+  return {
+    ...detached,
+    payload: new Uint8Array(frame.payload ?? new Uint8Array()),
+  };
+}
+
+function detachInvokeResponse(response = {}) {
+  const outputs = (response.outputs ?? response.outputFrames ?? []).map(
+    detachInvokeFrame,
+  );
+  const {
+    arenaLease,
+    outputs: _outputs,
+    outputFrames: _outputFrames,
+    payloadArena,
+    ...detached
+  } = response;
+  try {
+    return {
+      ...detached,
+      outputs,
+      outputFrames: outputs,
+      payloadArena: new Uint8Array(payloadArena ?? new Uint8Array()),
+    };
+  } finally {
+    arenaLease?.close?.();
+  }
+}
+
+const port = await resolvePort();
+let harness = null;
+
+async function handleInit(message) {
+  const dispatch = createSabHostcallClientDispatch({
+    buffer: message.buffer,
+    timeoutMs: message.hostcallTimeoutMs,
+    postRequest: (request) => port.post({ type: "hostcall", ...request }),
+  });
+  const harnessOptions = {
+    ...(message.harnessOptions ?? {}),
+    wasmSource:
+      message.wasmBytes instanceof Uint8Array
+        ? message.wasmBytes
+        : new Uint8Array(message.wasmBytes),
+    hostcallDispatch: dispatch,
+  };
+  harness = await createBrowserModuleHarness(harnessOptions);
+  const exportNames = Object.keys(harness.instance.exports);
+  port.post({
+    type: "ready",
+    exports: exportNames,
+    runtime: harness.runtime,
+  });
+}
+
+async function handleCommand(message) {
+  if (!harness) {
+    throw new Error("Worker harness is not initialized.");
+  }
+  switch (message.type) {
+    case "callExport": {
+      const fn = harness.instance.exports[message.name];
+      if (typeof fn !== "function") {
+        throw new Error(`Guest export "${message.name}" is not a function.`);
+      }
+      return fn(...(message.args ?? []));
+    }
+    case "invoke":
+      return detachInvokeResponse(await harness.invoke(message.request));
+    case "invokeRaw":
+      return harness.invokeRaw(
+        message.requestBytes instanceof Uint8Array
+          ? message.requestBytes
+          : new Uint8Array(message.requestBytes),
+      );
+    case "readMemory": {
+      const memory = harness.memory ?? harness.instance.exports.memory;
+      if (!memory) {
+        throw new Error("Guest module exposes no memory.");
+      }
+      return new Uint8Array(memory.buffer, message.ptr, message.length).slice();
+    }
+    case "readManifest":
+      return harness.readManifest();
+    default:
+      throw new Error(`Unknown worker harness command: ${message.type}`);
+  }
+}
+
+port.on(async (message) => {
+  if (!message || typeof message !== "object") return;
+  if (message.type === "init") {
+    try {
+      await handleInit(message);
+    } catch (error) {
+      port.post({ type: "init-error", error: serializeWorkerError(error) });
+    }
+    return;
+  }
+  if (message.type === "destroy") {
+    try {
+      harness?.destroy();
+    } finally {
+      port.post({ type: "destroyed" });
+    }
+    return;
+  }
+  if (typeof message.id !== "number") return;
+  try {
+    const value = await handleCommand(message);
+    port.post({ type: "result", id: message.id, ok: true, value });
+  } catch (error) {
+    port.post({
+      type: "result",
+      id: message.id,
+      ok: false,
+      error: serializeWorkerError(error),
+    });
+  }
+});
+
+port.post({ type: "worker-online" });

package/src/transport/index.js

@@ -1,11 +1,14 @@
 export {
+  createMarketplaceContentAad,
   decryptProtectedBytes,
   decryptPublicationRecordCollection,
   decryptBytesFromEnvelope,
+  decryptMarketplaceContentKeyWrap,
   decryptJsonFromEnvelope,
   encryptBytesForRecipient,
   encryptJsonForRecipient,
   generateX25519Keypair,
+  protectMarketplaceContent,
 } from "./pki.js";
 
 export {