space-data-module-sdk 0.5.18 → 0.5.21

package/src/testing/browserModuleHarness.js

@@ -0,0 +1,275 @@
+/**
+ * Browser-side module harness.
+ *
+ * Loads the same standalone WASI .wasm artifact that WasmEdge runs,
+ * instantiating it in the browser with the WASI shim + optional sdn_host
+ * bridge. Matches the createModuleHarness() API surface.
+ *
+ * Supports two invoke paths:
+ *   1. "direct" — call plugin_invoke_stream(ptr, len, &outLen) and read
+ *      the FlatBuffer response from WASM memory.
+ *   2. "command" — call _start() with stdin piped via WASI shim, read
+ *      stdout for the response bytes.
+ */
+
+import { createBrowserWasiShim, WasiExitError } from "../host/wasiShim.js";
+import { createBrowserHost } from "../host/browserHost.js";
+import {
+  createJsonHostcallBridge,
+  createNodeHostSyncDispatcher,
+  DEFAULT_HOSTCALL_IMPORT_MODULE,
+} from "../host/abi.js";
+import {
+  DefaultInvokeExports,
+  DefaultManifestExports,
+} from "../runtime/constants.js";
+import {
+  encodePluginInvokeRequest,
+  decodePluginInvokeResponse,
+} from "../invoke/codec.js";
+
+/**
+ * Detect artifact profile from WebAssembly.Module imports.
+ * Returns "standalone" (WASI-only), "sdn-abi" (WASI + sdn_host),
+ * or "emscripten" (env.* with invoke trampolines).
+ */
+export function detectArtifactProfile(wasmModule) {
+  const imports = WebAssembly.Module.imports(wasmModule);
+  const moduleNames = new Set(imports.map((i) => i.module));
+
+  if (moduleNames.has("env")) {
+    const envImports = imports.filter((i) => i.module === "env");
+    const hasInvokeTrampolines = envImports.some((i) => i.name.startsWith("invoke_"));
+    const hasPthreads = envImports.some(
+      (i) => i.name.includes("pthread") || i.name.includes("thread"),
+    );
+    if (hasInvokeTrampolines || hasPthreads) {
+      return "emscripten";
+    }
+  }
+
+  if (moduleNames.has(DEFAULT_HOSTCALL_IMPORT_MODULE)) {
+    return "sdn-abi";
+  }
+
+  if (moduleNames.has("wasi_snapshot_preview1") || moduleNames.has("wasi_unstable")) {
+    return "standalone";
+  }
+
+  return "unknown";
+}
+
+async function compileWasmModule(source) {
+  if (source instanceof WebAssembly.Module) {
+    return source;
+  }
+  if (source instanceof Response) {
+    return WebAssembly.compileStreaming(source);
+  }
+  if (typeof source === "string") {
+    return WebAssembly.compileStreaming(fetch(source));
+  }
+  const bytes = source instanceof ArrayBuffer ? new Uint8Array(source) : source;
+  return WebAssembly.compile(bytes);
+}
+
+async function instantiateBrowserModule(options = {}) {
+  const wasi = createBrowserWasiShim({
+    args: options.args ?? [],
+    env: options.env ?? {},
+    stdinBytes: options.stdinBytes ?? new Uint8Array(),
+    logOutput: options.logOutput === true,
+    performance: options.performance,
+  });
+  const importObject = { ...wasi.imports };
+  const moduleImports = WebAssembly.Module.imports(options.wasmModule);
+  const needsHostBridge = moduleImports.some(
+    (entry) => entry.module === DEFAULT_HOSTCALL_IMPORT_MODULE,
+  );
+
+  let instance = null;
+  let bridge = null;
+  if (needsHostBridge) {
+    const dispatch = createNodeHostSyncDispatcher(options.host);
+    bridge = createJsonHostcallBridge({
+      dispatch,
+      getMemory: () => instance.exports.memory,
+    });
+    Object.assign(importObject, bridge.imports);
+  }
+
+  instance = await WebAssembly.instantiate(options.wasmModule, importObject);
+  if (instance.exports.memory) {
+    wasi.setMemory(instance.exports.memory);
+  }
+  if (instance.exports._initialize) {
+    instance.exports._initialize();
+  }
+
+  return {
+    instance,
+    bridge,
+    wasi,
+  };
+}
+
+/**
+ * Create a browser-side module harness for a standalone WASI artifact.
+ *
+ * @param {Object} options
+ * @param {Uint8Array|ArrayBuffer|Response|string} options.wasmSource
+ *   WASM bytes, ArrayBuffer, fetch Response, or URL string.
+ * @param {Object} [options.host] - BrowserHost instance (created if omitted).
+ * @param {string[]} [options.args] - WASI args passed to the module.
+ * @param {Object} [options.env] - WASI environment variables.
+ * @param {string} [options.surface] - "direct" or "command" (default: auto-detect).
+ */
+export async function createBrowserModuleHarness(options = {}) {
+  const host = options.host ?? createBrowserHost(options.hostOptions);
+  const wasmModule = await compileWasmModule(options.wasmSource);
+
+  const profile = detectArtifactProfile(wasmModule);
+  const moduleExports = WebAssembly.Module.exports(wasmModule);
+  const exportNames = new Set(moduleExports.map((e) => e.name));
+
+  const hasDirectInvoke = exportNames.has(DefaultInvokeExports.invokeSymbol);
+  const hasCommand = exportNames.has(DefaultInvokeExports.commandSymbol);
+  const surface =
+    options.surface ?? (hasDirectInvoke ? "direct" : hasCommand ? "command" : "direct");
+  if (profile === "emscripten") {
+    throw new Error(
+      "Browser harness only supports standalone WASI or sdn_host artifacts. " +
+        'Compile shared browser/WasmEdge modules with runtimeTargets: ["browser", "wasmedge"] ' +
+        'or override threadModel to "single-thread".',
+    );
+  }
+
+  const activeContext = await instantiateBrowserModule({
+    wasmModule,
+    host,
+    args: options.args,
+    env: options.env,
+    performance: options.performance ?? host?.performance,
+    logOutput: options.logOutput === true,
+  });
+  const { instance, bridge, wasi } = activeContext;
+
+  // --- Invoke helpers ---
+  function invokeDirectRaw(requestBytes) {
+    const alloc = instance.exports[DefaultInvokeExports.allocSymbol];
+    const free = instance.exports[DefaultInvokeExports.freeSymbol];
+    const invokeStream = instance.exports[DefaultInvokeExports.invokeSymbol];
+    const memory = instance.exports.memory;
+    if (
+      typeof alloc !== "function" ||
+      typeof free !== "function" ||
+      typeof invokeStream !== "function" ||
+      !memory
+    ) {
+      throw new Error(
+        "Direct browser invoke requires plugin_alloc, plugin_free, plugin_invoke_stream, and memory exports.",
+      );
+    }
+    const reqLen = requestBytes.length;
+    const reqPtr = alloc(reqLen);
+    if (!reqPtr) throw new Error("plugin_alloc returned null for request.");
+
+    new Uint8Array(memory.buffer, reqPtr, reqLen).set(requestBytes);
+
+    // Allocate space for the response length output
+    const outLenPtr = alloc(4);
+    if (!outLenPtr) throw new Error("plugin_alloc returned null for response length.");
+
+    new DataView(memory.buffer).setUint32(outLenPtr, 0, true);
+
+    const resPtr = invokeStream(reqPtr, reqLen, outLenPtr);
+    const resLen = new DataView(memory.buffer).getUint32(outLenPtr, true);
+
+    free(reqPtr, reqLen);
+    free(outLenPtr, 4);
+
+    if (!resPtr || !resLen) {
+      throw new Error("plugin_invoke_stream returned null response.");
+    }
+
+    const responseBytes = new Uint8Array(memory.buffer, resPtr, resLen).slice();
+    free(resPtr, resLen);
+    return responseBytes;
+  }
+
+  async function invokeCommandRaw(stdinBytes) {
+    const commandContext = await instantiateBrowserModule({
+      wasmModule,
+      host,
+      args: options.args,
+      env: options.env,
+      stdinBytes,
+      performance: options.performance ?? host?.performance,
+      logOutput: false,
+    });
+    try {
+      const commandExport = commandContext.instance.exports[DefaultInvokeExports.commandSymbol];
+      if (typeof commandExport !== "function") {
+        throw new Error(
+          `Command-surface browser invoke requires the ${DefaultInvokeExports.commandSymbol} export.`,
+        );
+      }
+      commandExport();
+    } catch (error) {
+      if (!(error instanceof WasiExitError) || error.code !== 0) {
+        throw error;
+      }
+    }
+    return commandContext.wasi.stdout;
+  }
+
+  // --- Public API ---
+
+  async function invokeRaw(requestBytes) {
+    if (surface === "command") {
+      return invokeCommandRaw(requestBytes);
+    }
+    return invokeDirectRaw(requestBytes);
+  }
+
+  async function invoke(request) {
+    const requestBytes = encodePluginInvokeRequest(request);
+    const responseBytes = await invokeRaw(requestBytes);
+    return decodePluginInvokeResponse(responseBytes);
+  }
+
+  function readManifest() {
+    const getBytesExport =
+      instance.exports[DefaultManifestExports.pluginBytesSymbol];
+    const getSizeExport =
+      instance.exports[DefaultManifestExports.pluginSizeSymbol];
+    if (!getBytesExport || !getSizeExport) return null;
+
+    const ptr = getBytesExport();
+    const size = getSizeExport();
+    if (!ptr || !size) return null;
+
+    return new Uint8Array(instance.exports.memory.buffer, ptr, size).slice();
+  }
+
+  function destroy() {
+    wasi.flushOutput();
+  }
+
+  return {
+    runtime: {
+      kind: "browser",
+      profile,
+      surface,
+    },
+    instance,
+    module: wasmModule,
+    host,
+    bridge,
+    wasi,
+    invoke,
+    invokeRaw,
+    readManifest,
+    destroy,
+  };
+}

package/src/testing/index.d.ts

@@ -317,6 +317,40 @@ export function createPublicationProtectionDemoSummary(options?: {
   };
 }): Promise<PublicationProtectionDemoSummary>;
 
+export interface BrowserModuleHarness {
+  runtime: {
+    kind: "browser";
+    profile: string;
+    surface: string;
+  };
+  instance: WebAssembly.Instance;
+  module: WebAssembly.Module;
+  host: unknown;
+  bridge: unknown;
+  wasi: {
+    imports: Record<string, Record<string, (...args: number[]) => number>>;
+    setMemory(mem: { buffer: ArrayBuffer | SharedArrayBuffer }): void;
+    getMemory(): { buffer: ArrayBuffer | SharedArrayBuffer } | null;
+    flushOutput(): void;
+    stdout: Uint8Array;
+    stderr: Uint8Array;
+  };
+  invokeRaw(
+    requestBytes: Uint8Array | ArrayBuffer | ArrayBufferView,
+  ): Promise<Uint8Array>;
+  invoke(request: {
+    methodId?: string | null;
+    inputs?: HarnessInputFrame[];
+  }): Promise<{
+    statusCode: number;
+    errorCode?: string | null;
+    errorMessage?: string | null;
+    outputs: HarnessInputFrame[];
+  }>;
+  readManifest(): Uint8Array | null;
+  destroy(): void;
+}
+
 export function generateManifestHarnessPlan(options: {
   manifest: PluginManifest;
   includeOptionalInputs?: boolean;
@@ -375,6 +409,22 @@ export function resolveModuleHarnessLaunchPlan(options: {
   runtime?: ModuleHarnessRuntimeDescriptor;
 } | ModuleHarnessRuntimeDescriptor): PluginInvokeProcessLaunchPlan;
 
+export function detectArtifactProfile(wasmModule: WebAssembly.Module): string;
+
+export function createBrowserModuleHarness(options?: {
+  wasmSource: Uint8Array | ArrayBuffer | string | WebAssembly.Module | unknown;
+  host?: unknown;
+  hostOptions?: unknown;
+  args?: string[];
+  env?: Record<string, string>;
+  surface?: "direct" | "command";
+  performance?: {
+    now(): number;
+    timeOrigin: number;
+  };
+  logOutput?: boolean;
+}): Promise<BrowserModuleHarness>;
+
 export function createModuleHarness(options: {
   runtime?: ModuleHarnessRuntimeDescriptor;
 } | ModuleHarnessRuntimeDescriptor): Promise<ModuleHarness>;

package/src/testing/index.js

@@ -7,6 +7,10 @@ export {
   createPublicationProtectionDemoManifest,
   createPublicationProtectionDemoSummary,
 } from "./publicationProtectionDemo.js";
+export {
+  createBrowserModuleHarness,
+  detectArtifactProfile,
+} from "./browserModuleHarness.js";
 export {
   buildWasmEdgeSpawnEnv,
   createPluginInvokeProcessClient,
@@ -22,6 +26,7 @@ export {
   createModuleHarness,
   resolveModuleHarnessLaunchPlan,
 } from "./moduleHarness.js";
+export { createModuleFlatBufferStreamPump } from "./moduleFlatbufferStreamPump.js";
 
 const CapabilitySurfaceMatrix = Object.freeze({
   logging: Object.freeze({

package/src/testing/moduleFlatbufferStreamPump.js

@@ -0,0 +1,255 @@
+import { clonePayloadTypeRef } from "../manifest/typeRefs.js";
+
+function assertNonEmptyString(value, label) {
+  const normalized = String(value ?? "").trim();
+  if (!normalized) {
+    throw new TypeError(`${label} is required.`);
+  }
+  return normalized;
+}
+
+function normalizePositiveInteger(value, fallback) {
+  const normalized = Number(value ?? fallback);
+  if (!Number.isInteger(normalized) || normalized <= 0) {
+    return fallback;
+  }
+  return normalized;
+}
+
+function toUint8Array(data) {
+  if (data instanceof Uint8Array) {
+    return data;
+  }
+  if (data instanceof ArrayBuffer) {
+    return new Uint8Array(data);
+  }
+  if (ArrayBuffer.isView(data)) {
+    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  }
+  throw new TypeError(
+    "Module FlatBuffer stream pump expects Uint8Array, ArrayBuffer, or ArrayBufferView chunks.",
+  );
+}
+
+function concatUint8Arrays(chunks) {
+  let totalLength = 0;
+  for (const chunk of chunks) {
+    totalLength += chunk.byteLength;
+  }
+  const combined = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combined.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return combined;
+}
+
+function readFrameSize(bytes, offset) {
+  return (
+    bytes[offset] |
+    (bytes[offset + 1] << 8) |
+    (bytes[offset + 2] << 16) |
+    ((bytes[offset + 3] << 24) >>> 0)
+  );
+}
+
+function readFileIdentifier(payload) {
+  if (!(payload instanceof Uint8Array) || payload.byteLength < 8) {
+    return null;
+  }
+  return String.fromCharCode(payload[4], payload[5], payload[6], payload[7]);
+}
+
+function normalizeSchemaFileId(fileIdentifier) {
+  if (typeof fileIdentifier !== "string") {
+    return null;
+  }
+  const normalized = fileIdentifier.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function resolveInvoke(options) {
+  if (typeof options.invoke === "function") {
+    return options.invoke;
+  }
+  if (options.harness && typeof options.harness.invoke === "function") {
+    return options.harness.invoke.bind(options.harness);
+  }
+  throw new TypeError(
+    "createModuleFlatBufferStreamPump requires an invoke(request) function or harness with invoke().",
+  );
+}
+
+function defaultTypeResolver(_payload, context) {
+  return {
+    schemaName: null,
+    fileIdentifier: context.rawFileIdentifier,
+    acceptsAnyFlatbuffer: true,
+  };
+}
+
+function resolveFrameTemplate(frameTemplate, payload, context) {
+  if (typeof frameTemplate === "function") {
+    const resolved = frameTemplate(payload, context);
+    return resolved && typeof resolved === "object" ? resolved : {};
+  }
+  if (frameTemplate && typeof frameTemplate === "object") {
+    return frameTemplate;
+  }
+  return {};
+}
+
+export function createModuleFlatBufferStreamPump(options = {}) {
+  const invoke = resolveInvoke(options);
+  const methodId = assertNonEmptyString(options.methodId, "methodId");
+  const portId = assertNonEmptyString(options.portId, "portId");
+  const maxFramesPerInvoke = normalizePositiveInteger(options.maxFramesPerInvoke, 1);
+  const streamId = normalizePositiveInteger(options.streamId, 1);
+  const typeResolver =
+    typeof options.typeResolver === "function"
+      ? options.typeResolver
+      : defaultTypeResolver;
+  const onResponse =
+    typeof options.onResponse === "function" ? options.onResponse : null;
+
+  const stats = {
+    bytesReceived: 0,
+    chunksReceived: 0,
+    framesDecoded: 0,
+    framesInvoked: 0,
+    invokes: 0,
+    parseErrors: 0,
+  };
+
+  let pendingBytes = new Uint8Array(0);
+  let pendingFrames = [];
+  let nextSequence = normalizePositiveInteger(options.sequenceStart, 1);
+  let lastResponse = null;
+
+  async function flushPendingFrames(isFinalBatch) {
+    if (pendingFrames.length === 0) {
+      return lastResponse;
+    }
+
+    const frames = pendingFrames.map((decodedFrame, index) => {
+      const context = {
+        rawFileIdentifier: decodedFrame.rawFileIdentifier,
+        schemaFileId: decodedFrame.schemaFileId,
+        methodId,
+        portId,
+        streamId: decodedFrame.streamId,
+        sequence: decodedFrame.sequence,
+        stats,
+      };
+      const resolvedTypeRef = clonePayloadTypeRef(
+        typeResolver(decodedFrame.payload, context) ?? defaultTypeResolver(decodedFrame.payload, context),
+      );
+      if (!resolvedTypeRef.fileIdentifier && decodedFrame.rawFileIdentifier) {
+        resolvedTypeRef.fileIdentifier = decodedFrame.rawFileIdentifier;
+      }
+      if (resolvedTypeRef.acceptsAnyFlatbuffer !== true) {
+        resolvedTypeRef.acceptsAnyFlatbuffer = false;
+      }
+      return {
+        ...resolveFrameTemplate(options.frameTemplate, decodedFrame.payload, context),
+        portId,
+        typeRef: resolvedTypeRef,
+        payload: decodedFrame.payload,
+        streamId: decodedFrame.streamId,
+        sequence: decodedFrame.sequence,
+        endOfStream:
+          isFinalBatch === true && index === pendingFrames.length - 1,
+      };
+    });
+
+    pendingFrames = [];
+    const response = await invoke({
+      methodId,
+      inputs: frames,
+    });
+    stats.invokes += 1;
+    stats.framesInvoked += frames.length;
+    lastResponse = response;
+    if (onResponse) {
+      await onResponse(response, {
+        methodId,
+        portId,
+        frames,
+        isFinalBatch: isFinalBatch === true,
+        stats,
+      });
+    }
+    return response;
+  }
+
+  async function pushBytes(data) {
+    const bytes = toUint8Array(data);
+    stats.bytesReceived += bytes.byteLength;
+    stats.chunksReceived += 1;
+
+    const combined =
+      pendingBytes.byteLength > 0 ? concatUint8Arrays([pendingBytes, bytes]) : bytes;
+
+    let offset = 0;
+    let decodedCount = 0;
+
+    while (offset + 4 <= combined.byteLength) {
+      const frameSize = readFrameSize(combined, offset);
+      if (frameSize <= 0) {
+        stats.parseErrors += 1;
+        throw new Error("Invalid FlatBuffer stream frame size.");
+      }
+      if (offset + 4 + frameSize > combined.byteLength) {
+        break;
+      }
+
+      const payload = combined.subarray(offset + 4, offset + 4 + frameSize).slice();
+      const rawFileIdentifier = readFileIdentifier(payload);
+      const schemaFileId = normalizeSchemaFileId(rawFileIdentifier);
+      if (!schemaFileId) {
+        stats.parseErrors += 1;
+        throw new Error(
+          "FlatBuffer stream frame is missing a readable file identifier.",
+        );
+      }
+
+      pendingFrames.push({
+        payload,
+        rawFileIdentifier,
+        schemaFileId,
+        streamId,
+        sequence: nextSequence,
+      });
+      nextSequence += 1;
+      stats.framesDecoded += 1;
+      decodedCount += 1;
+      offset += 4 + frameSize;
+
+      if (pendingFrames.length >= maxFramesPerInvoke) {
+        await flushPendingFrames(false);
+      }
+    }
+
+    pendingBytes =
+      offset < combined.byteLength ? combined.slice(offset) : new Uint8Array(0);
+
+    return decodedCount;
+  }
+
+  async function finish() {
+    if (pendingBytes.byteLength > 0) {
+      throw new Error("FlatBuffer stream ended with a partial frame.");
+    }
+    return flushPendingFrames(true);
+  }
+
+  return {
+    stats,
+    get lastResponse() {
+      return lastResponse;
+    },
+    pushBytes,
+    finish,
+  };
+}