space-data-module-sdk 0.5.14 → 0.5.15

package/README.md

@@ -14,11 +14,28 @@ This repository is the source of truth for module-level concerns:
 - the `sds.bundle` single-file container
 - deployment authorization plus SDS publication records (`REC`, `PNM`, `ENC`)
 - the first canonical module hostcall/import ABI surface
+- shared module-level testing/runtime harnesses, including the WasmEdge
+  process runner used by package-level validation suites
 
 <p align="center">
   <img src="docs/architecture.svg" alt="Module architecture overview" width="820" />
 </p>
 
+## Shared Harness Ownership
+
+This repo owns the generic module-side runtime harnesses used across the stack:
+
+- `createModuleHarness(...)` for process and WasmEdge-backed module invocation
+- `resolveModuleHarnessLaunchPlan(...)` for portable launch planning
+- `buildWasmEdgeEmscriptenPthreadRunner(...)` plus the shared native runner
+  source and runner-level pthread smoke test
+
+Flow-specific standalone harnessing lives in `sdn-flow`, which layers flow
+enqueue/drain behavior on top of its compiled standalone runtime host. Package-
+specific validation suites, including conjunction replay/V&V, are expected to
+sit on top of these shared harnesses instead of defining their own runtime
+process model.
+
 ## Module Artifact Model
 
 A compliant module built with this SDK is always a valid `.wasm` artifact with:
@@ -178,6 +195,11 @@ semantics.
 This repo now exposes a manifest-driven harness generator from
 `space-data-module-sdk/testing` and two complementary integration suites:
 
+- shared process-level helpers for command-surface runtimes:
+  - `createPluginInvokeProcessClient(...)`
+  - `resolveWasmEdgePluginLaunchPlan(...)`
+  - `buildWasmEdgeEmscriptenPthreadRunner(...)`
+
 - `npm run test:runtime-matrix`
   - cross-language runtime smoke across the same WASM in Node.js, Go, Python,
     Rust, Java, C#, and Swift
@@ -299,6 +321,46 @@ The reference path lives in
 Standard bundle payloads now include the optional `deployment-plan` JSON entry
 for resolved protocol installations and producer input bindings.
 
+## Publication Protection Extensions
+
+Digital signature and encrypted-delivery metadata are publication-layer
+extensions, not a second module format. The canonical module is still:
+
+- valid `.wasm`
+- optionally carrying `sds.bundle` as a wasm custom section
+- optionally carrying one appended SDS `REC` trailer after the wasm bytes
+
+That trailing `REC` FlatBuffer is the standards-backed container for
+publication records:
+
+- `PNM` is the publication notice / digital-signature record
+  - identifies the artifact
+  - carries the CID and publish timestamp
+  - carries signature metadata so a loader can verify who published the module
+- `ENC` is the encrypted-delivery record
+  - carries the X25519 / AES-CTR / HKDF parameters needed to decrypt the
+    protected transport payload
+  - does not change the canonical module ABI or manifest shape
+
+The loader contract is always:
+
+1. Start with the protected blob.
+2. Scan backward for the trailing `REC` footer.
+3. Parse `REC` using the standards-generated `REC`, `PNM`, and `ENC`
+   FlatBuffers from `spacedatastandards.org`.
+4. Resolve `PNM` for signature/publication metadata.
+5. Resolve `ENC` if the delivery was encrypted.
+6. Strip the trailer or decrypt the protected payload.
+7. Hand the remaining raw wasm bytes to the runtime.
+
+Aligned-binary payloads are a separate invoke-ABI optimization. They do not
+replace the canonical FlatBuffer schema. If a port advertises
+`wireFormat: "aligned-binary"`, it must also advertise the regular
+`wireFormat: "flatbuffer"` fallback for the same schema and file identifier in
+the same accepted type set. The publication demo in the local lab uses a
+regular `OMM.fbs` request and a `StateVector.fbs` response that advertises both
+the canonical FlatBuffer fallback and the aligned-binary layout metadata.
+
 ## Module Publication
 
 Packages that publish Space Data modules use the canonical `sdn-module`
@@ -402,10 +464,14 @@ npm run start:lab
 
 Then open `http://localhost:4318`.
 
+Use the `Run Publication Demo` button to generate a protected demo artifact and
+inspect the parsed `REC`, `PNM`, and `ENC` records produced by the real
+`spacedatastandards.org` generated message classes.
+
 ## Related Projects
 
 - [`spacedatastandards.org`](https://spacedatastandards.org)
-- [`@digitalarsenal/sdn-flow`](https://github.com/DigitalArsenal/sdn-flow)
+- [`sdn-flow`](https://github.com/DigitalArsenal/sdn-flow)
 - [`hd-wallet-wasm`](https://github.com/nicktj-dev/hd-wallet-wasm)
 
 ## Development

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "space-data-module-sdk",
-  "version": "0.5.14",
+  "version": "0.5.15",
   "description": "Module SDK for building, validating, signing, and deploying WebAssembly modules on the Space Data Network.",
   "type": "module",
   "types": "./src/index.d.ts",

package/src/compliance/pluginCompliance.js

@@ -167,134 +167,6 @@ function normalizePayloadWireFormatName(value) {
   return null;
 }
 
-function normalizedPluginFamilyName(value) {
-  if (!isNonEmptyString(value)) {
-    return null;
-  }
-  return value
-    .trim()
-    .toLowerCase()
-    .replace(/-/g, "_");
-}
-
-function typeSetHasDualWireFormats(typeSet) {
-  if (!Array.isArray(typeSet?.allowedTypes) || typeSet.allowedTypes.length === 0) {
-    return false;
-  }
-  let hasFlatbuffer = false;
-  let hasAlignedBinary = false;
-  for (const allowedType of typeSet.allowedTypes) {
-    const wireFormat = normalizePayloadWireFormatName(allowedType?.wireFormat);
-    if (wireFormat === "flatbuffer") {
-      hasFlatbuffer = true;
-    } else if (wireFormat === "aligned-binary") {
-      hasAlignedBinary = true;
-    }
-  }
-  return hasFlatbuffer && hasAlignedBinary;
-}
-
-function validateCanonicalDualFormatPort(port, issues, location, methodId, portId) {
-  if (!port) {
-    pushIssue(
-      issues,
-      "error",
-      "propagator-missing-canonical-port",
-      `Canonical propagator method "${methodId}" must declare port "${portId}".`,
-      location,
-    );
-    return;
-  }
-  if (!Array.isArray(port.acceptedTypeSets) || port.acceptedTypeSets.length === 0) {
-    pushIssue(
-      issues,
-      "error",
-      "propagator-missing-canonical-port",
-      `Canonical propagator method "${methodId}" port "${portId}" must declare acceptedTypeSets.`,
-      `${location}.acceptedTypeSets`,
-    );
-    return;
-  }
-  port.acceptedTypeSets.forEach((typeSet, index) => {
-    if (!typeSetHasDualWireFormats(typeSet)) {
-      pushIssue(
-        issues,
-        "error",
-        "canonical-port-missing-dual-wire-format",
-        `Canonical propagator method "${methodId}" port "${portId}" acceptedTypeSet "${typeSet?.setId ?? index}" must accept both regular flatbuffer and aligned-binary payloads.`,
-        `${location}.acceptedTypeSets[${index}]`,
-      );
-    }
-  });
-}
-
-function validateCanonicalPropagatorContract(manifest, issues, sourceName) {
-  if (normalizedPluginFamilyName(manifest?.pluginFamily) !== "propagator") {
-    return;
-  }
-
-  const methods = Array.isArray(manifest?.methods) ? manifest.methods : [];
-  const methodRequirements = [
-    {
-      methodId: "ingest_omm",
-      inputPorts: ["omm"],
-      outputPorts: [],
-    },
-    {
-      methodId: "describe_sources_batch",
-      inputPorts: ["request"],
-      outputPorts: ["result"],
-    },
-    {
-      methodId: "propagate_state",
-      inputPorts: ["request"],
-      outputPorts: ["state"],
-    },
-  ];
-
-  for (const requirement of methodRequirements) {
-    const methodIndex = methods.findIndex(
-      (method) => method?.methodId === requirement.methodId,
-    );
-    if (methodIndex === -1) {
-      pushIssue(
-        issues,
-        "error",
-        "propagator-missing-canonical-method",
-        `Propagator plugins must declare canonical method "${requirement.methodId}".`,
-        `${sourceName}.methods`,
-      );
-      continue;
-    }
-    const method = methods[methodIndex];
-    const location = `${sourceName}.methods[${methodIndex}]`;
-    for (const portId of requirement.inputPorts) {
-      const port = Array.isArray(method.inputPorts)
-        ? method.inputPorts.find((entry) => entry?.portId === portId)
-        : null;
-      validateCanonicalDualFormatPort(
-        port,
-        issues,
-        `${location}.inputPorts`,
-        requirement.methodId,
-        portId,
-      );
-    }
-    for (const portId of requirement.outputPorts) {
-      const port = Array.isArray(method.outputPorts)
-        ? method.outputPorts.find((entry) => entry?.portId === portId)
-        : null;
-      validateCanonicalDualFormatPort(
-        port,
-        issues,
-        `${location}.outputPorts`,
-        requirement.methodId,
-        portId,
-      );
-    }
-  }
-}
-
 function validateStringField(issues, value, location, label) {
   if (!isNonEmptyString(value)) {
     pushIssue(issues, "error", "missing-string", `${label} must be a non-empty string.`, location);
@@ -494,7 +366,7 @@ function validateAllowedType(type, issues, location) {
       "Allowed type fixedStringLength",
       { min: 0 },
     );
-    validateOptionalIntegerField(
+    validateIntegerField(
       issues,
       type.byteLength,
       `${location}.byteLength`,
@@ -1387,8 +1259,6 @@ export function validatePluginManifest(manifest, options = {}) {
       }
     });
 
-    validateCanonicalPropagatorContract(manifest, issues, sourceName);
-
     if (manifest.timers !== undefined && !Array.isArray(manifest.timers)) {
       pushIssue(
         issues,

package/src/index.d.ts

@@ -11,7 +11,7 @@ export type ProtocolRoleName = "handle" | "dial" | "both";
 
 export interface PayloadTypeRef {
   schemaName?: string;
-  fileIdentifier?: string | null;
+  fileIdentifier?: string;
   schemaHash?: string | number[] | Uint8Array;
   acceptsAnyFlatbuffer?: boolean;
   wireFormat?: PayloadWireFormat;
@@ -662,12 +662,29 @@ export type {
   HarnessInvokeScenario,
   HarnessRawScenario,
   ManifestHarnessPlan,
+  ModuleHarness,
+  ModuleHarnessRuntimeDescriptor,
+  PublicationProtectionDemoAlignedType,
+  PublicationProtectionDemoSummary,
+  PluginInvokeProcessClient,
+  PluginInvokeProcessLaunchPlan,
+  WasmEdgeRunnerBuildPlan,
 } from "./testing/index.js";
 
 export {
+  buildWasmEdgeEmscriptenPthreadRunner,
+  buildWasmEdgeSpawnEnv,
+  createPublicationProtectionDemoManifest,
+  createPublicationProtectionDemoSummary,
+  createModuleHarness,
+  createPluginInvokeProcessClient,
   describeCapabilityRuntimeSurface,
   generateManifestHarnessPlan,
   materializeHarnessScenario,
+  resolveModuleHarnessLaunchPlan,
+  resolveWasmEdgeRunnerBuildPlan,
+  resolveWasmEdgeRunnerSourcePath,
+  resolveWasmEdgePluginLaunchPlan,
   serializeHarnessPlan,
 } from "./testing/index.js";
 

package/src/invoke/codec.js

@@ -12,7 +12,7 @@ import { InvokeSurface } from "../generated/orbpro/manifest/invoke-surface.js";
 import { BufferMutability } from "../generated/orbpro/stream/buffer-mutability.js";
 import { BufferOwnership } from "../generated/orbpro/stream/buffer-ownership.js";
 import { FlatBufferTypeRefT } from "../generated/orbpro/stream/flat-buffer-type-ref.js";
-import { TypedArenaBuffer, TypedArenaBufferT } from "../generated/orbpro/stream/typed-arena-buffer.js";
+import { TypedArenaBufferT } from "../generated/orbpro/stream/typed-arena-buffer.js";
 import { toUint8Array } from "../runtime/bufferLike.js";
 
 function toByteBuffer(data) {
@@ -149,20 +149,22 @@ function normalizeArenaFrame(frame = {}, offset) {
 
 function packArenaFrames(frames = []) {
   const packedFrames = [];
-  const arena = [];
+  const normalizedFrames = [];
   let offset = 0;
   for (const frame of frames) {
     const normalized = normalizeArenaFrame(frame, offset);
-    for (let index = 0; index < normalized.padding; index += 1) {
-      arena.push(0);
-    }
-    arena.push(...normalized.payload);
     offset = normalized.buffer.offset + normalized.buffer.size;
     packedFrames.push(normalized.buffer);
+    normalizedFrames.push(normalized);
+  }
+
+  const arena = new Uint8Array(offset);
+  for (const normalized of normalizedFrames) {
+    arena.set(normalized.payload, normalized.buffer.offset);
   }
   return {
     frames: packedFrames,
-    arena: Uint8Array.from(arena),
+    arena,
   };
 }
 

package/src/manifest/typeRefs.js

@@ -1,62 +1,31 @@
 function cloneSchemaHash(value) {
   if (value instanceof Uint8Array) {
-    return value.byteLength > 0 ? new Uint8Array(value) : undefined;
+    return new Uint8Array(value);
   }
   if (Array.isArray(value)) {
-    return value.length > 0 ? [...value] : undefined;
+    return [...value];
   }
   return value ?? undefined;
 }
 
-function normalizeNullableString(value) {
-  if (value === undefined || value === null) {
-    return null;
-  }
-  const normalized = String(value).trim();
-  return normalized.length > 0 ? normalized : null;
-}
-
-function normalizeAlignedMetadataScalar(value) {
-  if (value === undefined || value === null || value === "") {
-    return undefined;
-  }
-  const numeric = Number(value);
-  if (Number.isFinite(numeric)) {
-    return numeric === 0 ? undefined : numeric;
-  }
-  return value;
-}
-
 export function clonePayloadTypeRef(value = null) {
   if (!value || typeof value !== "object") {
-    return { acceptsAnyFlatbuffer: true, fileIdentifier: null };
+    return { acceptsAnyFlatbuffer: true };
   }
   return {
-    schemaName: normalizeNullableString(
-      value.schemaName ?? value.schema_name,
-    ),
-    fileIdentifier: normalizeNullableString(
-      value.fileIdentifier ?? value.file_identifier,
-    ),
+    schemaName: value.schemaName ?? value.schema_name ?? undefined,
+    fileIdentifier: value.fileIdentifier ?? value.file_identifier ?? undefined,
     schemaHash: cloneSchemaHash(value.schemaHash ?? value.schema_hash),
     acceptsAnyFlatbuffer: Boolean(
       value.acceptsAnyFlatbuffer ?? value.accepts_any_flatbuffer ?? false,
     ),
-    wireFormat: normalizePayloadWireFormatName(
-      value.wireFormat ?? value.wire_format,
-    ),
-    rootTypeName: normalizeNullableString(
-      value.rootTypeName ?? value.root_type_name,
-    ),
-    fixedStringLength: normalizeAlignedMetadataScalar(
-      value.fixedStringLength ?? value.fixed_string_length,
-    ),
-    byteLength: normalizeAlignedMetadataScalar(
-      value.byteLength ?? value.byte_length,
-    ),
-    requiredAlignment: normalizeAlignedMetadataScalar(
-      value.requiredAlignment ?? value.required_alignment,
-    ),
+    wireFormat: value.wireFormat ?? value.wire_format ?? undefined,
+    rootTypeName: value.rootTypeName ?? value.root_type_name ?? undefined,
+    fixedStringLength:
+      value.fixedStringLength ?? value.fixed_string_length ?? undefined,
+    byteLength: value.byteLength ?? value.byte_length ?? undefined,
+    requiredAlignment:
+      value.requiredAlignment ?? value.required_alignment ?? undefined,
   };
 }
 
@@ -64,12 +33,6 @@ export function normalizePayloadWireFormatName(value) {
   if (value === undefined || value === null || value === "") {
     return null;
   }
-  if (value === 1 || value === "1") {
-    return "aligned-binary";
-  }
-  if (value === 0 || value === "0") {
-    return "flatbuffer";
-  }
   const normalized = String(value).trim().toLowerCase().replace(/_/g, "-");
   if (normalized === "aligned-binary") {
     return "aligned-binary";

package/src/testing/buildWasmEdgeRunner.js

@@ -0,0 +1,214 @@
+import { accessSync, existsSync } from "node:fs";
+import { execFile } from "node:child_process";
+import path from "node:path";
+import process from "node:process";
+import { promisify } from "node:util";
+import { fileURLToPath } from "node:url";
+
+const execFileAsync = promisify(execFile);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+function normalizeResolvedPath(value) {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    return null;
+  }
+  return path.resolve(value);
+}
+
+function ensureReadableSync(targetPath, label) {
+  try {
+    accessSync(targetPath);
+  } catch {
+    throw new Error(`${label} not found: ${targetPath}`);
+  }
+}
+
+function resolveWasmEdgeSharedLibraryFilename() {
+  if (process.platform === "darwin") {
+    return "libwasmedge.0.dylib";
+  }
+  if (process.platform === "win32") {
+    return "wasmedge.dll";
+  }
+  return "libwasmedge.so.0";
+}
+
+function resolveGeneratedIncludeDir(requestedIncludeDir) {
+  const directHeaderPath = path.join(
+    requestedIncludeDir,
+    "wasmedge",
+    "enum_configure.h",
+  );
+  if (existsSync(directHeaderPath)) {
+    return requestedIncludeDir;
+  }
+
+  const repoRoot = path.resolve(requestedIncludeDir, "..", "..");
+  const generatedDir = path.join(repoRoot, "build", "include", "api");
+  if (existsSync(path.join(generatedDir, "wasmedge", "enum_configure.h"))) {
+    return generatedDir;
+  }
+
+  throw new Error(
+    `WasmEdge include directory does not contain wasmedge/enum_configure.h: ${requestedIncludeDir}`,
+  );
+}
+
+export function resolveWasmEdgeRunnerSourcePath() {
+  return path.resolve(
+    __dirname,
+    "native",
+    "wasmedge_emscripten_pthread_runner.c",
+  );
+}
+
+export function resolveWasmEdgeRunnerBuildPlan(options = {}) {
+  const requestedIncludeDir = normalizeResolvedPath(
+    options.wasmedgeIncludeDir ?? process.env.WASMEDGE_INCLUDE_DIR,
+  );
+  const wasmedgeLibDir = normalizeResolvedPath(
+    options.wasmedgeLibDir ?? process.env.WASMEDGE_LIB_DIR,
+  );
+  const outputPath = normalizeResolvedPath(
+    options.outputPath ?? options.output,
+  );
+
+  if (!requestedIncludeDir) {
+    throw new Error(
+      "Missing WasmEdge include directory. Set wasmedgeIncludeDir, --wasmedge-include-dir, or WASMEDGE_INCLUDE_DIR.",
+    );
+  }
+  if (!wasmedgeLibDir) {
+    throw new Error(
+      "Missing WasmEdge library directory. Set wasmedgeLibDir, --wasmedge-lib-dir, or WASMEDGE_LIB_DIR.",
+    );
+  }
+  if (!outputPath) {
+    throw new Error("Missing runner outputPath.");
+  }
+
+  const runnerSourcePath = resolveWasmEdgeRunnerSourcePath();
+  const wasmedgeSharedLibraryPath = path.join(
+    wasmedgeLibDir,
+    resolveWasmEdgeSharedLibraryFilename(),
+  );
+
+  return {
+    runnerSourcePath,
+    requestedIncludeDir,
+    wasmedgeIncludeDir: requestedIncludeDir,
+    wasmedgeLibDir,
+    wasmedgeSharedLibraryPath,
+    outputPath,
+    compilerCommand:
+      process.platform === "darwin" ? "xcrun" : process.env.CC ?? "cc",
+    compilerArgs:
+      process.platform === "darwin"
+        ? [
+            "clang",
+            runnerSourcePath,
+            "-std=c11",
+            "-O2",
+            "-pthread",
+            "-Wall",
+            "-Wextra",
+            "-Werror",
+            "-Wno-unused-parameter",
+            "-Wno-error=visibility",
+            `-I${requestedIncludeDir}`,
+            `-L${wasmedgeLibDir}`,
+            "-lwasmedge",
+            `-Wl,-rpath,${wasmedgeLibDir}`,
+            "-o",
+            outputPath,
+          ]
+        : [
+            runnerSourcePath,
+            "-std=c11",
+            "-O2",
+            "-pthread",
+            "-Wall",
+            "-Wextra",
+            "-Werror",
+            "-Wno-unused-parameter",
+            "-Wno-error=visibility",
+            `-I${requestedIncludeDir}`,
+            `-L${wasmedgeLibDir}`,
+            "-lwasmedge",
+            `-Wl,-rpath,${wasmedgeLibDir}`,
+            "-o",
+            outputPath,
+          ],
+  };
+}
+
+export async function buildWasmEdgeEmscriptenPthreadRunner(options = {}) {
+  const basePlan = resolveWasmEdgeRunnerBuildPlan(options);
+  const wasmedgeIncludeDir = resolveGeneratedIncludeDir(
+    basePlan.requestedIncludeDir,
+  );
+  const compilerArgs =
+    process.platform === "darwin"
+      ? [
+          "clang",
+          basePlan.runnerSourcePath,
+          "-std=c11",
+          "-O2",
+          "-pthread",
+          "-Wall",
+          "-Wextra",
+          "-Werror",
+          "-Wno-unused-parameter",
+          "-Wno-error=visibility",
+          `-I${wasmedgeIncludeDir}`,
+          `-L${basePlan.wasmedgeLibDir}`,
+          "-lwasmedge",
+          `-Wl,-rpath,${basePlan.wasmedgeLibDir}`,
+          "-o",
+          basePlan.outputPath,
+        ]
+      : [
+          basePlan.runnerSourcePath,
+          "-std=c11",
+          "-O2",
+          "-pthread",
+          "-Wall",
+          "-Wextra",
+          "-Werror",
+          "-Wno-unused-parameter",
+          "-Wno-error=visibility",
+          `-I${wasmedgeIncludeDir}`,
+          `-L${basePlan.wasmedgeLibDir}`,
+          "-lwasmedge",
+          `-Wl,-rpath,${basePlan.wasmedgeLibDir}`,
+          "-o",
+          basePlan.outputPath,
+        ];
+  const plan = {
+    ...basePlan,
+    wasmedgeIncludeDir,
+    compilerArgs,
+  };
+
+  ensureReadableSync(plan.runnerSourcePath, "Runner source");
+  ensureReadableSync(plan.wasmedgeIncludeDir, "WasmEdge include directory");
+  ensureReadableSync(plan.wasmedgeLibDir, "WasmEdge library directory");
+  ensureReadableSync(
+    plan.wasmedgeSharedLibraryPath,
+    "WasmEdge shared library",
+  );
+
+  await execFileAsync(plan.compilerCommand, plan.compilerArgs, {
+    cwd: options.cwd ?? process.cwd(),
+  });
+  if (process.platform === "darwin") {
+    await execFileAsync("install_name_tool", [
+      "-change",
+      "@rpath/libwasmedge.0.dylib",
+      plan.wasmedgeSharedLibraryPath,
+      plan.outputPath,
+    ]);
+  }
+  return plan.outputPath;
+}