space-data-module-sdk 0.2.6 → 0.2.7

package/README.md

@@ -34,6 +34,7 @@ A module built with this SDK is a `.wasm` artifact with:
 - an exported `_start` entry when the artifact supports WASI command mode
 - optional `sds.bundle` custom-section payloads for:
   - manifest bytes
+  - resolved deployment plans and input bindings
   - deployment authorization
   - detached signatures
   - encrypted transport envelopes
@@ -81,6 +82,14 @@ outputs. The reference invoke examples live in
 - `manifest.hybrid.json`
 - `module.c`
 
+Input and output ports can independently declare regular `flatbuffer` payloads
+or `aligned-binary` layouts. Mixed contracts are valid. When a port advertises
+an `aligned-binary` layout, it must also advertise a regular `flatbuffer`
+fallback for the same schema in the same accepted type set. A module can accept
+a regular `OMM.fbs` request and emit an aligned-binary `StateVector.fbs`
+response, provided the output port also declares the regular `StateVector.fbs`
+fallback and the aligned type ref carries the correct layout metadata.
+
 ## Runtime Portability
 
 The module format is language-neutral. A host can load modules from this SDK
@@ -115,6 +124,25 @@ This repo currently includes:
 - a reference Node host and sync `sdn_host` bridge for the first hostcall
   surface
 
+## Testing
+
+This repo now exposes a manifest-driven harness generator from
+`space-data-module-sdk/testing` and two complementary integration suites:
+
+- `npm run test:runtime-matrix`
+  - cross-language runtime smoke across the same WASM in Node.js, Go, Python,
+    Rust, Java, C#, and Swift
+  - covers method calling, aligned-binary envelope metadata preservation,
+    stdin/stdout/stderr, args, env, preopened filesystem access, and basic WASI
+    clock/time smoke
+- `npm run test:host-surfaces`
+  - authoritative Node-host coverage for HTTP, TCP, UDP, TLS, WebSocket, MQTT,
+    process execution, timers, filesystem, and the sync `sdn_host` ABI
+
+The detailed edge cases and the current WASI-vs-host portability boundary are
+documented in
+[`docs/testing-harness.md`](./docs/testing-harness.md).
+
 ## Install
 
 ```bash
@@ -161,8 +189,36 @@ import { createDeploymentAuthorization } from "space-data-module-sdk/auth";
 import { encryptJsonForRecipient } from "space-data-module-sdk/transport";
 import { compileModuleFromSource } from "space-data-module-sdk/compiler";
 import { createSingleFileBundle } from "space-data-module-sdk/bundle";
+import { validateDeploymentPlan } from "space-data-module-sdk/deployment";
+import { generateManifestHarnessPlan } from "space-data-module-sdk/testing";
 ```
 
+## Protocol Installation
+
+Modules can declare hosted protocol contracts in `manifest.protocols`.
+
+Those declarations are for stable artifact identity:
+
+- `wireId`
+- `transportKind`
+- `role`
+- `specUri`
+- hosting hints like `defaultPort` and `requireSecureTransport`
+
+Concrete multiaddrs, peer IDs, and producer routing do not belong in the
+canonical manifest. They belong in deployment metadata attached to the final
+package or bundle.
+
+This repo exposes that deployment surface from
+`space-data-module-sdk/deployment`. Use it to:
+
+- validate resolved protocol installations
+- describe input bindings from producers to module ports
+- attach a deployment plan to `sds.bundle`
+
+The full contract split is documented in
+[`docs/protocol-installation.md`](./docs/protocol-installation.md).
+
 ## Single-File Bundles
 
 `sds.bundle` keeps module delivery to one file without changing WebAssembly
@@ -179,6 +235,9 @@ The reference path lives in
 - the `go` and `python` directories show non-JS readers against the same
   bundle contract
 
+Standard bundle payloads now include the optional `deployment-plan` JSON entry
+for resolved protocol installations and producer input bindings.
+
 ## Module Publication
 
 Packages that publish SDN modules now use the canonical `sdn-module`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "space-data-module-sdk",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "Module SDK for building, validating, signing, and deploying WebAssembly modules on the Space Data Network.",
   "type": "module",
   "types": "./src/index.d.ts",
@@ -20,8 +20,10 @@
     "./compiler": "./src/compiler/index.js",
     "./compiler/emception": "./src/compiler/emception.js",
     "./bundle": "./src/bundle/index.js",
+    "./deployment": "./src/deployment/index.js",
     "./invoke": "./src/invoke/index.js",
     "./runtime": "./src/runtime/index.js",
+    "./testing": "./src/testing/index.js",
     "./standards": "./src/standards/index.js",
     "./schemas/*": "./schemas/*"
   },
@@ -32,6 +34,8 @@
   ],
   "scripts": {
     "test": "node --test",
+    "test:host-surfaces": "node --test test/node-host.test.js test/host-abi.test.js",
+    "test:runtime-matrix": "SPACE_DATA_MODULE_SDK_ENABLE_RUNTIME_MATRIX=1 node --test test/runtime-matrix.test.js",
     "start:lab": "node ./lab/server.js",
     "check:compliance": "node ./bin/space-data-module.js check --repo-root .",
     "generate:vectors": "node ./examples/single-file-bundle/generate-vectors.mjs",

package/schemas/PluginManifest.fbs

@@ -134,6 +134,15 @@ table ProtocolSpec {
   input_port_id: string;
   output_port_id: string;
   description: string;
+  wire_id: string;
+  transport_kind: string;
+  role: string;
+  spec_uri: string;
+  auto_install: bool = true;
+  advertise: bool = false;
+  discovery_key: string;
+  default_port: uint16;
+  require_secure_transport: bool = false;
 }
 
 /// Build artifact emitted by the plugin toolchain.

package/src/bundle/constants.js

@@ -5,4 +5,7 @@ export const DEFAULT_HASH_ALGORITHM = "sha256";
 export const DEFAULT_MANIFEST_EXPORT_SYMBOL = "plugin_get_manifest_flatbuffer";
 export const DEFAULT_MANIFEST_SIZE_SYMBOL =
   "plugin_get_manifest_flatbuffer_size";
-
+export const SDS_DEPLOYMENT_SECTION_NAME = "sds.deployment";
+export const SDS_DEPLOYMENT_ENTRY_ID = "deployment-plan";
+export const SDS_DEPLOYMENT_MEDIA_TYPE =
+  "application/vnd.space-data.module.deployment+json";

package/src/bundle/wasm.js

@@ -3,11 +3,14 @@ import { decodePluginManifest, encodePluginManifest } from "../manifest/index.js
 import { toUint8Array } from "../runtime/bufferLike.js";
 import { sha256Bytes } from "../utils/crypto.js";
 import { bytesToHex } from "../utils/encoding.js";
+import { createDeploymentPlanBundleEntry } from "../deployment/index.js";
 import {
   DEFAULT_MANIFEST_EXPORT_SYMBOL,
   DEFAULT_MANIFEST_SIZE_SYMBOL,
   SDS_BUNDLE_SECTION_NAME,
   SDS_CUSTOM_SECTION_PREFIX,
+  SDS_DEPLOYMENT_ENTRY_ID,
+  SDS_DEPLOYMENT_SECTION_NAME,
 } from "./constants.js";
 import {
   decodeModuleBundle,
@@ -268,6 +271,9 @@ function normalizeStandardEntries(options = {}) {
       description: "Transport envelope metadata.",
     });
   }
+  if (options.deploymentPlan !== undefined) {
+    entries.push(createDeploymentPlanBundleEntry(options.deploymentPlan));
+  }
   return entries.filter((entry) => entry.payload !== null);
 }
 
@@ -282,7 +288,12 @@ function normalizeAdditionalEntries(entries = []) {
 }
 
 async function withSha256(entry) {
-  const payloadBytes = normalizeBytes(entry.payload, `entry "${entry.entryId}" payload`);
+  const payloadBytes =
+    toUint8Array(entry.payload) ??
+    ((entry.payloadEncoding === "json-utf8" ||
+      moduleBundleEncodingToName(entry.payloadEncoding) === "json-utf8")
+      ? canonicalBytes(entry.payload)
+      : normalizeBytes(entry.payload, `entry "${entry.entryId}" payload`));
   return {
     ...entry,
     payload: payloadBytes,
@@ -315,6 +326,17 @@ function buildParsedEntries(bundle) {
         parsedEntry.decodedManifest = null;
       }
     }
+    if (
+      parsedEntry.entryId === SDS_DEPLOYMENT_ENTRY_ID ||
+      parsedEntry.sectionName === SDS_DEPLOYMENT_SECTION_NAME
+    ) {
+      parsedEntry.decodedDeploymentPlan =
+        parsedEntry.payloadEncodingName === "json-utf8" &&
+        parsedEntry.decodedPayload &&
+        typeof parsedEntry.decodedPayload === "object"
+          ? parsedEntry.decodedPayload
+          : null;
+    }
     return parsedEntry;
   });
 }
@@ -432,16 +454,22 @@ export async function parseSingleFileBundle(bytes, options = {}) {
       manifest = null;
     }
   }
+  const deploymentEntry =
+    parsedEntries.find(
+      (entry) =>
+        entry.entryId === SDS_DEPLOYMENT_ENTRY_ID ||
+        entry.sectionName === SDS_DEPLOYMENT_SECTION_NAME,
+    ) ?? null;
   return {
     wasmBytes,
     bundleBytes,
     bundle,
     entries: parsedEntries,
     manifest,
+    deploymentPlan: deploymentEntry?.decodedDeploymentPlan ?? null,
     customSections,
     canonicalWasmBytes: canonical.canonicalWasmBytes,
     canonicalModuleHash: canonical.hashBytes,
     canonicalModuleHashHex: canonical.hashHex,
   };
 }
-

package/src/compliance/pluginCompliance.js

@@ -8,6 +8,8 @@ import {
   ExternalInterfaceDirection,
   ExternalInterfaceKind,
   InvokeSurface,
+  ProtocolRole,
+  ProtocolTransportKind,
   RuntimeTarget,
 } from "../runtime/constants.js";
 
@@ -59,6 +61,8 @@ const ExternalInterfaceDirectionSet = new Set(
   Object.values(ExternalInterfaceDirection),
 );
 const ExternalInterfaceKindSet = new Set(Object.values(ExternalInterfaceKind));
+const ProtocolRoleSet = new Set(Object.values(ProtocolRole));
+const ProtocolTransportKindSet = new Set(Object.values(ProtocolTransportKind));
 const BrowserIncompatibleCapabilitySet = new Set([
   "filesystem",
   "pipe",
@@ -120,6 +124,57 @@ function hasNonEmptyByteSequence(value) {
   return false;
 }
 
+function normalizeTypeIdentityString(value) {
+  if (!isNonEmptyString(value)) {
+    return null;
+  }
+  return value.trim().toLowerCase();
+}
+
+function normalizeTypeIdentityBytes(value) {
+  if (typeof value === "string") {
+    const trimmed = value.trim().toLowerCase();
+    return trimmed.length > 0 ? trimmed : null;
+  }
+  const bytes = ArrayBuffer.isView(value)
+    ? Array.from(value)
+    : Array.isArray(value)
+      ? value
+      : null;
+  if (!bytes || bytes.length === 0) {
+    return null;
+  }
+  return bytes
+    .map((entry) => Number(entry).toString(16).padStart(2, "0"))
+    .join("");
+}
+
+function allowedTypesReferToSameLogicalSchema(left, right) {
+  const leftSchemaName = normalizeTypeIdentityString(left?.schemaName);
+  const rightSchemaName = normalizeTypeIdentityString(right?.schemaName);
+  if (leftSchemaName && rightSchemaName && leftSchemaName === rightSchemaName) {
+    return true;
+  }
+
+  const leftFileIdentifier = normalizeTypeIdentityString(left?.fileIdentifier);
+  const rightFileIdentifier = normalizeTypeIdentityString(right?.fileIdentifier);
+  if (
+    leftFileIdentifier &&
+    rightFileIdentifier &&
+    leftFileIdentifier === rightFileIdentifier
+  ) {
+    return true;
+  }
+
+  const leftSchemaHash = normalizeTypeIdentityBytes(left?.schemaHash);
+  const rightSchemaHash = normalizeTypeIdentityBytes(right?.schemaHash);
+  if (leftSchemaHash && rightSchemaHash && leftSchemaHash === rightSchemaHash) {
+    return true;
+  }
+
+  return false;
+}
+
 function normalizePayloadWireFormatName(value) {
   if (value === undefined || value === null || value === "") {
     return "flatbuffer";
@@ -151,7 +206,13 @@ function validateStringField(issues, value, location, label) {
   return true;
 }
 
-function validateIntegerField(issues, value, location, label, { min = null } = {}) {
+function validateIntegerField(
+  issues,
+  value,
+  location,
+  label,
+  { min = null, max = null } = {},
+) {
   if (!Number.isInteger(value)) {
     pushIssue(issues, "error", "invalid-integer", `${label} must be an integer.`, location);
     return false;
@@ -166,6 +227,16 @@ function validateIntegerField(issues, value, location, label, { min = null } = {
     );
     return false;
   }
+  if (max !== null && value > max) {
+    pushIssue(
+      issues,
+      "error",
+      "integer-range",
+      `${label} must be less than or equal to ${max}.`,
+      location,
+    );
+    return false;
+  }
   return true;
 }
 
@@ -182,6 +253,61 @@ function validateOptionalIntegerField(
   return validateIntegerField(issues, value, location, label, options);
 }
 
+function validateOptionalBooleanField(issues, value, location, label) {
+  if (value === undefined || value === null) {
+    return true;
+  }
+  if (typeof value !== "boolean") {
+    pushIssue(
+      issues,
+      "error",
+      "invalid-boolean",
+      `${label} must be a boolean when present.`,
+      location,
+    );
+    return false;
+  }
+  return true;
+}
+
+function validateOptionalStringField(issues, value, location, label) {
+  if (value === undefined || value === null) {
+    return true;
+  }
+  return validateStringField(issues, value, location, label);
+}
+
+function normalizeProtocolTransportKind(value) {
+  if (value === undefined || value === null || value === "") {
+    return null;
+  }
+  const normalized = String(value)
+    .trim()
+    .toLowerCase()
+    .replace(/_/g, "-");
+  if (normalized === "websocket") {
+    return ProtocolTransportKind.WS;
+  }
+  if (normalized === "pipe") {
+    return ProtocolTransportKind.WASI_PIPE;
+  }
+  return normalized;
+}
+
+function normalizeProtocolRole(value) {
+  if (value === undefined || value === null || value === "") {
+    return null;
+  }
+  const normalized = String(value)
+    .trim()
+    .toLowerCase()
+    .replace(/_/g, "-");
+  if (normalized === "handler") {
+    return ProtocolRole.HANDLE;
+  }
+  return normalized;
+}
+
 function validateAllowedType(type, issues, location) {
   if (!type || typeof type !== "object" || Array.isArray(type)) {
     pushIssue(issues, "error", "invalid-type-record", "Allowed type entries must be objects.", location);
@@ -307,6 +433,35 @@ function validateAcceptedTypeSet(typeSet, issues, location) {
   typeSet.allowedTypes.forEach((allowedType, index) => {
     validateAllowedType(allowedType, issues, `${location}.allowedTypes[${index}]`);
   });
+
+  const regularConcreteTypes = [];
+  const alignedTypes = [];
+
+  typeSet.allowedTypes.forEach((allowedType, index) => {
+    const wireFormat = normalizePayloadWireFormatName(allowedType?.wireFormat);
+    if (wireFormat === "aligned-binary") {
+      alignedTypes.push({ allowedType, index });
+      return;
+    }
+    if (wireFormat === "flatbuffer" && allowedType?.acceptsAnyFlatbuffer !== true) {
+      regularConcreteTypes.push({ allowedType, index });
+    }
+  });
+
+  alignedTypes.forEach(({ allowedType, index }) => {
+    const hasRegularFallback = regularConcreteTypes.some(({ allowedType: regularType }) =>
+      allowedTypesReferToSameLogicalSchema(allowedType, regularType),
+    );
+    if (!hasRegularFallback) {
+      pushIssue(
+        issues,
+        "error",
+        "missing-flatbuffer-fallback",
+        "Aligned-binary allowed types must be paired with a regular flatbuffer fallback for the same schema in the same acceptedTypeSet.",
+        `${location}.allowedTypes[${index}]`,
+      );
+    }
+  });
 }
 
 function validatePort(port, issues, location, label) {
@@ -576,6 +731,85 @@ function validateProtocol(protocol, issues, location, methodLookup, declaredCapa
       );
     }
   }
+  const wireIdValid = validateStringField(
+    issues,
+    protocol.wireId,
+    `${location}.wireId`,
+    "Protocol wireId",
+  );
+  const normalizedTransportKind = normalizeProtocolTransportKind(
+    protocol.transportKind,
+  );
+  if (!validateStringField(
+    issues,
+    protocol.transportKind,
+    `${location}.transportKind`,
+    "Protocol transportKind",
+  )) {
+    // already reported
+  } else if (!ProtocolTransportKindSet.has(normalizedTransportKind)) {
+    pushIssue(
+      issues,
+      "error",
+      "unknown-protocol-transport-kind",
+      `Protocol "${protocol.protocolId ?? "protocol"}" transportKind must be one of: ${Array.from(ProtocolTransportKindSet).join(", ")}.`,
+      `${location}.transportKind`,
+    );
+  }
+  const normalizedRole = normalizeProtocolRole(protocol.role);
+  if (!validateStringField(
+    issues,
+    protocol.role,
+    `${location}.role`,
+    "Protocol role",
+  )) {
+    // already reported
+  } else if (!ProtocolRoleSet.has(normalizedRole)) {
+    pushIssue(
+      issues,
+      "error",
+      "unknown-protocol-role",
+      `Protocol "${protocol.protocolId ?? "protocol"}" role must be one of: ${Array.from(ProtocolRoleSet).join(", ")}.`,
+      `${location}.role`,
+    );
+  }
+  validateOptionalStringField(
+    issues,
+    protocol.specUri,
+    `${location}.specUri`,
+    "Protocol specUri",
+  );
+  validateOptionalStringField(
+    issues,
+    protocol.discoveryKey,
+    `${location}.discoveryKey`,
+    "Protocol discoveryKey",
+  );
+  validateOptionalBooleanField(
+    issues,
+    protocol.autoInstall,
+    `${location}.autoInstall`,
+    "Protocol autoInstall",
+  );
+  validateOptionalBooleanField(
+    issues,
+    protocol.advertise,
+    `${location}.advertise`,
+    "Protocol advertise",
+  );
+  validateOptionalBooleanField(
+    issues,
+    protocol.requireSecureTransport,
+    `${location}.requireSecureTransport`,
+    "Protocol requireSecureTransport",
+  );
+  validateOptionalIntegerField(
+    issues,
+    protocol.defaultPort,
+    `${location}.defaultPort`,
+    "Protocol defaultPort",
+    { min: 0, max: 65535 },
+  );
   if (
     protocolIdValid &&
     Array.isArray(declaredCapabilities) &&
@@ -590,6 +824,77 @@ function validateProtocol(protocol, issues, location, methodLookup, declaredCapa
       location,
     );
   }
+  if (
+    protocolIdValid &&
+    normalizedRole &&
+    (normalizedRole === ProtocolRole.HANDLE ||
+      normalizedRole === ProtocolRole.BOTH) &&
+    Array.isArray(declaredCapabilities) &&
+    !declaredCapabilities.includes("protocol_handle")
+  ) {
+    pushIssue(
+      issues,
+      "error",
+      "missing-handle-protocol-capability",
+      `Protocol "${protocol.protocolId}" with role "${normalizedRole}" requires the "protocol_handle" capability.`,
+      location,
+    );
+  }
+  if (
+    protocolIdValid &&
+    normalizedRole &&
+    (normalizedRole === ProtocolRole.DIAL ||
+      normalizedRole === ProtocolRole.BOTH) &&
+    Array.isArray(declaredCapabilities) &&
+    !declaredCapabilities.includes("protocol_dial")
+  ) {
+    pushIssue(
+      issues,
+      "error",
+      "missing-dial-protocol-capability",
+      `Protocol "${protocol.protocolId}" with role "${normalizedRole}" requires the "protocol_dial" capability.`,
+      location,
+    );
+  }
+  if (
+    protocol.advertise === true &&
+    normalizedRole === ProtocolRole.DIAL
+  ) {
+    pushIssue(
+      issues,
+      "error",
+      "protocol-advertise-role-conflict",
+      `Protocol "${protocol.protocolId ?? "protocol"}" cannot advertise when role is "dial".`,
+      `${location}.advertise`,
+    );
+  }
+  if (
+    normalizedTransportKind === ProtocolTransportKind.LIBP2P &&
+    Array.isArray(declaredCapabilities) &&
+    !declaredCapabilities.includes("ipfs")
+  ) {
+    pushIssue(
+      issues,
+      "error",
+      "missing-ipfs-capability",
+      `Protocol "${protocol.protocolId ?? "protocol"}" with transportKind "libp2p" requires the "ipfs" capability.`,
+      location,
+    );
+  }
+  if (
+    wireIdValid &&
+    normalizedTransportKind === ProtocolTransportKind.WS &&
+    protocol.defaultPort === 443 &&
+    protocol.requireSecureTransport !== true
+  ) {
+    pushIssue(
+      issues,
+      "warning",
+      "insecure-secure-port-hint",
+      `Protocol "${protocol.protocolId ?? "protocol"}" uses defaultPort 443 without requireSecureTransport=true.`,
+      `${location}.defaultPort`,
+    );
+  }
 }
 
 function validateRuntimeTargets(runtimeTargets, declaredCapabilities, issues, sourceName) {