space-data-module-sdk 0.8.6 → 0.8.8

package/src/{testing → host}/workerModuleHarness.js

@@ -18,14 +18,15 @@
  * cross-origin isolated (COOP+COEP). In Node, worker_threads is used.
  */
 
-import { createBrowserHost } from "../host/browserHost.js";
-import { createAsyncHostDispatcher } from "../host/abi.js";
+import { createBrowserHost } from "./browserHost.js";
+import { createAsyncHostDispatcher } from "./abi.js";
 import {
   createSabHostcallBuffer,
   createSabHostcallServer,
   DEFAULT_SAB_HOSTCALL_RESPONSE_BYTES,
-} from "../host/sabHostcallChannel.js";
-import { WASI_THREAD_HOSTCALL_MESSAGE } from "../host/wasiThreadWorkerRuntime.js";
+} from "./sabHostcallChannel.js";
+import { WASI_THREAD_HOSTCALL_MESSAGE } from "./wasiThreadWorkerRuntime.js";
+import { importNodeBuiltin } from "./nodeBuiltinSpecifier.js";
 
 const WORKER_URL = new URL("./workerModuleHarnessWorker.js", import.meta.url);
 
@@ -51,7 +52,10 @@ function isNodeRuntime() {
 
 async function spawnWorker(workerUrl) {
   if (isNodeRuntime()) {
-    const { Worker: NodeWorker } = await import("node:worker_threads");
+    // Specifier assembled at runtime on purpose — see nodeBuiltinSpecifier.js.
+    // A literal here is statically resolved by every browser bundler even
+    // though this branch is dead in a browser.
+    const { Worker: NodeWorker } = await importNodeBuiltin("worker_threads");
     const worker = new NodeWorker(workerUrl);
     return {
       post: (message) => worker.postMessage(message),
@@ -75,17 +79,46 @@ async function spawnWorker(workerUrl) {
   };
 }
 
-function toBytes(source, label) {
-  if (source instanceof Uint8Array) return new Uint8Array(source);
-  if (source instanceof ArrayBuffer) return new Uint8Array(source.slice(0));
+/**
+ * Compile `source` to a `WebAssembly.Module` on THIS (controlling) thread
+ * before ever touching a worker.
+ *
+ * Plaintext-hygiene contract (mirrors OrbPro's `wasmModuleHygiene.js`,
+ * `wasm-plaintext-memory-hygiene`): a compiled `WebAssembly.Module` is
+ * structured-cloneable to a Worker (browser) or `node:worker_threads`
+ * (Node) — proven by `createWasiThreadSpawn` in `src/host/wasiThreadHost.js`,
+ * which hands the same compiled module to every pooled worker in both
+ * lanes. Shipping the MODULE instead of raw bytes means there is no second
+ * plaintext copy alive in the worker, and no re-compile once it arrives —
+ * `createBrowserModuleHarness`'s own `compileWasmModule` already accepts a
+ * `WebAssembly.Module` directly and returns it unchanged.
+ *
+ * `source` is never retained past this call and never zeroed here: a
+ * caller-supplied `Uint8Array`/`ArrayBuffer` is not this harness's buffer to
+ * mutate (compile does not retain a reference to it either — the caller is
+ * free to reuse or scrub it immediately after this resolves).
+ *
+ * @param {WebAssembly.Module|Uint8Array|ArrayBuffer} source
+ * @param {string} label diagnostic label
+ * @returns {Promise<WebAssembly.Module>}
+ */
+async function toWasmModule(source, label) {
+  if (source instanceof WebAssembly.Module) {
+    return source;
+  }
+  if (source instanceof Uint8Array || source instanceof ArrayBuffer) {
+    return WebAssembly.compile(source);
+  }
   throw new TypeError(
-    `${label} must be a Uint8Array or ArrayBuffer (the worker harness ships bytes to the worker).`,
+    `${label} must be a WebAssembly.Module, Uint8Array, or ArrayBuffer.`,
   );
 }
 
 /**
  * @param {object} options
- * @param {Uint8Array|ArrayBuffer} options.wasmSource module bytes
+ * @param {WebAssembly.Module|Uint8Array|ArrayBuffer} options.wasmSource module
+ *   bytes OR an already-compiled module (preferred — skips a redundant
+ *   compile and avoids ever materializing plaintext bytes in this harness)
  * @param {object} [options.host] host servicing guest hostcalls (default createBrowserHost(hostOptions))
  * @param {object} [options.hostOptions] options for the default host
  * @param {(operation: string, params: any) => Promise<any>} [options.dispatchHost] dispatch override
@@ -96,7 +129,7 @@ function toBytes(source, label) {
  *   createBrowserModuleHarness (surface, args, env, allowRawInvoke, logOutput, ...)
  */
 export async function createWorkerModuleHarness(options = {}) {
-  const wasmBytes = toBytes(options.wasmSource, "wasmSource");
+  const wasmModule = await toWasmModule(options.wasmSource, "wasmSource");
   const host = options.host ?? createBrowserHost(options.hostOptions);
   const dispatch = options.dispatchHost ?? createAsyncHostDispatcher(host);
   const buffer = createSabHostcallBuffer({
@@ -150,7 +183,7 @@ export async function createWorkerModuleHarness(options = {}) {
       case "worker-online":
         port.post({
           type: "init",
-          wasmBytes,
+          wasmModule,
           buffer,
           hostcallTimeoutMs: options.hostcallTimeoutMs,
           harnessOptions: {

package/src/{testing → host}/workerModuleHarnessWorker.js

@@ -10,7 +10,7 @@
  */
 
 import { createBrowserModuleHarness } from "./browserModuleHarness.js";
-import { createSabHostcallClientDispatch } from "../host/sabHostcallChannel.js";
+import { createSabHostcallClientDispatch } from "./sabHostcallChannel.js";
 
 async function resolvePort() {
   if (
@@ -90,12 +90,13 @@ async function handleInit(message) {
     timeoutMs: message.hostcallTimeoutMs,
     postRequest: (request) => port.post({ type: "hostcall", ...request }),
   });
+  // Plaintext-hygiene contract: the controlling thread compiles once and
+  // ships the WebAssembly.Module (never raw bytes) — see toWasmModule() in
+  // workerModuleHarness.js. createBrowserModuleHarness's compileWasmModule
+  // already returns a Module input unchanged, so this never re-compiles.
   const harnessOptions = {
     ...(message.harnessOptions ?? {}),
-    wasmSource:
-      message.wasmBytes instanceof Uint8Array
-        ? message.wasmBytes
-        : new Uint8Array(message.wasmBytes),
+    wasmSource: message.wasmModule,
     hostcallDispatch: dispatch,
   };
   harness = await createBrowserModuleHarness(harnessOptions);

package/src/index.d.ts

@@ -1962,7 +1962,11 @@ export interface BrowserModuleHarness {
 }
 export function detectArtifactProfile(wasmModule: WebAssembly.Module): string;
 export function createBrowserModuleHarness(options?: {
-  wasmSource: Uint8Array | ArrayBuffer | string | WebAssembly.Module | unknown;
+  wasmSource: Uint8Array | ArrayBuffer | string | Response | WebAssembly.Module | unknown;
+  // `true` uses env/global-configured trust; an object sets it explicitly;
+  // `false` disables verification. Ignored for a precompiled WebAssembly.Module
+  // (unverifiable — pass bytes/URL/Response when verification is required).
+  verifySignature?: boolean | { trustedPublicKeys?: string[]; requireSignature?: boolean };
   host?: BrowserHost | RuntimeHost | Record<string, unknown>;
   hostOptions?: BrowserHostOptions;
   args?: string[];
@@ -1979,6 +1983,25 @@ export function createBrowserModuleHarness(options?: {
   allowRawInvoke?: boolean;
   initialMemoryBytes?: number;
   maximumMemoryBytes?: number;
+  // Upper bound on guest wasi-threads spawns; sizes the browser warm worker
+  // pool (defaults to hardware concurrency). No effect on a non-threaded guest.
+  maxThreads?: number;
+  // Request-isolated BroadcastChannel descriptor a nested pthread hostcall
+  // dispatches over; supplied by createWorkerModuleHarness for the in-worker
+  // harness instance, not something a top-level caller usually sets by hand.
+  threadHostcallChannel?: {
+    channelName: string;
+    token: string;
+    maxResponseBytes?: number;
+    timeoutMs?: number;
+  };
+  // Explicit successful cross-origin-isolation negotiation for wasi-threads
+  // guests; defaults to true. See docs/browser-wasmedge-isomorphic.md's
+  // cross-origin-isolation section for when this is required.
+  enableBrowserWasiThreads?: boolean;
+  // Capacity of the module-memory arena a direct-invoke request is written
+  // into (bytes). Defaults to 64 KiB.
+  directInvokeRequestArenaBytes?: number;
   logOutput?: boolean;
   hostcallDispatch?: (operation: string, params: unknown) => unknown;
 }): Promise<BrowserModuleHarness>;
@@ -1997,7 +2020,11 @@ export interface WorkerModuleHarness {
   destroy(): Promise<void>;
 }
 export function createWorkerModuleHarness(options: {
-  wasmSource: Uint8Array | ArrayBuffer;
+  // A precompiled WebAssembly.Module is preferred: it skips a redundant
+  // compile and is structured-cloned to the worker directly (no plaintext
+  // bytes cross the postMessage boundary at all). Uint8Array/ArrayBuffer is
+  // compiled once on the controlling thread before the worker is spawned.
+  wasmSource: WebAssembly.Module | Uint8Array | ArrayBuffer;
   host?: unknown;
   hostOptions?: BrowserHostOptions;
   dispatchHost?: (operation: string, params: unknown) => Promise<unknown>;

package/src/standards/browser.js

@@ -0,0 +1,95 @@
+/**
+ * Standards catalog — BROWSER leg (the `browser` condition of `./standards`).
+ *
+ * Same catalog, same validation verdicts as the Node leg; only acquisition
+ * differs, and that difference is absorbed here rather than by a runtime check
+ * inside shared code. A browser has no `require.resolve` and no readFile, so
+ * the manifest must be handed in or fetched:
+ *
+ *   loadStandardsCatalog({ manifest })     // already-parsed manifest.json
+ *   loadStandardsCatalog({ manifestUrl })  // fetched, then parsed
+ *
+ * Neither one supplied is a caller defect, not a fallback: it throws. Silently
+ * returning an empty catalog would let a browser "validate" a manifest against
+ * nothing and agree with a Node run that actually checked it — exactly the kind
+ * of quiet divergence the tri-runtime contract exists to forbid.
+ */
+
+import {
+  buildStandardsCatalog,
+  validateManifestAgainstCatalog,
+  withSharedModuleCatalog,
+} from "./catalogCore.js";
+
+export {
+  buildStandardsCatalog,
+  resolveStandardsTypeRef,
+  validateManifestAgainstCatalog,
+  withSharedModuleCatalog,
+} from "./catalogCore.js";
+
+const standardsCatalogPromises = new Map();
+const knownTypeCatalogPromises = new Map();
+
+function catalogCacheKey(options) {
+  if (typeof options?.manifestUrl === "string" && options.manifestUrl) {
+    return `url:${options.manifestUrl}`;
+  }
+  return null;
+}
+
+async function readManifest(options) {
+  if (options?.manifest && typeof options.manifest === "object") {
+    return options.manifest;
+  }
+  if (typeof options?.manifestUrl === "string" && options.manifestUrl) {
+    const response = await fetch(options.manifestUrl);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch the spacedatastandards.org manifest from ${options.manifestUrl}: ${response.status}`,
+      );
+    }
+    return response.json();
+  }
+  throw new Error(
+    "The browser standards catalog needs the spacedatastandards.org manifest: " +
+      "pass { manifest } (already parsed) or { manifestUrl } (fetched). " +
+      "There is no on-disk manifest to read in a browser.",
+  );
+}
+
+export async function loadStandardsCatalog(options = {}) {
+  const key = catalogCacheKey(options);
+  if (key === null) {
+    return buildStandardsCatalog(await readManifest(options));
+  }
+  if (!standardsCatalogPromises.has(key)) {
+    standardsCatalogPromises.set(
+      key,
+      (async () => buildStandardsCatalog(await readManifest(options)))(),
+    );
+  }
+  return standardsCatalogPromises.get(key);
+}
+
+export async function loadKnownTypeCatalog(options = {}) {
+  const key = catalogCacheKey(options);
+  if (key === null) {
+    return withSharedModuleCatalog(await loadStandardsCatalog(options));
+  }
+  if (!knownTypeCatalogPromises.has(key)) {
+    knownTypeCatalogPromises.set(
+      key,
+      loadStandardsCatalog(options).then(withSharedModuleCatalog),
+    );
+  }
+  return knownTypeCatalogPromises.get(key);
+}
+
+export async function validateManifestAgainstStandardsCatalog(
+  manifest,
+  options = {},
+) {
+  const catalog = options.catalog ?? (await loadKnownTypeCatalog(options));
+  return validateManifestAgainstCatalog(catalog, manifest, options);
+}

package/src/standards/catalogCore.js

@@ -0,0 +1,290 @@
+/**
+ * Standards catalog — the runtime-independent half.
+ *
+ * Parsing, freshness validation, type-ref resolution and manifest checking are
+ * identical everywhere. Only HOW the `spacedatastandards.org` manifest is
+ * OBTAINED differs: Node reads it off disk (`./index.js`), a browser is handed
+ * it or fetches it (`./browser.js`). The package's `browser` export condition
+ * picks the leg, so a browser bundle never statically resolves
+ * node:fs/promises, node:path or node:module
+ * (`module-sdk-browser-entry-node-builtins`).
+ */
+
+import { sharedModuleCatalog } from "./sharedCatalog.js";
+import { normalizePayloadSchemaHash } from "../manifest/typeRefs.js";
+const requiredCurrentScvTokens = Object.freeze([
+  "SENSOR_LOCAL",
+  "table SCVResult",
+  "table SCVSensorShapeContract",
+  "SAR_ANNULAR_SECTOR",
+  "enum scvSensorRangeBoundaryKind",
+  "table SCVAggregateStatistics",
+  "AGGREGATE_STATISTICS:SCVAggregateStatistics",
+  "table SCVPackedRasterProducts",
+  "table SCVPackedRasterBand",
+  "RASTER_PRODUCTS:SCVPackedRasterProducts",
+  "enum scvRasterProductKind",
+]);
+const currentScvResultFields = Object.freeze(
+  new Set([
+    "JOB_ID",
+    "TRACE_ID",
+    "STATUS",
+    "TIME_GRID",
+    "TARGET_BODY",
+    "TOTAL_SENSORS",
+    "TOTAL_WINDOWS",
+    "CELL_STATS",
+    "INTERVALS",
+    "LATITUDE_BANDS",
+    "TIME_SERIES",
+    "HISTOGRAMS",
+    "CONTRIBUTIONS",
+    "GEOMETRY",
+    "RASTER_PRODUCTS",
+    "MESSAGE",
+    "AGGREGATE_STATISTICS",
+    "TARGET_RESULTS",
+  ]),
+);
+
+function collectFlatbufferTableFields(idl, tableName) {
+  const match = String(idl ?? "").match(
+    new RegExp(`\\btable\\s+${tableName}\\s*\\{([\\s\\S]*?)\\n\\}`, "m"),
+  );
+  if (!match) {
+    return [];
+  }
+  return match[1]
+    .split("\n")
+    .map((line) => line.replace(/\/\/.*$/, "").trim())
+    .map((line) => line.match(/^([A-Z0-9_]+)\s*:/)?.[1] ?? null)
+    .filter(Boolean);
+}
+
+function normalizeSchemaName(value) {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function normalizeFileIdentifier(value) {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function parseStandardsEntry([schemaCode, entry]) {
+  const idl = String(entry?.IDL ?? "");
+  const fileIdentifierMatch = idl.match(/file_identifier\s+"([^"]+)"/i);
+  const hashMatch = idl.match(/\/\/ Hash:\s*([a-f0-9]+)/i);
+  const versionMatch = idl.match(/\/\/ Version:\s*([^\n]+)/i);
+  const rootTypeMatch = idl.match(/root_type\s+([A-Za-z_][A-Za-z0-9_.]*)\s*;/i);
+  return {
+    schemaCode: schemaCode.toUpperCase(),
+    schemaName: `${schemaCode.toUpperCase()}.fbs`,
+    fileIdentifier: fileIdentifierMatch
+      ? normalizeFileIdentifier(fileIdentifierMatch[1])
+      : null,
+    hash: hashMatch ? hashMatch[1].toLowerCase() : null,
+    rootTypeName: rootTypeMatch ? rootTypeMatch[1] : null,
+    idl,
+    version: versionMatch ? versionMatch[1].trim() : null,
+    files: Array.isArray(entry?.files) ? entry.files : [],
+  };
+}
+
+function validateStandardsCatalogFreshness(catalog, sourceName) {
+  const issues = [];
+  const scvEntry = catalog.find((entry) => entry.schemaCode === "SCV");
+  if (!scvEntry) {
+    return issues;
+  }
+  const missingTokens = requiredCurrentScvTokens.filter(
+    (token) => !scvEntry.idl.includes(token),
+  );
+  if (missingTokens.length > 0) {
+    issues.push({
+      severity: "error",
+      code: "stale-scv-contract",
+      message:
+        "The loaded spacedatastandards.org SCV catalog is stale and does not " +
+        `include required current coverage fields: ${missingTokens.join(", ")}.`,
+      location: `${sourceName}.standards.SCV`,
+    });
+  }
+  const unsupportedResultFields = collectFlatbufferTableFields(
+    scvEntry.idl,
+    "SCVResult",
+  ).filter((field) => !currentScvResultFields.has(field));
+  if (unsupportedResultFields.length > 0) {
+    issues.push({
+      severity: "error",
+      code: "stale-scv-contract",
+      message:
+        "The loaded spacedatastandards.org SCV catalog contains unsupported " +
+        `SCVResult fields that are not part of the current contract: ${unsupportedResultFields.join(", ")}.`,
+      location: `${sourceName}.standards.SCV`,
+    });
+  }
+  return issues;
+}
+
+/**
+ * Build the standards catalog from an already-parsed manifest object. Same
+ * bytes in, same catalog out, in every runtime.
+ *
+ * @param {object} manifest - the parsed `spacedatastandards.org` manifest.json
+ * @returns {Array} sorted standards catalog
+ */
+export function buildStandardsCatalog(manifest) {
+  return Object.entries(manifest?.STANDARDS ?? {})
+    .map(parseStandardsEntry)
+    .sort((left, right) => left.schemaCode.localeCompare(right.schemaCode));
+}
+
+/**
+ * Append the SDK's own shared-module catalog to a standards catalog.
+ *
+ * @param {Array} catalog
+ * @returns {Array}
+ */
+export function withSharedModuleCatalog(catalog) {
+  return [...catalog, ...sharedModuleCatalog];
+}
+
+export function resolveStandardsTypeRef(typeRef, catalog = []) {
+  const schemaName = normalizeSchemaName(typeRef?.schemaName);
+  const fileIdentifier = normalizeFileIdentifier(typeRef?.fileIdentifier);
+  if (!schemaName && !fileIdentifier) {
+    return null;
+  }
+  return (
+    catalog.find(
+      (entry) =>
+        (!schemaName || normalizeSchemaName(entry.schemaName) === schemaName) &&
+        (!fileIdentifier ||
+          normalizeFileIdentifier(entry.fileIdentifier) === fileIdentifier),
+    ) ?? null
+  );
+}
+
+function schemaHashMatchesCatalog(typeRefHash, catalogHash) {
+  if (typeRefHash === undefined || typeRefHash === null) return true;
+  const declared = normalizePayloadSchemaHash(typeRefHash);
+  if (!declared) return true;
+  const canonical = normalizePayloadSchemaHash(catalogHash);
+  if (!canonical || declared.length !== canonical.length) {
+    return false;
+  }
+  return declared.every((byte, index) => byte === canonical[index]);
+}
+
+function collectTypeRefs(manifest) {
+  const refs = [];
+  for (const method of Array.isArray(manifest?.methods) ? manifest.methods : []) {
+    for (const portsKey of ["inputPorts", "outputPorts"]) {
+      for (const port of Array.isArray(method?.[portsKey]) ? method[portsKey] : []) {
+        for (const typeSet of Array.isArray(port?.acceptedTypeSets)
+          ? port.acceptedTypeSets
+          : []) {
+          for (const allowedType of Array.isArray(typeSet?.allowedTypes)
+            ? typeSet.allowedTypes
+            : []) {
+            refs.push({
+              source: `${method?.methodId ?? "method"}.${port?.portId ?? "port"}`,
+              typeRef: allowedType,
+            });
+          }
+        }
+      }
+    }
+  }
+  for (const typeRef of Array.isArray(manifest?.schemasUsed)
+    ? manifest.schemasUsed
+    : []) {
+    refs.push({
+      source: "schemasUsed",
+      typeRef,
+    });
+  }
+  return refs;
+}
+
+/**
+ * Validate a module manifest against an ALREADY-RESOLVED catalog.
+ *
+ * Both legs of `./standards` funnel into this, so the verdicts cannot diverge
+ * between runtimes; only catalog acquisition differs.
+ *
+ * @param {Array} catalog
+ * @param {object} manifest
+ * @param {object} [options]
+ * @returns {{catalog: Array, issues: Array}}
+ */
+export function validateManifestAgainstCatalog(catalog, manifest, options = {}) {
+  const sourceName = options.sourceName ?? "manifest";
+  const issues = validateStandardsCatalogFreshness(catalog, sourceName);
+  for (const { source, typeRef } of collectTypeRefs(manifest)) {
+    if (typeRef?.acceptsAnyFlatbuffer === true) {
+      continue;
+    }
+    const resolved = resolveStandardsTypeRef(typeRef, catalog);
+    if (!resolved) {
+      const schemaName = normalizeSchemaName(typeRef?.schemaName);
+      const fileIdentifier = normalizeFileIdentifier(typeRef?.fileIdentifier);
+      const partiallyKnown = catalog.some(
+        (entry) =>
+          (schemaName && normalizeSchemaName(entry.schemaName) === schemaName) ||
+          (fileIdentifier &&
+            normalizeFileIdentifier(entry.fileIdentifier) === fileIdentifier),
+      );
+      issues.push({
+        severity: "error",
+        code: partiallyKnown
+          ? "standards-type-identity-mismatch"
+          : "unresolved-standards-type",
+        message: partiallyKnown
+          ? `Type reference from ${source} mixes a known schemaName or exact four-byte fileIdentifier with a different standards entry.`
+          : `Type reference from ${source} does not resolve to a known shared-module or \`spacedatastandards.org\` schema by its exact schemaName and fileIdentifier.`,
+        location: `${sourceName}.${source}`,
+      });
+      continue;
+    }
+    if (
+      typeRef?.rootTypeName &&
+      resolved.rootTypeName &&
+      typeRef.rootTypeName !== resolved.rootTypeName
+    ) {
+      issues.push({
+        severity: "error",
+        code: "standards-root-type-mismatch",
+        message: `Type reference from ${source} declares rootTypeName ${JSON.stringify(typeRef.rootTypeName)} but the canonical SDS root is ${JSON.stringify(resolved.rootTypeName)}.`,
+        location: `${sourceName}.${source}`,
+      });
+    }
+    if (
+      typeRef?.schemaVersion &&
+      resolved.version &&
+      typeRef.schemaVersion !== resolved.version
+    ) {
+      issues.push({
+        severity: "error",
+        code: "standards-schema-version-mismatch",
+        message: `Type reference from ${source} declares schemaVersion ${JSON.stringify(typeRef.schemaVersion)} but the canonical SDS version is ${JSON.stringify(resolved.version)}.`,
+        location: `${sourceName}.${source}`,
+      });
+    }
+    if (
+      resolved.hash &&
+      !schemaHashMatchesCatalog(typeRef?.schemaHash, resolved.hash)
+    ) {
+      issues.push({
+        severity: "error",
+        code: "standards-schema-hash-mismatch",
+        message: `Type reference from ${source} declares a schemaHash that differs from the canonical SDS schema hash.`,
+        location: `${sourceName}.${source}`,
+      });
+    }
+  }
+  return {
+    catalog,
+    issues,
+  };
+}