space-data-module-sdk 0.5.19 → 0.5.21

package/src/runtime-host/AGENTS.md

@@ -0,0 +1,32 @@
+# AGENTS
+
+Apply the root and `src/AGENTS.md` files first.
+
+## Area Ownership
+
+This directory owns the canonical runtime-host storage model: row handles,
+region handles, FlatSQL-backed storage, and binary FlatBuffer ingest on the host
+side.
+
+## Storage And Streaming Rules
+
+- Keep durable row identity host-owned as `(schemaFileId, rowId)`.
+- Keep runtime aligned-binary identity host-owned as `(regionId, recordIndex)`.
+- The canonical ingest path is binary FlatBuffer bytes, not JSON.
+- Use size-prefixed FlatBuffer frames for streaming transport.
+- Do not coerce row payloads through JSON serialization.
+- If the host owns persistence, use runtime-host ingest helpers.
+- If a resident module owns state, keep the stream binary and push into the
+  module through the harness/pump path rather than inventing JSON wrappers.
+
+## Key Files
+
+- `flatbufferStreamIngestor.js`
+- `flatsqlRuntimeStore.js`
+- `index.js`
+
+## Verification
+
+- `npm run test:stream-ingest`
+- `node --test test/flatsql-local-node.test.js`
+- `npm run benchmark:stream-1gib` for large-stream changes

package/src/runtime-host/flatbufferStreamIngestor.js

@@ -0,0 +1,181 @@
+import { createFlatSqlRuntimeStore } from "./flatsqlRuntimeStore.js";
+
+function toUint8Array(data) {
+  if (data instanceof Uint8Array) {
+    return data;
+  }
+  if (data instanceof ArrayBuffer) {
+    return new Uint8Array(data);
+  }
+  if (ArrayBuffer.isView(data)) {
+    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  }
+  throw new TypeError(
+    "FlatBuffer stream ingestor expects Uint8Array, ArrayBuffer, or ArrayBufferView chunks.",
+  );
+}
+
+function concatUint8Arrays(chunks) {
+  let totalLength = 0;
+  for (let index = 0; index < chunks.length; index += 1) {
+    totalLength += chunks[index].byteLength;
+  }
+
+  const combined = new Uint8Array(totalLength);
+  let offset = 0;
+  for (let index = 0; index < chunks.length; index += 1) {
+    combined.set(chunks[index], offset);
+    offset += chunks[index].byteLength;
+  }
+  return combined;
+}
+
+function readFrameSize(bytes, offset) {
+  return (
+    bytes[offset] |
+    (bytes[offset + 1] << 8) |
+    (bytes[offset + 2] << 16) |
+    ((bytes[offset + 3] << 24) >>> 0)
+  );
+}
+
+function readFileIdentifier(payload) {
+  if (!(payload instanceof Uint8Array) || payload.byteLength < 8) {
+    return null;
+  }
+  return String.fromCharCode(payload[4], payload[5], payload[6], payload[7]);
+}
+
+function normalizeSchemaFileId(fileIdentifier) {
+  if (typeof fileIdentifier !== "string") {
+    return null;
+  }
+  const normalized = fileIdentifier.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function resolveRouteHandler(frameRouter, rawFileIdentifier, schemaFileId) {
+  if (typeof frameRouter === "function") {
+    return frameRouter;
+  }
+  if (!frameRouter || typeof frameRouter !== "object") {
+    return null;
+  }
+  if (typeof frameRouter[schemaFileId] === "function") {
+    return frameRouter[schemaFileId];
+  }
+  if (typeof frameRouter[rawFileIdentifier] === "function") {
+    return frameRouter[rawFileIdentifier];
+  }
+  return null;
+}
+
+export function createFlatBufferStreamIngestor(options = {}) {
+  const rows = options.rows ?? createFlatSqlRuntimeStore();
+  const frameRouter = options.frameRouter ?? null;
+  const appendFrame =
+    typeof options.appendFrame === "function" ? options.appendFrame : null;
+
+  const stats = {
+    bytesReceived: 0,
+    chunksReceived: 0,
+    framesDecoded: 0,
+    framesAppended: 0,
+    framesRouted: 0,
+    parseErrors: 0,
+  };
+
+  let pending = new Uint8Array(0);
+
+  function appendDecodedFrame(payload, context) {
+    if (appendFrame) {
+      appendFrame(payload, context);
+      return;
+    }
+    rows.appendRow({
+      schemaFileId: context.schemaFileId,
+      payload,
+    });
+  }
+
+  function pushBytes(data) {
+    const bytes = toUint8Array(data);
+    stats.bytesReceived += bytes.byteLength;
+    stats.chunksReceived += 1;
+
+    const combined =
+      pending.byteLength > 0 ? concatUint8Arrays([pending, bytes]) : bytes;
+
+    let offset = 0;
+    let appendedCount = 0;
+
+    while (offset + 4 <= combined.byteLength) {
+      const frameSize = readFrameSize(combined, offset);
+      if (frameSize <= 0) {
+        stats.parseErrors += 1;
+        throw new Error("Invalid FlatBuffer stream frame size.");
+      }
+      if (offset + 4 + frameSize > combined.byteLength) {
+        break;
+      }
+
+      const payload = combined.subarray(offset + 4, offset + 4 + frameSize);
+      const rawFileIdentifier = readFileIdentifier(payload);
+      const schemaFileId = normalizeSchemaFileId(rawFileIdentifier);
+      if (!schemaFileId) {
+        stats.parseErrors += 1;
+        throw new Error(
+          "FlatBuffer stream frame is missing a readable file identifier.",
+        );
+      }
+
+      const context = {
+        rawFileIdentifier,
+        schemaFileId,
+        rows,
+        stats,
+      };
+      const routeHandler = resolveRouteHandler(
+        frameRouter,
+        rawFileIdentifier,
+        schemaFileId,
+      );
+
+      stats.framesDecoded += 1;
+      if (routeHandler) {
+        const routeResult = routeHandler(payload, context);
+        if (routeResult !== false) {
+          stats.framesRouted += 1;
+          offset += 4 + frameSize;
+          continue;
+        }
+      }
+
+      appendDecodedFrame(payload, context);
+      stats.framesAppended += 1;
+      appendedCount += 1;
+      offset += 4 + frameSize;
+    }
+
+    pending =
+      offset < combined.byteLength
+        ? combined.slice(offset)
+        : new Uint8Array(0);
+
+    return appendedCount;
+  }
+
+  function finish() {
+    if (pending.byteLength > 0) {
+      throw new Error("FlatBuffer stream ended with a partial frame.");
+    }
+    return 0;
+  }
+
+  return {
+    rows,
+    stats,
+    pushBytes,
+    finish,
+  };
+}

package/src/runtime-host/flatsqlRuntimeStore.js

@@ -5,7 +5,6 @@ const RUNTIME_ROW_SCHEMA = `
 table RuntimeHostRow {
   schemaFileId: string;
   rowId: ulong;
-  payloadJson: string;
 }
 `;
 
@@ -19,14 +18,85 @@ function clonePayload(payload) {
   if (typeof structuredClone === "function") {
     return structuredClone(payload);
   }
+  return clonePayloadFallback(payload, new WeakMap());
+}
+
+function clonePayloadFallback(payload, seen) {
+  if (payload instanceof Date) {
+    return new Date(payload.getTime());
+  }
+  if (payload instanceof RegExp) {
+    return new RegExp(payload.source, payload.flags);
+  }
+  if (payload instanceof Map) {
+    if (seen.has(payload)) {
+      return seen.get(payload);
+    }
+    const cloned = new Map();
+    seen.set(payload, cloned);
+    for (const [key, value] of payload.entries()) {
+      cloned.set(clonePayloadFallback(key, seen), clonePayloadFallback(value, seen));
+    }
+    return cloned;
+  }
+  if (payload instanceof Set) {
+    if (seen.has(payload)) {
+      return seen.get(payload);
+    }
+    const cloned = new Set();
+    seen.set(payload, cloned);
+    for (const value of payload.values()) {
+      cloned.add(clonePayloadFallback(value, seen));
+    }
+    return cloned;
+  }
   if (ArrayBuffer.isView(payload)) {
-    return payload.slice(0);
+    if (payload instanceof DataView) {
+      return new DataView(
+        payload.buffer.slice(
+          payload.byteOffset,
+          payload.byteOffset + payload.byteLength,
+        ),
+      );
+    }
+    const clonedBuffer = payload.buffer.slice(
+      payload.byteOffset,
+      payload.byteOffset + payload.byteLength,
+    );
+    return new payload.constructor(clonedBuffer, 0, payload.length);
   }
   if (payload instanceof ArrayBuffer) {
     return payload.slice(0);
   }
+  if (Array.isArray(payload)) {
+    if (seen.has(payload)) {
+      return seen.get(payload);
+    }
+    const cloned = [];
+    seen.set(payload, cloned);
+    for (const value of payload) {
+      cloned.push(clonePayloadFallback(value, seen));
+    }
+    return cloned;
+  }
   if (typeof payload === "object") {
-    return JSON.parse(JSON.stringify(payload));
+    if (seen.has(payload)) {
+      return seen.get(payload);
+    }
+    const prototype = Object.getPrototypeOf(payload);
+    const cloned = Object.create(prototype ?? Object.prototype);
+    seen.set(payload, cloned);
+    for (const key of Reflect.ownKeys(payload)) {
+      const descriptor = Object.getOwnPropertyDescriptor(payload, key);
+      if (!descriptor) {
+        continue;
+      }
+      if ("value" in descriptor) {
+        descriptor.value = clonePayloadFallback(descriptor.value, seen);
+      }
+      Object.defineProperty(cloned, key, descriptor);
+    }
+    return cloned;
   }
   return payload;
 }
@@ -54,48 +124,46 @@ function escapeSqlStringLiteral(value) {
   return String(value).replaceAll("'", "''");
 }
 
-function serializePayload(payload) {
-  if (payload === null || payload === undefined) {
-    return {
-      kind: "json",
-      value: null,
-    };
-  }
-  if (ArrayBuffer.isView(payload)) {
-    return {
-      kind: "bytes",
-      bytes: Array.from(
-        new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength),
-      ),
-    };
+function encodeRuntimeRowMetadata({ schemaFileId, rowId }) {
+  const schemaBytes = encoder.encode(normalizeSchemaFileId(schemaFileId));
+  const encoded = new Uint8Array(4 + schemaBytes.byteLength + 8);
+  const view = new DataView(encoded.buffer);
+  view.setUint32(0, schemaBytes.byteLength, true);
+  encoded.set(schemaBytes, 4);
+  view.setBigUint64(4 + schemaBytes.byteLength, BigInt(rowId), true);
+  return encoded;
+}
+
+function decodeRuntimeRowMetadata(data) {
+  const bytes =
+    data instanceof Uint8Array
+      ? data
+      : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  if (bytes.byteLength < 12) {
+    throw new Error("Invalid runtime row metadata buffer.");
   }
-  if (payload instanceof ArrayBuffer) {
-    return {
-      kind: "bytes",
-      bytes: Array.from(new Uint8Array(payload)),
-    };
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const schemaLength = view.getUint32(0, true);
+  const schemaStart = 4;
+  const schemaEnd = schemaStart + schemaLength;
+  const rowIdOffset = schemaEnd;
+  if (schemaEnd + 8 > bytes.byteLength) {
+    throw new Error("Invalid runtime row metadata layout.");
   }
   return {
-    kind: "json",
-    value: clonePayload(payload),
+    schemaFileId: decoder.decode(bytes.subarray(schemaStart, schemaEnd)),
+    rowId: Number(view.getBigUint64(rowIdOffset, true)),
   };
 }
 
-function deserializePayload(payloadJson) {
-  const payload = JSON.parse(String(payloadJson ?? "null"));
-  if (payload?.kind === "bytes") {
-    return Uint8Array.from(payload.bytes ?? []);
-  }
-  if (payload?.kind === "json") {
-    return clonePayload(payload.value);
-  }
-  return clonePayload(payload);
+function buildRowKey(schemaFileId, rowId) {
+  return `${normalizeSchemaFileId(schemaFileId)}:${Number(rowId)}`;
 }
 
 function createRuntimeRowAccessor() {
   const accessor = new DirectAccessor();
   accessor.registerAccessor(RUNTIME_ROW_TABLE, (data, path) => {
-    const row = JSON.parse(decoder.decode(data));
+    const row = decodeRuntimeRowMetadata(data);
     let current = row;
     for (const segment of Array.isArray(path) ? path : []) {
       if (current === null || current === undefined) {
@@ -106,13 +174,10 @@ function createRuntimeRowAccessor() {
     return current ?? null;
   });
   accessor.registerBuilder(RUNTIME_ROW_TABLE, (fields) =>
-    encoder.encode(
-      JSON.stringify({
-        schemaFileId: normalizeSchemaFileId(fields.schemaFileId),
-        rowId: Number(fields.rowId),
-        payloadJson: String(fields.payloadJson ?? "null"),
-      }),
-    ),
+    encodeRuntimeRowMetadata({
+      schemaFileId: fields.schemaFileId,
+      rowId: Number(fields.rowId),
+    }),
   );
   return accessor;
 }
@@ -127,18 +192,21 @@ function cloneQueryResult(result) {
   };
 }
 
-function rowViewFromQueryRow(row) {
+function rowViewFromQueryRow(row, payloadStore) {
+  const schemaFileId = String(row[0]);
+  const rowId = Number(row[1]);
   return {
     handle: {
-      schemaFileId: String(row[0]),
-      rowId: Number(row[1]),
+      schemaFileId,
+      rowId,
     },
-    payload: deserializePayload(row[2]),
+    payload: clonePayload(payloadStore.get(buildRowKey(schemaFileId, rowId))),
   };
 }
 
 export function createFlatSqlRuntimeStore(options = {}) {
   const accessor = options.accessor ?? createRuntimeRowAccessor();
+  const payloadStore = options.payloadStore ?? new Map();
   const database =
     options.database ??
     FlatSQLDatabase.fromSchema(
@@ -168,8 +236,11 @@ export function createFlatSqlRuntimeStore(options = {}) {
     database.insert(RUNTIME_ROW_TABLE, {
       schemaFileId: normalizedSchemaFileId,
       rowId: nextRowId,
-      payloadJson: JSON.stringify(serializePayload(payload)),
     });
+    payloadStore.set(
+      buildRowKey(normalizedSchemaFileId, nextRowId),
+      clonePayload(payload),
+    );
     return {
       schemaFileId: normalizedSchemaFileId,
       rowId: nextRowId,
@@ -193,12 +264,12 @@ export function createFlatSqlRuntimeStore(options = {}) {
     const result =
       normalizedSchemaFileId === null
         ? database.query(
-            `SELECT schemaFileId, rowId, payloadJson FROM ${RUNTIME_ROW_TABLE} ORDER BY schemaFileId, rowId`,
+            `SELECT schemaFileId, rowId FROM ${RUNTIME_ROW_TABLE} ORDER BY schemaFileId, rowId`,
           )
         : database.query(
-            `SELECT schemaFileId, rowId, payloadJson FROM ${RUNTIME_ROW_TABLE} WHERE schemaFileId = '${escapeSqlStringLiteral(normalizedSchemaFileId)}' ORDER BY rowId`,
+            `SELECT schemaFileId, rowId FROM ${RUNTIME_ROW_TABLE} WHERE schemaFileId = '${escapeSqlStringLiteral(normalizedSchemaFileId)}' ORDER BY rowId`,
           );
-    return (result.rows ?? []).map(rowViewFromQueryRow);
+    return (result.rows ?? []).map((row) => rowViewFromQueryRow(row, payloadStore));
   }
 
   function query(sql) {

package/src/runtime-host/index.js

@@ -1,4 +1,5 @@
 import { createFlatSqlRuntimeStore } from "./flatsqlRuntimeStore.js";
+import { createFlatBufferStreamIngestor } from "./flatbufferStreamIngestor.js";
 import { createRuntimeRegionStore } from "./runtimeRegionStore.js";
 import { createModuleRegistry } from "./moduleRegistry.js";
 
@@ -15,6 +16,7 @@ export function createRuntimeHost(options = {}) {
 }
 
 export {
+  createFlatBufferStreamIngestor,
   createFlatSqlRuntimeStore,
   createModuleRegistry,
   createRuntimeRegionStore,

package/src/testing/AGENTS.md

@@ -0,0 +1,33 @@
+# AGENTS
+
+Apply the root and `src/AGENTS.md` files first.
+
+## Area Ownership
+
+This directory owns SDK test harnesses and runtime-facing helper surfaces used
+by examples and downstream consumers: browser harnesses, generic process invoke
+clients, runtime-matrix helpers, and resident-module FlatBuffer pumps.
+
+## Harness Rules
+
+- Keep browser harness behavior aligned with the same standalone artifacts that
+  WasmEdge runs.
+- Prefer portable invoke envelopes and portable WASI behavior over runtime-
+  specific shortcuts.
+- `createModuleFlatBufferStreamPump(...)` is the canonical no-JSON path for
+  feeding binary FlatBuffer streams into a resident module instance.
+- Avoid hiding stateful behavior inside one-off demos; if a harness contract is
+  real, test it here.
+
+## Key Files
+
+- `browserModuleHarness.js`
+- `moduleFlatbufferStreamPump.js`
+- `processInvoke.js`
+
+## Verification
+
+- `node --test test/browser-harness.test.js`
+- `node --test test/isomorphic-loader.test.js`
+- `npm run test:module-stream`
+- `node --test test/process-invoke.test.js`

package/src/testing/index.js

@@ -26,6 +26,7 @@ export {
   createModuleHarness,
   resolveModuleHarnessLaunchPlan,
 } from "./moduleHarness.js";
+export { createModuleFlatBufferStreamPump } from "./moduleFlatbufferStreamPump.js";
 
 const CapabilitySurfaceMatrix = Object.freeze({
   logging: Object.freeze({