space-data-module-sdk 0.8.4 → 0.8.5

package/docs/architecture.svg

@@ -0,0 +1,141 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 520" fill="none">
+  <defs>
+    <linearGradient id="bg" x1="0" y1="0" x2="900" y2="520" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#0d1117"/>
+      <stop offset="1" stop-color="#161b22"/>
+    </linearGradient>
+    <linearGradient id="accent" x1="0" y1="0" x2="1" y2="1">
+      <stop offset="0" stop-color="#58a6ff"/>
+      <stop offset="1" stop-color="#1f6feb"/>
+    </linearGradient>
+    <linearGradient id="green" x1="0" y1="0" x2="1" y2="1">
+      <stop offset="0" stop-color="#3fb950"/>
+      <stop offset="1" stop-color="#238636"/>
+    </linearGradient>
+    <linearGradient id="purple" x1="0" y1="0" x2="1" y2="1">
+      <stop offset="0" stop-color="#bc8cff"/>
+      <stop offset="1" stop-color="#8b5cf6"/>
+    </linearGradient>
+    <linearGradient id="orange" x1="0" y1="0" x2="1" y2="1">
+      <stop offset="0" stop-color="#f0883e"/>
+      <stop offset="1" stop-color="#d29922"/>
+    </linearGradient>
+    <filter id="glow">
+      <feGaussianBlur stdDeviation="3" result="blur"/>
+      <feMerge><feMergeNode in="blur"/><feMergeNode in="SourceGraphic"/></feMerge>
+    </filter>
+  </defs>
+
+  <rect width="900" height="520" rx="12" fill="url(#bg)"/>
+
+  <!-- Title -->
+  <text x="450" y="40" text-anchor="middle" font-family="system-ui, -apple-system, sans-serif" font-size="18" font-weight="700" fill="#e6edf3">Space Data Module Architecture</text>
+
+  <!-- Source Layer -->
+  <rect x="30" y="70" width="180" height="170" rx="10" fill="#161b22" stroke="#30363d" stroke-width="1.5"/>
+  <text x="120" y="95" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="600" fill="#e6edf3">Source</text>
+
+  <rect x="50" y="110" width="140" height="36" rx="6" fill="#1c2128" stroke="#58a6ff" stroke-width="1"/>
+  <text x="120" y="133" text-anchor="middle" font-family="monospace" font-size="11" fill="#58a6ff">module.c / .cpp</text>
+
+  <rect x="50" y="155" width="140" height="36" rx="6" fill="#1c2128" stroke="#f0883e" stroke-width="1"/>
+  <text x="120" y="178" text-anchor="middle" font-family="monospace" font-size="11" fill="#f0883e">manifest.json</text>
+
+  <rect x="50" y="200" width="140" height="28" rx="6" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="120" y="219" text-anchor="middle" font-family="monospace" font-size="10" fill="#3fb950">spacedatastandards.org</text>
+
+  <!-- Arrow: Source -> SDK -->
+  <line x1="210" y1="155" x2="260" y2="155" stroke="#484f58" stroke-width="2" marker-end="url(#arrowGray)"/>
+
+  <!-- SDK Core -->
+  <rect x="265" y="70" width="370" height="280" rx="10" fill="#161b22" stroke="#58a6ff" stroke-width="1.5"/>
+  <rect x="265" y="70" width="370" height="30" rx="10" fill="#1f6feb" fill-opacity="0.15"/>
+  <text x="450" y="91" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="700" fill="#58a6ff">space-data-module-sdk</text>
+
+  <!-- Compile -->
+  <rect x="285" y="115" width="150" height="50" rx="8" fill="#1c2128" stroke="#f0883e" stroke-width="1"/>
+  <text x="360" y="136" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#f0883e">Compile</text>
+  <text x="360" y="153" text-anchor="middle" font-family="monospace" font-size="9" fill="#8b949e">emcc / em++ -> .wasm</text>
+
+  <!-- Validate -->
+  <rect x="465" y="115" width="150" height="50" rx="8" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="540" y="136" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#3fb950">Validate</text>
+  <text x="540" y="153" text-anchor="middle" font-family="monospace" font-size="9" fill="#8b949e">compliance + standards</text>
+
+  <!-- Arrow: Compile -> Validate -->
+  <line x1="435" y1="140" x2="465" y2="140" stroke="#484f58" stroke-width="1.5" marker-end="url(#arrowGray)"/>
+
+  <!-- Embed Manifest -->
+  <rect x="285" y="185" width="150" height="50" rx="8" fill="#1c2128" stroke="#bc8cff" stroke-width="1"/>
+  <text x="360" y="206" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#bc8cff">Embed Manifest</text>
+  <text x="360" y="223" text-anchor="middle" font-family="monospace" font-size="9" fill="#8b949e">FlatBuffers binary ABI</text>
+
+  <!-- Sign -->
+  <rect x="465" y="185" width="150" height="50" rx="8" fill="#1c2128" stroke="#d29922" stroke-width="1"/>
+  <text x="540" y="206" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#d29922">Sign</text>
+  <text x="540" y="223" text-anchor="middle" font-family="monospace" font-size="9" fill="#8b949e">HD wallet + secp256k1</text>
+
+  <!-- Encrypt -->
+  <rect x="375" y="255" width="150" height="50" rx="8" fill="#1c2128" stroke="#f778ba" stroke-width="1"/>
+  <text x="450" y="276" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#f778ba">Encrypt</text>
+  <text x="450" y="293" text-anchor="middle" font-family="monospace" font-size="9" fill="#8b949e">X25519 + AES-256-GCM</text>
+
+  <!-- Internal arrows -->
+  <line x1="360" y1="165" x2="360" y2="185" stroke="#484f58" stroke-width="1.5" marker-end="url(#arrowGray)"/>
+  <line x1="540" y1="165" x2="540" y2="185" stroke="#484f58" stroke-width="1.5" marker-end="url(#arrowGray)"/>
+  <line x1="435" y1="210" x2="465" y2="210" stroke="#484f58" stroke-width="1.5" marker-end="url(#arrowGray)"/>
+  <path d="M 450 235 L 450 255" stroke="#484f58" stroke-width="1.5" marker-end="url(#arrowGray)"/>
+
+  <!-- Arrow: SDK -> Output -->
+  <line x1="635" y1="210" x2="685" y2="210" stroke="#484f58" stroke-width="2" marker-end="url(#arrowGray)"/>
+
+  <!-- Output: Protected Module Package -->
+  <rect x="690" y="70" width="180" height="280" rx="10" fill="#161b22" stroke="#30363d" stroke-width="1.5"/>
+  <text x="780" y="95" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="600" fill="#e6edf3">Protected Package</text>
+
+  <rect x="710" y="110" width="140" height="30" rx="6" fill="#1c2128" stroke="#bc8cff" stroke-width="1"/>
+  <text x="780" y="130" text-anchor="middle" font-family="monospace" font-size="10" fill="#bc8cff">manifest.fb</text>
+
+  <rect x="710" y="150" width="140" height="30" rx="6" fill="#1c2128" stroke="#58a6ff" stroke-width="1"/>
+  <text x="780" y="170" text-anchor="middle" font-family="monospace" font-size="10" fill="#58a6ff">module.wasm</text>
+
+  <rect x="710" y="190" width="140" height="30" rx="6" fill="#1c2128" stroke="#d29922" stroke-width="1"/>
+  <text x="780" y="210" text-anchor="middle" font-family="monospace" font-size="10" fill="#d29922">authorization.sig</text>
+
+  <rect x="710" y="230" width="140" height="30" rx="6" fill="#1c2128" stroke="#f778ba" stroke-width="1"/>
+  <text x="780" y="250" text-anchor="middle" font-family="monospace" font-size="10" fill="#f778ba">encrypted envelope</text>
+
+  <rect x="710" y="280" width="140" height="30" rx="6" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="780" y="300" text-anchor="middle" font-family="monospace" font-size="10" fill="#3fb950">compliance report</text>
+
+  <!-- Deployment targets -->
+  <rect x="30" y="380" width="840" height="120" rx="10" fill="#161b22" stroke="#30363d" stroke-width="1.5"/>
+  <rect x="30" y="380" width="840" height="30" rx="10" fill="#238636" fill-opacity="0.1"/>
+  <text x="450" y="401" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="700" fill="#3fb950">Deployment Targets</text>
+
+  <rect x="54" y="420" width="180" height="60" rx="8" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="144" y="447" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" font-weight="600" fill="#e6edf3">Mission Systems</text>
+  <text x="144" y="465" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#8b949e">Analysis / Planning / Ops</text>
+
+  <rect x="258" y="420" width="180" height="60" rx="8" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="348" y="447" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" font-weight="600" fill="#e6edf3">SDN Nodes</text>
+  <text x="348" y="465" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#8b949e">Peer-to-Peer Data Network</text>
+
+  <rect x="462" y="420" width="180" height="60" rx="8" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="552" y="447" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" font-weight="600" fill="#e6edf3">Edge Runtimes</text>
+  <text x="552" y="465" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#8b949e">Satellite / Ground Station</text>
+
+  <rect x="666" y="420" width="180" height="60" rx="8" fill="#1c2128" stroke="#3fb950" stroke-width="1"/>
+  <text x="756" y="447" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" font-weight="600" fill="#e6edf3">Browser</text>
+  <text x="756" y="465" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#8b949e">Module Lab / WebAssembly</text>
+
+  <!-- Arrows down to deployment -->
+  <line x1="450" y1="350" x2="450" y2="380" stroke="#484f58" stroke-width="2" marker-end="url(#arrowGray)"/>
+
+  <!-- Arrow markers -->
+  <defs>
+    <marker id="arrowGray" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#484f58"/>
+    </marker>
+  </defs>
+</svg>

package/docs/browser-wasmedge-isomorphic.md

@@ -0,0 +1,179 @@
+# Browser + WasmEdge Isomorphic Artifacts
+
+Use this profile when you want one compiled `.wasm` artifact that can be loaded
+unchanged in:
+
+- the browser harness
+- the WasmEdge command/runtime harness
+
+## Canonical Build Rule
+
+Declare:
+
+```json
+{
+  "runtimeTargets": ["browser", "wasmedge"]
+}
+```
+
+That target pair now defaults to the shared `single-thread` artifact profile.
+
+The compiler logic lives in
+[`src/compiler/compileModule.js`](/Users/tj/software/space-data-module-sdk/src/compiler/compileModule.js).
+
+The practical effect is:
+
+- `["wasmedge"]` keeps the higher-capability WasmEdge pthread default
+- `["browser", "wasmedge"]` chooses the portable single-thread artifact instead
+
+Use the pure `["wasmedge"]` target when you want maximum WasmEdge-native guest
+capability and do not need browser loading from the same binary.
+
+### Pthreads variant (shared-memory, isomorphic threading)
+
+When a module needs real guest threads (the `emscripten-pthreads` thread model,
+default for `["wasmedge"]`), the SDK compiles it through the **wasi-threads**
+toolchain (`clang --target=wasm32-wasip1-threads -pthread`), **not** Emscripten
+`-pthread` (which emits a browser-only Web-Worker build that cannot thread under
+WasmEdge). It **enforces** the wasi-threads link flags and **validates the
+emitted `.wasm`** — the artifact must import `wasi.thread-spawn`, export
+`wasi_thread_start`, be a shared-memory/atomics wasm, and carry no Emscripten
+worker hooks, or the compile is rejected. That guardrail is documented in
+[`docs/isomorphic-pthreads.md`](./isomorphic-pthreads.md). This present document
+covers the portable `single-thread` loading profile; read the pthreads doc
+before shipping a threaded WasmEdge artifact.
+
+## Canonical Module Repo Layout
+
+Module repos should publish the shared compiled artifact under a stable runtime
+path, not a plugin-named filename:
+
+- required: `dist/isomorphic/module.wasm`
+
+If a repo also ships a browser-specific adapter, place it under:
+
+- optional: `dist/browser/module.js`
+- optional: `dist/browser/module.wasm`
+
+That keeps the artifact name stable across repos and lets runtime intent live in
+the path rather than in the filename.
+
+## Toolchain Options
+
+The recommended browser-side build selector for module repos is:
+
+- `SDN_WASM_TOOLCHAIN=local-emsdk`: default. Build with a repo-local
+  `deps/emsdk` checkout and avoid Homebrew or any other machine-global
+  Emscripten install.
+- `SDN_WASM_TOOLCHAIN=sdn-emception`: optional SDK-first path for repos that use
+  `compileModuleFromSource(...)` or an SDK-driven build script to emit the
+  shared `dist/isomorphic/module.wasm` artifact.
+- `SDN_WASM_TOOLCHAIN=path`: explicit escape hatch when a repo intentionally
+  wants to use a preinstalled Emscripten toolchain from `PATH`.
+
+The isomorphic contract is the compiled wasm path, not the browser wrapper. A
+repo can publish only `dist/isomorphic/module.wasm` and still satisfy the
+browser/WasmEdge shared-artifact requirement when it loads through the SDK
+harnesses.
+
+## Loader Entry Points
+
+The supported browser/WasmEdge entry points are:
+
+- browser entry bundle: [`src/browser.js`](/Users/tj/software/space-data-module-sdk/src/browser.js)
+- isomorphic loader: [`src/host/isomorphicLoader.js`](/Users/tj/software/space-data-module-sdk/src/host/isomorphicLoader.js)
+- browser harness: [`src/testing/browserModuleHarness.js`](/Users/tj/software/space-data-module-sdk/src/testing/browserModuleHarness.js)
+- browser WASI shim: [`src/host/wasiShim.js`](/Users/tj/software/space-data-module-sdk/src/host/wasiShim.js)
+- browser edge shims: [`src/host/browserEdgeShims.js`](/Users/tj/software/space-data-module-sdk/src/host/browserEdgeShims.js)
+- browser host adapter: [`src/host/browserHost.js`](/Users/tj/software/space-data-module-sdk/src/host/browserHost.js)
+
+On the server path, [`loadModule(...)`](/Users/tj/software/space-data-module-sdk/src/host/isomorphicLoader.js)
+now chooses the raw WasmEdge command harness automatically for standalone
+artifacts with `_start`. The `--serve-plugin-invoke` runner protocol remains
+available for explicit runtime-host / runner-backed flows.
+
+## What The Browser Shims Cover
+
+The browser edge shims map host capabilities onto browser-native surfaces:
+
+- `filesystem`: in-memory virtual filesystem with `resolvePath`, `readFile`,
+  `writeFile`, `appendFile`, `deleteFile`, `mkdir`, `readdir`, `stat`, `rename`
+- `http`: `fetch`
+- `websocket`: browser `WebSocket`
+- `network`: async browser-host dispatch that routes to the available transport
+  adapters
+- `ipfs`, `protocol_handle`, `protocol_dial`: async browser-host adapters that
+  can be supplied by the embedding runtime
+- `clock`, `random`, `timers`, `schedule_cron`, `context_*`, `crypto_*`:
+  browser-native implementations in the browser host adapter
+
+These are host shims, not raw WasmEdge socket imports. When an embedding host
+needs to override the reference behavior, pass `capabilityAdapters` keyed by
+the canonical capability ids. That same generic async capability boundary is
+shared by `BrowserHost`, `NodeHost`, `createRuntimeHost()`, `loadModule(...)`,
+and `createBrowserModuleHarness(...)`.
+
+## Current Boundary
+
+One binary can load in both browser and WasmEdge today when it stays within the
+shared profile:
+
+- standalone WASI imports
+- optional sync `space_data_module_host` imports
+- command invoke surfaces through `_start` on WasmEdge and direct or command
+  invoke surfaces in the browser harness
+- no Emscripten pthread imports
+
+Not browser-portable from the same raw guest binary:
+
+- WasmEdge-native socket/TLS extension imports
+- pthread-oriented `env.*` imports
+- raw guest async hostcalls that need a broader ABI than the current sync
+  `space_data_module_host` bridge
+
+Today’s portable split is:
+
+- the guest-visible `space_data_module_host` import remains a sync-only subset for sync-safe
+  operations
+- the host and harness APIs can still await filesystem, network, IPFS, and
+  protocol adapters through the generic async capability boundary in both
+  browser and Node-hosted test/runtime flows
+
+For browser-hosted networking and IPFS/protocol work, use the browser edge
+shims or host-delegated adapters instead of relying on raw WasmEdge socket
+extensions.
+
+## Checked-In Demo
+
+The canonical example is:
+
+- [`examples/isomorphic-loader/README.md`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/README.md)
+
+That example includes:
+
+- manifest: [`examples/isomorphic-loader/manifest.json`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/manifest.json)
+- guest source: [`examples/isomorphic-loader/module.c`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/module.c)
+- build script: [`examples/isomorphic-loader/build-demo.mjs`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/build-demo.mjs)
+- browser loader: [`examples/isomorphic-loader/browser-demo.mjs`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/browser-demo.mjs)
+- browser page: [`examples/isomorphic-loader/browser-demo.html`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/browser-demo.html)
+- WasmEdge loader: [`examples/isomorphic-loader/wasmedge-demo.mjs`](/Users/tj/software/space-data-module-sdk/examples/isomorphic-loader/wasmedge-demo.mjs)
+
+Both demos load the same generated artifact:
+
+- `examples/isomorphic-loader/generated/dist/isomorphic/module.wasm`
+
+## Streaming Into The Same Artifact
+
+If that shared artifact owns resident state, such as an SDN module that imports
+`flatsql` internally, stream raw FlatBuffer frames into the live instance with
+`createModuleFlatBufferStreamPump(...)`.
+
+Use:
+
+- a persistent `direct` browser harness for browser-resident state
+- a persistent runtime-backed harness on the server/WasmEdge side
+- chunked size-prefixed FlatBuffer stream input
+- many small invokes, not one monolithic invoke envelope
+
+That path is documented in
+[`docs/flatsql-streaming-standard.md`](/Users/tj/software/space-data-module-sdk/docs/flatsql-streaming-standard.md).

package/docs/flatsql-streaming-standard.md

@@ -0,0 +1,341 @@
+# FlatBuffer Streaming Into FlatSQL
+
+This document defines the recommended standard for getting streamed
+FlatBuffer payloads into FlatSQL-backed storage while keeping the same model
+portable across browser, server, OrbPro, and WasmEdge.
+
+## Short Version
+
+Use three layers, not one:
+
+1. Transport stream:
+   little-endian `u32` size prefix + one FlatBuffer payload per frame
+2. Module invoke ABI:
+   SDS `$PIV` request/response envelopes with `TAB` frame descriptors
+3. Durable storage identity:
+   host-owned append-only row handles and host-owned runtime regions
+
+Do not treat a module-owned mutable FlatSQL database as the canonical ABI.
+That can exist as an adapter or example, but it is not the durable cross-host
+contract.
+
+## Canonical Transport
+
+The canonical transport for streamed FlatBuffers is:
+
+- `4-byte little-endian payload length`
+- followed by one FlatBuffer payload
+- repeated until end of stream
+- payload bytes stay binary end-to-end; the canonical ingest path does not
+  transcode frames through JSON
+
+Each payload must carry a readable FlatBuffer file identifier. The runtime uses
+that identifier to derive the durable `schemaFileId`.
+
+This transport is intentionally separate from module invocation. It is the
+right shape for:
+
+- WebSocket feeds
+- HTTP chunk/file ingestion
+- local file replay
+- browser worker or server-side stream processing
+
+It is not the same thing as a SDS `$PIV` invoke envelope.
+
+## Canonical Durable Storage
+
+The durable storage model is host-owned.
+
+Standards records are addressed by:
+
+- `($SCHEMA_FILE_ID, rowId)`
+
+Rules:
+
+- `rowId` is append-only
+- `rowId` is never reused
+- logical updates create new rows
+- "latest" or "upsert" views are host-side projections or indexes
+- row payloads remain raw binary or native in-memory values; the canonical
+  runtime-host row path does not stringify payloads
+
+High-performance derived runtime state is addressed by:
+
+- `(regionId, recordIndex)`
+
+Rules:
+
+- the host allocates or registers regions
+- regions are fixed-layout aligned-binary buffers
+- regions are not indexed in FlatSQL
+- raw pointers remain internal execution details only
+
+The stable storage ABI lives in [`schemas/HostStorageAbi.fbs`](../schemas/HostStorageAbi.fbs).
+
+## Canonical Module ABI
+
+When streamed payloads must cross into a module, use the invoke ABI:
+
+- SDS `$PIV` `REQUEST`
+- SDS `$PIV` `RESPONSE`
+- SDS `TAB` frame descriptors
+
+Each input frame should preserve:
+
+- `portId`
+- `typeRef`
+- `streamId`
+- `sequence`
+- `endOfStream`
+- alignment metadata when required
+
+This is the correct way to tell a module "these frames belong to the same
+logical stream."
+
+It is not a byte-streaming transport. The current invoke ABI is still
+buffer-oriented:
+
+- the full request is materialized before `plugin_invoke_stream(...)`
+- the full response is materialized before it is returned
+
+So a single 1 GiB SDS `$PIV` invoke request is not the intended path.
+
+## FlatSQL-Specific Rules
+
+If the goal is "stream FlatBuffers into FlatSQL", standardize the host-side
+behavior first:
+
+1. split the outer size-prefixed transport stream into payload frames
+2. derive `schemaFileId` from the FlatBuffer file identifier
+3. append immutable host-owned rows
+4. maintain host-side logical indexes or latest-record projections as needed
+5. expose query and row-resolution APIs on top of that store
+
+That means the durable primitive is `append`, not `upsert`.
+
+`upsert_records` is allowed as a higher-level adapter surface, but only if it
+is defined as:
+
+- append new immutable row(s)
+- update a logical index/view
+- never rewrite an existing durable `rowId`
+
+If a module or flow needs fast numeric state, keep that data in runtime regions,
+not in FlatSQL tables.
+
+## Browser and WasmEdge
+
+The same transport/storage model should be used in both:
+
+- browser: host-side stream ingest + host-owned rows/regions
+- WasmEdge: host-side stream ingest + host-owned rows/regions
+
+Use module invocation for:
+
+- compute batches
+- typed transformation steps
+- stateful direct-surface module sessions
+
+Use host-side stream ingest for:
+
+- large continuous feeds
+- catalog replay
+- ingestion into host-owned FlatSQL-backed stores
+- any workload that should not be forced through one contiguous invoke buffer
+
+The new helper for this outer transport path is
+`createFlatBufferStreamIngestor(...)`, exported from
+`space-data-module-sdk/runtime-host`.
+
+## Streaming Into A Module-Owned FlatSQL Engine
+
+If the goal is not a host-owned durable store, but a stateful SDN module that
+imports `flatsql` internally and owns its own in-memory query state, use a
+persistent direct-surface module instance plus chunked binary invokes.
+
+Rules:
+
+- do not change `flatsql` itself
+- import `flatsql` inside the SDN module implementation
+- keep the module instance resident across invokes
+- feed it many small FlatBuffer frames, not one giant invoke envelope
+- use command surface only for stateless one-shot work; use direct surface for
+  resident FlatSQL state
+
+The SDK helper for this path is `createModuleFlatBufferStreamPump(...)`,
+exported from `space-data-module-sdk/testing` and the browser/root package
+surfaces.
+
+That helper:
+
+- accepts the same outer size-prefixed FlatBuffer stream chunks
+- decodes frames incrementally
+- emits small SDS `$PIV` request batches into a live module instance
+- preserves `streamId`, `sequence`, and final `endOfStream`
+- never routes payloads through JSON
+
+Example:
+
+```js
+import {
+  createBrowserModuleHarness,
+  createModuleFlatBufferStreamPump,
+} from "space-data-module-sdk";
+
+const harness = await createBrowserModuleHarness({
+  wasmSource,
+  surface: "direct",
+});
+
+const pump = createModuleFlatBufferStreamPump({
+  harness,
+  methodId: "upsert_records",
+  portId: "records",
+  maxFramesPerInvoke: 64,
+  typeResolver(_payload, context) {
+    return {
+      acceptsAnyFlatbuffer: true,
+      fileIdentifier: context.rawFileIdentifier,
+    };
+  },
+});
+
+await pump.pushBytes(chunkA);
+await pump.pushBytes(chunkB);
+await pump.finish();
+```
+
+This is the right shape for OrbPro-style browser ingest when:
+
+- the raw FlatBuffer payloads are the source of truth
+- the UI/runtime should not materialize JS record mirrors
+- a resident module should own FlatSQL/query state directly
+
+It is still batch-oriented per invoke, but it removes the architectural
+anti-pattern of building one monolithic request envelope for a long-running
+stream.
+
+## Example Import Path
+
+```js
+import {
+  createFlatBufferStreamIngestor,
+  createRuntimeHost,
+} from "space-data-module-sdk/runtime-host";
+
+const host = createRuntimeHost();
+const ingestor = createFlatBufferStreamIngestor({
+  rows: host.rows,
+});
+
+ingestor.pushBytes(chunkA);
+ingestor.pushBytes(chunkB);
+ingestor.finish();
+
+const rows = host.rows.listRows("OMM");
+```
+
+## Performance Guidance
+
+There are two distinct performance questions:
+
+1. Outer transport ingest throughput
+2. Module invoke throughput
+
+Do not mix them into one benchmark.
+
+### Outer Transport Benchmark
+
+Use the runtime-host stream ingestor.
+
+Commands:
+
+```bash
+npm run test:stream-ingest
+npm run benchmark:stream-1gib
+```
+
+Optional tuning:
+
+- `SPACE_DATA_MODULE_SDK_STREAM_BENCH_BYTES`
+- `SPACE_DATA_MODULE_SDK_STREAM_BENCH_PAYLOAD_BYTES`
+- `SPACE_DATA_MODULE_SDK_STREAM_BENCH_CHUNK_BYTES`
+
+The 1 GiB benchmark is env-gated on purpose. It is a local stress path, not a
+default suite member.
+
+### Module Invoke Benchmark
+
+Benchmark direct invoke separately with many smaller requests and a tiny
+response. Treat the current invoke ABI as batch-oriented, not as a raw stream
+transport.
+
+Recommended total sizes:
+
+- CI / regular local: `1 MiB`, `8 MiB`, `32 MiB`, `128 MiB`
+- local stress: `256 MiB` or higher in chunked totals
+
+Avoid a single 1 GiB request envelope on the current codepath.
+
+For the resident module-ingest shape, use:
+
+```bash
+npm run test:module-stream
+npm run benchmark:module-stream-1gib
+```
+
+That benchmark exercises the chunked module stream-pump path, not a single huge
+invoke buffer.
+
+## Current Non-Canonical Example
+
+[`examples/flatsql-store-local`](../examples/flatsql-store-local) is still
+useful as an adapter example, but it is not the canonical durable identity
+model. It exposes a module-owned mutable logical database. The canonical model
+for cross-host durability is host-owned rows plus host-owned runtime regions.
+
+## Body-Reference Delivery (`"deliver":"ref"`)
+
+Loop C.5c adds an OPTIONAL near-zero-copy egress mode for capability
+hostcalls whose result is an aligned stream that the module passes through
+VERBATIM to an HTTP response body (the retrieval flow's flatbuffer branch).
+The election is always the GUEST's; hosts never invent it.
+
+1. The module adds `"deliver":"ref"` to the hostcall payload
+   (`storage.flatsql_query_stream` / `storage.flatsql_epoch_stream`).
+2. A ref-capable host keeps the materialized bytes in ITS memory, registers
+   them on the calling instance's hostcall-bridge body-ref registry, and
+   answers with NO binary segment:
+
+   ```json
+   {"ok":true,"result":{"rows":N,"columns":M,
+     "ref":{"token":T,"size":S,"frames":F,"fnv1a64":"<16 hex>"}}}
+   ```
+
+   - `token` — opaque, single-use, scoped to this module instance's bridge
+     and the current exchange.
+   - `size` — byte length of the referenced stream.
+   - `frames` — size-prefixed frame count, skipping zero-length prefixes
+     (the `x-sdn-record-count` rule); omitted when the framing is malformed.
+   - `fnv1a64` — word-folded FNV-1a 64 content hash (`fnv1a64Hex` in
+     `space-data-module-sdk/http` is the reference implementation; it is
+     bit-identical to foundation/decision-gate's in-wasm hasher, so
+     reference-mode entity tags equal hashed-stream entity tags).
+
+   A host that does not understand `deliver` simply returns the byte
+   segment as always; modules MUST fall back to verbatim byte passthrough.
+3. The module forwards the reference in-band as a small JSON descriptor
+   frame `{"$sdnbodyref":1,"token":T,"size":S,"frames":F,"fnv1a64":"…"}`
+   (an aligned stream can never collide: it starts with a `u32le` size
+   prefix, not `{`).
+4. `foundation/http-respond` recognizes the descriptor on its body port and
+   emits `$HTR` with `BODY_REF_TOKEN`/`BODY_REF_SIZE` set and NO inline
+   body (schema `HttpResponseAbi.fbs`).
+5. The host egress substitutes the byte buffer registered under the token
+   (Go: `flowrt.htrPipe` + `modulert.HostBridge.TakeBodyRef`; JS:
+   `createBodyRefRegistry` from `space-data-module-sdk/http`). Tokens are
+   single-use; hosts drop unconsumed references at end-of-exchange (304 and
+   error paths never consume theirs).
+
+The stream bytes therefore never enter the flow's linear memory: the only
+byte movement left on a warm request is the host's socket write.