space-data-module-sdk 0.8.20 → 0.8.22

package/src/host/opfsIoBackend.js

@@ -0,0 +1,280 @@
+// OPFS backend for the FlatSQL I/O server (docs/architecture/flatsql-partition-store.md
+// §5.5, A7, A13, 22.3a-6).
+//
+// Every open is async by construction (getDirectoryHandle, getFileHandle,
+// createSyncAccessHandle), and runs inside the I/O worker, which awaits it
+// without blocking any other guest thread. The files it returns ARE
+// FileSystemSyncAccessHandles, whose read/write/truncate/flush/getSize are
+// synchronous and serve the guest's blocking imports directly.
+//
+// Handle modes (A7). A sync access handle is exclusive per file by default. A
+// reader I/O worker can only read a file the writer I/O worker holds when both
+// open it in "readwrite-unsafe" mode. Chromium supports the mode; Firefox 155
+// and WebKit 26.6 ignore it (measured: the second handle fails). The owner
+// ruled (§22.4-6) that a browser without shared handle modes gets no local
+// store; `probeOpfsSharedHandleModes` is the gate's measurement.
+//
+// Directory durability on OPFS is best effort (22.3a-4); CREATE_PARENTS
+// creates the directories and cannot fsync them.
+
+import {
+  FLATSQL_IO_CREATE,
+  FLATSQL_IO_CREATE_PARENTS,
+  FLATSQL_IO_ERR_GENERIC,
+  FLATSQL_IO_EXCL,
+  FLATSQL_IO_TRUNC,
+  FlatsqlIoStatusError,
+} from "./flatsqlIoContract.js";
+
+export const OPFS_SHARED_HANDLE_MODE = "readwrite-unsafe";
+
+function notFound(message) {
+  const error = new Error(message);
+  error.name = "NotFoundError";
+  return error;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Does this context support shared sync access handles? Opens two
+ * "readwrite-unsafe" handles on one probe file; both must succeed and report
+ * the mode. Resolves to { supported, detail }.
+ */
+export async function probeOpfsSharedHandleModes(root, name = ".sdm-mode-probe") {
+  let first = null;
+  let second = null;
+  try {
+    const file = await root.getFileHandle(name, { create: true });
+    first = await file.createSyncAccessHandle({ mode: OPFS_SHARED_HANDLE_MODE });
+    if (first.mode !== OPFS_SHARED_HANDLE_MODE) {
+      return { supported: false, detail: "mode option ignored" };
+    }
+    second = await file.createSyncAccessHandle({ mode: OPFS_SHARED_HANDLE_MODE });
+    return { supported: true, detail: "two readwrite-unsafe handles" };
+  } catch (error) {
+    return { supported: false, detail: `${error?.name ?? "Error"}: ${error?.message ?? error}` };
+  } finally {
+    try {
+      second?.close();
+    } catch {
+      // ignore
+    }
+    try {
+      first?.close();
+    } catch {
+      // ignore
+    }
+    try {
+      await root.removeEntry(name);
+    } catch {
+      // ignore
+    }
+  }
+}
+
+/**
+ * Does this context's sync access handle accept views over a
+ * SharedArrayBuffer (direct shared-memory transfer)? Resolves to a boolean.
+ */
+export async function probeOpfsSharedViews(root, name = ".sdm-view-probe") {
+  let handle = null;
+  try {
+    const file = await root.getFileHandle(name, { create: true });
+    handle = await file.createSyncAccessHandle();
+    const shared = new Uint8Array(new SharedArrayBuffer(16));
+    shared.fill(0x5a);
+    const written = handle.write(shared, { at: 0 });
+    const back = new Uint8Array(new SharedArrayBuffer(16));
+    const read = handle.read(back, { at: 0 });
+    return written === 16 && read === 16 && back[15] === 0x5a;
+  } catch {
+    return false;
+  } finally {
+    try {
+      handle?.close();
+    } catch {
+      // ignore
+    }
+    try {
+      await root.removeEntry(name);
+    } catch {
+      // ignore
+    }
+  }
+}
+
+/**
+ * Create the OPFS backend.
+ *
+ * @param {object} [options]
+ * @param {string} [options.rootDirectory] a directory below the origin's OPFS
+ *   root that every path is confined to (created on init).
+ * @param {"auto"|"shared"|"exclusive"} [options.handleMode="auto"] "shared"
+ *   opens every handle "readwrite-unsafe" (requires support), "exclusive"
+ *   uses the default exclusive handles, "auto" picks shared when supported.
+ * @param {(path: string) => number} [options.openDelayMs] test hook: delay an
+ *   open by this many milliseconds (acceptance: an open that resolves after
+ *   500 ms blocks only its caller).
+ * @param {FileSystemDirectoryHandle} [options.root] use this directory as the
+ *   root instead of navigator.storage.getDirectory().
+ * @param {{ attempts: number, baseMs?: number }} [options.busyRetry] retry a
+ *   sync-handle creation refused because another context still holds the file
+ *   (A37 leader takeover), with exponential backoff. Default: no retry.
+ */
+export function createOpfsIoBackend(options = {}) {
+  let root = options.root ?? null;
+  let sharedModes = false;
+  let sharedViews = null;
+  const handleMode = options.handleMode ?? "auto";
+  const openDelayMs = typeof options.openDelayMs === "function" ? options.openDelayMs : null;
+  const directoryCache = new Map();
+  // A37: a new leader retries NoModificationAllowedError (WebKit:
+  // InvalidStateError) with backoff while the previous leader's handles close.
+  const busyRetry = {
+    attempts: Math.max(0, options.busyRetry?.attempts ?? 0),
+    baseMs: Math.max(1, options.busyRetry?.baseMs ?? 10),
+  };
+
+  async function createAccessHandle(fileHandle) {
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        return sharedModes
+          ? await fileHandle.createSyncAccessHandle({ mode: OPFS_SHARED_HANDLE_MODE })
+          : await fileHandle.createSyncAccessHandle();
+      } catch (error) {
+        const busy =
+          error?.name === "NoModificationAllowedError" || error?.name === "InvalidStateError";
+        if (!busy || attempt >= busyRetry.attempts) throw error;
+        await sleep(busyRetry.baseMs * 2 ** Math.min(attempt, 8));
+      }
+    }
+  }
+
+  async function init() {
+    if (!root) {
+      if (typeof navigator === "undefined" || !navigator.storage?.getDirectory) {
+        throw new FlatsqlIoStatusError(FLATSQL_IO_ERR_GENERIC, "OPFS is unavailable here");
+      }
+      root = await navigator.storage.getDirectory();
+      if (options.rootDirectory) {
+        for (const part of String(options.rootDirectory).split("/").filter(Boolean)) {
+          root = await root.getDirectoryHandle(part, { create: true });
+        }
+      }
+    }
+    const probe = await probeOpfsSharedHandleModes(root);
+    sharedViews = await probeOpfsSharedViews(root);
+    if (handleMode === "shared" && !probe.supported) {
+      throw new FlatsqlIoStatusError(
+        FLATSQL_IO_ERR_GENERIC,
+        `shared OPFS handle modes are unsupported here (${probe.detail})`,
+      );
+    }
+    sharedModes = handleMode !== "exclusive" && probe.supported;
+    directoryCache.set("", root);
+    return { sharedModes, sharedModesDetail: probe.detail, sharedViews };
+  }
+
+  async function directoryFor(components, create) {
+    const key = components.join("/");
+    const cached = directoryCache.get(key);
+    if (cached) return cached;
+    let dir = root;
+    let prefix = "";
+    for (const part of components) {
+      prefix = prefix ? `${prefix}/${part}` : part;
+      const known = directoryCache.get(prefix);
+      if (known) {
+        dir = known;
+        continue;
+      }
+      dir = await dir.getDirectoryHandle(part, { create });
+      directoryCache.set(prefix, dir);
+    }
+    return dir;
+  }
+
+  async function fileHandleIfExists(dir, name) {
+    try {
+      return await dir.getFileHandle(name);
+    } catch (error) {
+      if (error?.name === "NotFoundError") return null;
+      throw error;
+    }
+  }
+
+  return {
+    kind: "opfs",
+    init,
+    get sharedModes() {
+      return sharedModes;
+    },
+    get sharedViews() {
+      return sharedViews;
+    },
+    async openFile(path, components, flags) {
+      if (!root) await init();
+      if (openDelayMs) {
+        const delay = openDelayMs(path);
+        if (delay > 0) await sleep(delay);
+      }
+      const create = (flags & FLATSQL_IO_CREATE) !== 0;
+      const parents = create && (flags & FLATSQL_IO_CREATE_PARENTS) !== 0;
+      const dir = await directoryFor(components.slice(0, -1), parents);
+      const name = components[components.length - 1];
+      let fileHandle = null;
+      if (flags & FLATSQL_IO_EXCL) {
+        if (await fileHandleIfExists(dir, name)) {
+          throw new FlatsqlIoStatusError(FLATSQL_IO_ERR_GENERIC, `${path} exists`);
+        }
+      }
+      fileHandle = await dir.getFileHandle(name, { create });
+      const access = await createAccessHandle(fileHandle);
+      if (flags & FLATSQL_IO_TRUNC) {
+        access.truncate(0);
+      }
+      return access;
+    },
+    async exists(path, components) {
+      if (!root) await init();
+      let dir;
+      try {
+        dir = await directoryFor(components.slice(0, -1), false);
+      } catch (error) {
+        if (error?.name === "NotFoundError" || error?.name === "TypeMismatchError") return false;
+        throw error;
+      }
+      const name = components[components.length - 1];
+      if (await fileHandleIfExists(dir, name)) return true;
+      try {
+        await dir.getDirectoryHandle(name);
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    async remove(path, components) {
+      if (!root) await init();
+      let dir;
+      try {
+        dir = await directoryFor(components.slice(0, -1), false);
+      } catch (error) {
+        if (error?.name === "TypeMismatchError") throw notFound(path);
+        throw error;
+      }
+      await dir.removeEntry(components[components.length - 1]);
+    },
+    /** Remove everything below the root (tests, and a store wipe). */
+    async clear() {
+      if (!root) await init();
+      for await (const name of root.keys()) {
+        await root.removeEntry(name, { recursive: true });
+      }
+      directoryCache.clear();
+      directoryCache.set("", root);
+    },
+  };
+}

package/src/host/opfsIoWorker.mjs

@@ -0,0 +1,223 @@
+// The FlatSQL I/O worker entry (docs/architecture/flatsql-partition-store.md
+// §5.5, A7, A36, A38, A39).
+//
+// One dedicated worker that holds every file handle for the guest threads that
+// talk to it over a SAB request ring (sabIoChannel.js). It never blocks: sync
+// reads and writes run directly on OPFS sync access handles, into the guests'
+// shared memory; opens, unlinks and probes are awaited on the event loop while
+// the requesting guest thread waits on its own slot.
+//
+// Roles (A7): a WRITER I/O worker per writer holds that writer's active files
+// and splits writes into chunks of 256 KiB or less; READER I/O workers (1, or 2
+// sharded by path hash when hardwareConcurrency >= 8) hold sealed and immutable
+// files. The role only sets defaults; the protocol is the same.
+//
+// Backends: "opfs" (the local store) or "memory" (the dashboard window store:
+// nothing reaches OPFS, and `reset` drops it on a UI-state change).
+//
+// Runs as a browser Worker (module, or classic from the self-contained blob
+// bundle, A39) and as a Node worker_threads worker (tests, memory backend).
+//
+// Controller messages (see flatsqlIoWorkers.js):
+//   {t:"init", buffer, backend, role, chunkBytes, doorbell, mirror, rootDirectory,
+//    handleMode, memoryMaxBytes, openDelay, lock, busyRetry}
+//                                                    -> {t:"ready", ok, info|error}
+//   {t:"attach", instanceId, memory}                 -> {t:"attached", instanceId, ok}
+//   {t:"detach", instanceId}
+//   {t:"preopen", id, paths, flags}                  -> {t:"preopened", id, statuses}
+//   {t:"release-preopen", paths}
+//   {t:"stats", id}                                  -> {t:"stats", id, stats, inventory}
+//   {t:"reset", id}                                  -> {t:"reset", id}
+//   {t:"clear", id}                                  -> {t:"cleared", id}
+//   {t:"stop", id}                                   -> {t:"stopped", id}
+
+import { createFlatsqlIoServer, DEFAULT_FLATSQL_IO_CHUNK_BYTES } from "./flatsqlIoServer.js";
+import { createFlatsqlIoMemoryBackend } from "./flatsqlIoMemoryBackend.js";
+import { createOpfsIoBackend } from "./opfsIoBackend.js";
+import { createSabIoMirrorMatcher } from "./sabIoMirror.js";
+import { NODE_BUILTIN_PREFIX } from "./nodeBuiltinSpecifier.js";
+
+const IS_NODE =
+  typeof process !== "undefined" && !!process.release && process.release.name === "node";
+
+let server = null;
+let backend = null;
+let releaseLock = null;
+
+// A37: the store's Web Lock is taken inside the writer I/O worker, so it lives
+// exactly as long as the handles; `stop` closes the handles first, then
+// releases it. Resolves true when held, false when `ifAvailable` found it taken.
+function acquireLock({ name, ifAvailable = false }) {
+  if (typeof navigator === "undefined" || !navigator.locks) {
+    return Promise.reject(new Error("Web Locks are unavailable in this context"));
+  }
+  return new Promise((resolve, reject) => {
+    navigator.locks
+      .request(name, { mode: "exclusive", ifAvailable }, (lock) => {
+        if (!lock) {
+          resolve(false);
+          return undefined;
+        }
+        resolve(true);
+        return new Promise((release) => {
+          releaseLock = release;
+        });
+      })
+      .catch(reject);
+  });
+}
+
+function describeError(error) {
+  return `${error?.name ?? "Error"}: ${error?.message ?? String(error)}`;
+}
+
+function buildOpenDelay(openDelay) {
+  if (!openDelay || !openDelay.pattern || !(openDelay.ms > 0)) return undefined;
+  const pattern = new RegExp(openDelay.pattern);
+  return (path) => (pattern.test(path) ? openDelay.ms : 0);
+}
+
+async function handleInit(message, post) {
+  const role = message.role === "reader" ? "reader" : "writer";
+  if (message.lock?.name) {
+    const held = await acquireLock(message.lock);
+    if (!held) {
+      const error = new Error(`the store lock ${message.lock.name} is held by another context`);
+      error.name = "LockUnavailableError";
+      throw error;
+    }
+  }
+  if (message.backend === "memory") {
+    backend = createFlatsqlIoMemoryBackend({ maxBytes: message.memoryMaxBytes });
+  } else {
+    backend = createOpfsIoBackend({
+      rootDirectory: message.rootDirectory,
+      handleMode: message.handleMode ?? "auto",
+      openDelayMs: buildOpenDelay(message.openDelay),
+      busyRetry: message.busyRetry ?? undefined,
+    });
+  }
+  const info = { backend: backend.kind, role, lock: message.lock?.name ?? null };
+  if (backend.init) {
+    Object.assign(info, await backend.init());
+  }
+  const mirror = message.mirror?.buffer
+    ? {
+        buffer: message.mirror.buffer,
+        match: createSabIoMirrorMatcher(message.mirror.suffixes ?? ["/h.fsh"]),
+        write: message.mirror.write ?? role === "writer",
+      }
+    : null;
+  server = createFlatsqlIoServer({
+    buffer: message.buffer,
+    backend,
+    chunkBytes: message.chunkBytes ?? DEFAULT_FLATSQL_IO_CHUNK_BYTES,
+    doorbell: message.doorbell ?? "auto",
+    mirror,
+    onEvent(event) {
+      if (event.type === "loop-error" || event.type === "step-error") {
+        post({ t: "event", event: { type: event.type, error: describeError(event.error) } });
+      }
+    },
+  });
+  info.doorbellMode = server.doorbellMode === 1 ? "message" : "atomics";
+  // The loop runs until stop; a crash of the loop itself is a worker error the
+  // supervisor sees (A36).
+  server.start().catch((error) => {
+    setTimeout(() => {
+      throw error;
+    }, 0);
+  });
+  return info;
+}
+
+async function dispatch(message, post) {
+  switch (message?.t) {
+    case "init":
+      try {
+        const info = await handleInit(message, post);
+        post({ t: "ready", ok: true, info });
+      } catch (error) {
+        releaseLock?.();
+        releaseLock = null;
+        post({
+          t: "ready",
+          ok: false,
+          error: describeError(error),
+          lockUnavailable: error?.name === "LockUnavailableError",
+        });
+      }
+      return;
+    case "attach":
+      post({
+        t: "attached",
+        instanceId: message.instanceId,
+        ok: !!server?.attachMemory(message.instanceId, message.memory),
+      });
+      return;
+    case "detach":
+      server?.detachMemory(message.instanceId);
+      return;
+    case "preopen": {
+      const statuses = server ? await server.preopen(message.paths ?? [], message.flags) : [];
+      post({ t: "preopened", id: message.id, statuses });
+      return;
+    }
+    case "release-preopen":
+      server?.releasePreopen(message.paths ?? []);
+      return;
+    case "stats":
+      post({
+        t: "stats",
+        id: message.id,
+        stats: server ? { ...server.stats } : null,
+        inventory: server ? server.inventory() : null,
+        usage: backend?.usage ? backend.usage() : null,
+      });
+      return;
+    case "reset":
+      backend?.reset?.();
+      post({ t: "reset", id: message.id });
+      return;
+    case "clear":
+      try {
+        await backend?.clear?.();
+        post({ t: "cleared", id: message.id, ok: true });
+      } catch (error) {
+        post({ t: "cleared", id: message.id, ok: false, error: describeError(error) });
+      }
+      return;
+    case "stop":
+      await server?.stop();
+      server = null;
+      // Handles are closed; only now may another context take the store.
+      releaseLock?.();
+      releaseLock = null;
+      post({ t: "stopped", id: message.id });
+      return;
+    default:
+  }
+}
+
+function startBrowser() {
+  const post = (value) => self.postMessage(value);
+  self.onmessage = (event) => {
+    dispatch(event.data, post);
+  };
+}
+
+async function startNode() {
+  const specifier = NODE_BUILTIN_PREFIX + "worker_threads";
+  const { parentPort } = await import(/* @vite-ignore */ /* webpackIgnore: true */ specifier);
+  if (!parentPort) return;
+  const post = (value) => parentPort.postMessage(value);
+  parentPort.on("message", (message) => {
+    dispatch(message, post);
+  });
+}
+
+if (IS_NODE) {
+  startNode();
+} else if (typeof self !== "undefined" && typeof self.postMessage === "function" && typeof window === "undefined") {
+  startBrowser();
+}