tinybase 5.1.0-beta.0 → 5.1.0-beta.1

package/umd/synchronizers/synchronizer-ws-server/index.js

@@ -11,7 +11,12 @@
 
   const EMPTY_STRING = '';
   const UTF8 = 'utf8';
+  const UNDEFINED = '\uFFFC';
 
+  const promise = Promise;
+  const GLOBAL = globalThis;
+  const math = Math;
+  const mathFloor = math.floor;
   const isUndefined = (thing) => thing == void 0;
   const ifNotUndefined = (value, then, otherwise) =>
     isUndefined(value) ? otherwise?.() : then(value);
@@ -19,14 +24,46 @@
   const slice = (arrayOrString, start, end) => arrayOrString.slice(start, end);
   const size = (arrayOrString) => arrayOrString.length;
   const test = (regex, subject) => regex.test(subject);
+  const promiseNew = (resolver) => new promise(resolver);
+  const errorNew = (message) => {
+    throw new Error(message);
+  };
 
   const arrayForEach = (array, cb) => array.forEach(cb);
+  const arrayMap = (array, cb) => array.map(cb);
   const arrayReduce = (array, cb, initial) => array.reduce(cb, initial);
+  const arrayClear = (array, to) => array.splice(0, to);
   const arrayPush = (array, ...values) => array.push(...values);
   const arrayShift = (array) => array.shift();
 
   const object = Object;
+  const getPrototypeOf = (obj) => object.getPrototypeOf(obj);
+  const objEntries = object.entries;
+  const isObject = (obj) =>
+    !isUndefined(obj) &&
+    ifNotUndefined(
+      getPrototypeOf(obj),
+      (objPrototype) =>
+        objPrototype == object.prototype ||
+        isUndefined(getPrototypeOf(objPrototype)),
+
+      /* istanbul ignore next */
+      () => true,
+    );
+  const objIds = object.keys;
   const objFreeze = object.freeze;
+  const objNew = (entries = []) => object.fromEntries(entries);
+  const objHas = (obj, id) => id in obj;
+  const objForEach = (obj, cb) =>
+    arrayForEach(objEntries(obj), ([id, value]) => cb(value, id));
+  const objSize = (obj) => size(objIds(obj));
+  const objIsEmpty = (obj) => isObject(obj) && objSize(obj) == 0;
+  const objEnsure = (obj, id, getDefaultValue) => {
+    if (!objHas(obj, id)) {
+      obj[id] = getDefaultValue();
+    }
+    return obj[id];
+  };
 
   const collSizeN = (collSizer) => (coll) =>
     arrayReduce(
@@ -80,7 +117,420 @@
       },
     );
 
+  const jsonStringWithUndefined = (obj) =>
+    JSON.stringify(obj, (_key, value) =>
+      value === void 0 ? UNDEFINED : value,
+    );
+  const jsonParseWithUndefined = (str) =>
+    JSON.parse(str, (_key, value) => (value === UNDEFINED ? void 0 : value));
+
   const MESSAGE_SEPARATOR = '\n';
+  const ifPayloadValid = (payload, then) => {
+    const splitAt = payload.indexOf(MESSAGE_SEPARATOR);
+    splitAt !== -1
+      ? then(slice(payload, 0, splitAt), slice(payload, splitAt + 1))
+      : 0;
+  };
+  const receivePayload = (payload, receive) =>
+    ifPayloadValid(payload, (fromClientId, remainder) =>
+      receive(fromClientId, ...jsonParseWithUndefined(remainder)),
+    );
+  const createPayload = (toClientId, ...args) =>
+    createRawPayload(toClientId ?? EMPTY_STRING, jsonStringWithUndefined(args));
+  const createRawPayload = (toClientId, remainder) =>
+    toClientId + MESSAGE_SEPARATOR + remainder;
+
+  new GLOBAL.TextEncoder();
+
+  const newStamp = (value, time) => (time ? [value, time] : [value]);
+  const getLatestTime = (time1, time2) =>
+    ((time1 ?? '') > (time2 ?? '') ? time1 : time2) ?? '';
+  const stampNewObj = (time = EMPTY_STRING) => newStamp(objNew(), time);
+
+  const scheduleRunning = mapNew();
+  const scheduleActions = mapNew();
+  const getStoreFunctions = (persist = 1 /* StoreOnly */, store) =>
+    persist != 1 /* StoreOnly */ && store.isMergeable()
+      ? [
+          1,
+          store.getMergeableContent,
+          store.getTransactionMergeableChanges,
+          ([[changedTables], [changedValues]]) =>
+            !objIsEmpty(changedTables) || !objIsEmpty(changedValues),
+          store.setDefaultContent,
+        ]
+      : persist != 2 /* MergeableStoreOnly */
+        ? [
+            0,
+            store.getContent,
+            store.getTransactionChanges,
+            ([changedTables, changedValues]) =>
+              !objIsEmpty(changedTables) || !objIsEmpty(changedValues),
+            store.setContent,
+          ]
+        : errorNew('Store type not supported by this Persister');
+  const createCustomPersister = (
+    store,
+    getPersisted,
+    setPersisted,
+    addPersisterListener,
+    delPersisterListener,
+    onIgnoredError,
+    persist,
+    extra = {},
+    scheduleId = [],
+  ) => {
+    let loadSave = 0;
+    let loads = 0;
+    let saves = 0;
+    let action;
+    let autoLoadHandle;
+    let autoSaveListenerId;
+    mapEnsure(scheduleRunning, scheduleId, () => 0);
+    mapEnsure(scheduleActions, scheduleId, () => []);
+    const [
+      isMergeableStore,
+      getContent,
+      getChanges,
+      hasChanges,
+      setDefaultContent,
+    ] = getStoreFunctions(persist, store);
+    const run = async () => {
+      /* istanbul ignore else */
+      if (!mapGet(scheduleRunning, scheduleId)) {
+        mapSet(scheduleRunning, scheduleId, 1);
+        while (
+          !isUndefined(
+            (action = arrayShift(mapGet(scheduleActions, scheduleId))),
+          )
+        ) {
+          try {
+            await action();
+          } catch (error) {}
+        }
+        mapSet(scheduleRunning, scheduleId, 0);
+      }
+    };
+    const setContentOrChanges = (contentOrChanges) => {
+      (isMergeableStore && isArray(contentOrChanges?.[0])
+        ? contentOrChanges?.[2] === 1
+          ? store.applyMergeableChanges
+          : store.setMergeableContent
+        : contentOrChanges?.[2] === 1
+          ? store.applyChanges
+          : store.setContent)(contentOrChanges);
+    };
+    const load = async (initialContent) => {
+      /* istanbul ignore else */
+      if (loadSave != 2) {
+        loadSave = 1;
+        loads++;
+        await schedule(async () => {
+          try {
+            setContentOrChanges(await getPersisted());
+          } catch (error) {
+            if (initialContent) {
+              setDefaultContent(initialContent);
+            }
+          }
+          loadSave = 0;
+        });
+      }
+      return persister;
+    };
+    const startAutoLoad = async (initialContent) => {
+      await stopAutoLoad().load(initialContent);
+      try {
+        autoLoadHandle = addPersisterListener(async (content, changes) => {
+          if (changes || content) {
+            /* istanbul ignore else */
+            if (loadSave != 2) {
+              loadSave = 1;
+              loads++;
+              setContentOrChanges(changes ?? content);
+              loadSave = 0;
+            }
+          } else {
+            await load();
+          }
+        });
+      } catch (error) {}
+      return persister;
+    };
+    const stopAutoLoad = () => {
+      if (autoLoadHandle) {
+        delPersisterListener(autoLoadHandle);
+        autoLoadHandle = void 0;
+      }
+      return persister;
+    };
+    const isAutoLoading = () => !isUndefined(autoLoadHandle);
+    const save = async (changes) => {
+      /* istanbul ignore else */
+      if (loadSave != 1) {
+        loadSave = 2;
+        saves++;
+        await schedule(async () => {
+          try {
+            await setPersisted(getContent, changes);
+          } catch (error) {}
+          loadSave = 0;
+        });
+      }
+      return persister;
+    };
+    const startAutoSave = async () => {
+      await stopAutoSave().save();
+      autoSaveListenerId = store.addDidFinishTransactionListener(() => {
+        const changes = getChanges();
+        if (hasChanges(changes)) {
+          save(changes);
+        }
+      });
+      return persister;
+    };
+    const stopAutoSave = () => {
+      ifNotUndefined(autoSaveListenerId, store.delListener);
+      autoSaveListenerId = void 0;
+      return persister;
+    };
+    const isAutoSaving = () => !isUndefined(autoSaveListenerId);
+    const schedule = async (...actions) => {
+      arrayPush(mapGet(scheduleActions, scheduleId), ...actions);
+      await run();
+      return persister;
+    };
+    const getStore = () => store;
+    const destroy = () => {
+      arrayClear(mapGet(scheduleActions, scheduleId));
+      return stopAutoLoad().stopAutoSave();
+    };
+    const getStats = () => ({loads, saves});
+    const persister = {
+      load,
+      startAutoLoad,
+      stopAutoLoad,
+      isAutoLoading,
+      save,
+      startAutoSave,
+      stopAutoSave,
+      isAutoSaving,
+      schedule,
+      getStore,
+      destroy,
+      getStats,
+      ...extra,
+    };
+    return objFreeze(persister);
+  };
+
+  const MASK6 = 63;
+  const ENCODE =
+    '-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'.split(
+      EMPTY_STRING,
+    );
+  mapNew(arrayMap(ENCODE, (char, index) => [char, index]));
+  const encode = (num) => ENCODE[num & MASK6];
+
+  const getRandomValues = GLOBAL.crypto
+    ? (array) => GLOBAL.crypto.getRandomValues(array)
+    : /* istanbul ignore next */
+      (array) => arrayMap(array, () => mathFloor(math.random() * 256));
+  const getUniqueId = (length = 16) =>
+    arrayReduce(
+      getRandomValues(new Uint8Array(length)),
+      (uniqueId, number) => uniqueId + encode(number),
+      '',
+    );
+
+  const createCustomSynchronizer = (
+    store,
+    send,
+    registerReceive,
+    destroyImpl,
+    requestTimeoutSeconds,
+    onSend,
+    onReceive,
+    onIgnoredError,
+    extra = {},
+  ) => {
+    let persisterListener;
+    let sends = 0;
+    let receives = 0;
+    const pendingRequests = mapNew();
+    const sendImpl = (toClientId, requestId, message, body) => {
+      sends++;
+      send(toClientId, requestId, message, body);
+    };
+    const request = async (toClientId, message, body = EMPTY_STRING) =>
+      promiseNew((resolve, reject) => {
+        const requestId = getUniqueId();
+        const timeout = setTimeout(() => {
+          collDel(pendingRequests, requestId);
+          reject(
+            `No response from ${toClientId ?? 'anyone'} to ${requestId}, ` +
+              message,
+          );
+        }, requestTimeoutSeconds * 1e3);
+        mapSet(pendingRequests, requestId, [
+          toClientId,
+          (response, fromClientId) => {
+            clearTimeout(timeout);
+            collDel(pendingRequests, requestId);
+            resolve([response, fromClientId]);
+          },
+        ]);
+        sendImpl(toClientId, requestId, message, body);
+      });
+    const mergeTablesStamps = (tablesStamp, [tableStamps2, tablesTime2]) => {
+      objForEach(tableStamps2, ([rowStamps2, tableTime2], tableId) => {
+        const tableStamp = objEnsure(tablesStamp[0], tableId, stampNewObj);
+        objForEach(rowStamps2, ([cellStamps2, rowTime2], rowId) => {
+          const rowStamp = objEnsure(tableStamp[0], rowId, stampNewObj);
+          objForEach(
+            cellStamps2,
+            ([cell2, cellTime2], cellId) =>
+              (rowStamp[0][cellId] = newStamp(cell2, cellTime2)),
+          );
+          rowStamp[1] = getLatestTime(rowStamp[1], rowTime2);
+        });
+        tableStamp[1] = getLatestTime(tableStamp[1], tableTime2);
+      });
+      tablesStamp[1] = getLatestTime(tablesStamp[1], tablesTime2);
+    };
+    const getChangesFromOtherStore = async (
+      otherClientId = null,
+      otherContentHashes,
+    ) => {
+      if (isUndefined(otherContentHashes)) {
+        [otherContentHashes, otherClientId] = await request(
+          otherClientId,
+          1 /* GetContentHashes */,
+        );
+      }
+      const [otherTablesHash, otherValuesHash] = otherContentHashes;
+      const [tablesHash, valuesHash] = store.getMergeableContentHashes();
+      let tablesChanges = stampNewObj();
+      if (tablesHash != otherTablesHash) {
+        const [newTables, differentTableHashes] = (
+          await request(
+            otherClientId,
+            4 /* GetTableDiff */,
+            store.getMergeableTableHashes(),
+          )
+        )[0];
+        tablesChanges = newTables;
+        if (!objIsEmpty(differentTableHashes)) {
+          const [newRows, differentRowHashes] = (
+            await request(
+              otherClientId,
+              5 /* GetRowDiff */,
+              store.getMergeableRowHashes(differentTableHashes),
+            )
+          )[0];
+          mergeTablesStamps(tablesChanges, newRows);
+          if (!objIsEmpty(differentRowHashes)) {
+            const newCells = (
+              await request(
+                otherClientId,
+                6 /* GetCellDiff */,
+                store.getMergeableCellHashes(differentRowHashes),
+              )
+            )[0];
+            mergeTablesStamps(tablesChanges, newCells);
+          }
+        }
+      }
+      return [
+        tablesChanges,
+        valuesHash == otherValuesHash
+          ? stampNewObj()
+          : (
+              await request(
+                otherClientId,
+                7 /* GetValueDiff */,
+                store.getMergeableValueHashes(),
+              )
+            )[0],
+        1,
+      ];
+    };
+    const getPersisted = async () => {
+      const changes = await getChangesFromOtherStore();
+      return !objIsEmpty(changes[0][0]) || !objIsEmpty(changes[1][0])
+        ? changes
+        : void 0;
+    };
+    const setPersisted = async (_getContent, changes) =>
+      changes
+        ? sendImpl(null, null, 3 /* ContentDiff */, changes)
+        : sendImpl(
+            null,
+            null,
+            2 /* ContentHashes */,
+            store.getMergeableContentHashes(),
+          );
+    const addPersisterListener = (listener) => (persisterListener = listener);
+    const delPersisterListener = () => (persisterListener = void 0);
+    const startSync = async (initialContent) =>
+      await (await persister.startAutoLoad(initialContent)).startAutoSave();
+    const stopSync = () => persister.stopAutoLoad().stopAutoSave();
+    const destroy = () => {
+      return persister.stopSync();
+    };
+    const getSynchronizerStats = () => ({sends, receives});
+    const persister = createCustomPersister(
+      store,
+      getPersisted,
+      setPersisted,
+      addPersisterListener,
+      delPersisterListener,
+      onIgnoredError,
+      2,
+      // MergeableStoreOnly
+      {startSync, stopSync, destroy, getSynchronizerStats, ...extra},
+    );
+    registerReceive((fromClientId, requestId, message, body) => {
+      receives++;
+      if (message == 0 /* Response */) {
+        ifNotUndefined(
+          mapGet(pendingRequests, requestId),
+          ([toClientId, handleResponse]) =>
+            isUndefined(toClientId) || toClientId == fromClientId
+              ? handleResponse(body, fromClientId)
+              : /* istanbul ignore next */
+                0,
+        );
+      } else if (
+        message == 2 /* ContentHashes */ &&
+        persister.isAutoLoading()
+      ) {
+        getChangesFromOtherStore(fromClientId, body).then((changes) => {
+          persisterListener?.(void 0, changes);
+        });
+      } else if (message == 3 /* ContentDiff */ && persister.isAutoLoading()) {
+        persisterListener?.(void 0, body);
+      } else {
+        ifNotUndefined(
+          message == 1 /* GetContentHashes */ && persister.isAutoSaving()
+            ? store.getMergeableContentHashes()
+            : message == 4 /* GetTableDiff */
+              ? store.getMergeableTableDiff(body)
+              : message == 5 /* GetRowDiff */
+                ? store.getMergeableRowDiff(body)
+                : message == 6 /* GetCellDiff */
+                  ? store.getMergeableCellDiff(body)
+                  : message == 7 /* GetValueDiff */
+                    ? store.getMergeableValueDiff(body)
+                    : void 0,
+          (response) => {
+            sendImpl(fromClientId, requestId, 0 /* Response */, response);
+          },
+        );
+      }
+    });
+    return persister;
+  };
 
   const setNew = (entryOrEntries) =>
     new Set(
@@ -177,52 +627,110 @@
   };
 
   const PATH_REGEX = /\/([^?]*)/;
-  const createWsServer = (webSocketServer) => {
+  const SERVER_CLIENT_ID = 'S';
+  const createWsServer = (webSocketServer, createPersisterForPath) => {
     const pathIdListeners = mapNew();
     const clientIdListeners = mapNew();
     const clientsByPath = mapNew();
+    const serverClientsByPath = mapNew();
     const [addListener, callListeners, delListenerImpl] = getListenerFunctions(
       () => wsServer,
     );
+    const configureServerClient = (serverClient, pathId, clients) =>
+      ifNotUndefined(createPersisterForPath?.(pathId), (persister) => {
+        serverClient[0 /* State */] = 1;
+        serverClient[1 /* Persister */] = persister;
+        const messageHandler = getMessageHandler(
+          SERVER_CLIENT_ID,
+          clients,
+          serverClient,
+        );
+        serverClient[2 /* Synchronizer */] = createCustomSynchronizer(
+          persister.getStore(),
+          (toClientId, requestId, message, body) =>
+            messageHandler(createPayload(toClientId, requestId, message, body)),
+          (receive) =>
+            (serverClient[3 /* Send */] = (payload) =>
+              receivePayload(payload, receive)),
+          () => {},
+          1,
+        );
+        serverClient[4 /* Buffer */] = [];
+      });
+    const startServerClient = async (serverClient) => {
+      serverClient[0 /* State */] = 2 /* Starting */;
+      await serverClient[1 /* Persister */].schedule(
+        serverClient[1 /* Persister */].startAutoLoad,
+        serverClient[1 /* Persister */].startAutoSave,
+        serverClient[2 /* Synchronizer */].startSync,
+      );
+      serverClient[0 /* State */] = 0 /* Ready */;
+    };
+    const stopServerClient = (serverClient) => {
+      serverClient[1 /* Persister */]?.destroy();
+      serverClient[2 /* Synchronizer */]?.destroy();
+    };
+    const getMessageHandler = (clientId, clients, serverClient) => (payload) =>
+      ifPayloadValid(payload, (toClientId, remainder) => {
+        const forwardedPayload = createRawPayload(clientId, remainder);
+        if (toClientId === EMPTY_STRING) {
+          clientId !== SERVER_CLIENT_ID
+            ? serverClient[3 /* Send */]?.(forwardedPayload)
+            : 0;
+          mapForEach(clients, (otherClientId, otherWebSocket) =>
+            otherClientId !== clientId
+              ? otherWebSocket.send(forwardedPayload)
+              : 0,
+          );
+        } else {
+          toClientId === SERVER_CLIENT_ID
+            ? serverClient[3 /* Send */]?.(forwardedPayload)
+            : mapGet(clients, toClientId)?.send(forwardedPayload);
+        }
+      });
     webSocketServer.on('connection', (webSocket, request) =>
       ifNotUndefined(request.url?.match(PATH_REGEX), ([, pathId]) =>
-        ifNotUndefined(request.headers['sec-websocket-key'], (clientId) => {
-          const clients = mapEnsure(clientsByPath, pathId, mapNew);
-          mapSet(clients, clientId, webSocket);
-          if (clients.size == 1) {
-            callListeners(pathIdListeners, void 0, () => ({[pathId]: 1}));
-          }
-          callListeners(clientIdListeners, [pathId], () => ({[clientId]: 1}));
-          webSocket.on('message', (data) => {
-            const payload = data.toString(UTF8);
-            const splitAt = payload.indexOf(MESSAGE_SEPARATOR);
-            if (splitAt !== -1) {
-              const toClientId = slice(payload, 0, splitAt);
-              const message = slice(payload, splitAt + 1);
-              toClientId === EMPTY_STRING
-                ? mapForEach(clients, (otherClientId, otherWebSocket) =>
-                    otherClientId != clientId
-                      ? otherWebSocket.send(
-                          clientId + MESSAGE_SEPARATOR + message,
-                        )
-                      : 0,
-                  )
-                : mapGet(clients, toClientId)?.send(
-                    clientId + MESSAGE_SEPARATOR + message,
-                  );
-            }
-          });
-          webSocket.on('close', () => {
-            collDel(clients, clientId);
-            callListeners(clientIdListeners, [pathId], () => ({
-              [clientId]: -1,
-            }));
+        ifNotUndefined(
+          request.headers['sec-websocket-key'],
+          async (clientId) => {
+            const clients = mapEnsure(clientsByPath, pathId, mapNew);
+            const serverClient = mapEnsure(serverClientsByPath, pathId, () => [
+              0 /* Ready */,
+            ]);
+            const messageHandler = getMessageHandler(
+              clientId,
+              clients,
+              serverClient,
+            );
             if (collIsEmpty(clients)) {
-              collDel(clientsByPath, pathId);
-              callListeners(pathIdListeners, void 0, () => ({[pathId]: -1}));
+              callListeners(pathIdListeners, void 0, pathId, 1);
+              configureServerClient(serverClient, pathId, clients);
             }
-          });
-        }),
+            mapSet(clients, clientId, webSocket);
+            callListeners(clientIdListeners, [pathId], clientId, 1);
+            webSocket.on('message', (data) => {
+              const payload = data.toString(UTF8);
+              serverClient[0 /* State */] == 0 /* Ready */
+                ? messageHandler(payload)
+                : arrayPush(serverClient[4 /* Buffer */], payload);
+            });
+            if (serverClient[0 /* State */] == 1 /* Configured */) {
+              await startServerClient(serverClient);
+              arrayForEach(serverClient[4 /* Buffer */], messageHandler);
+              serverClient[4 /* Buffer */] = [];
+            }
+            webSocket.on('close', () => {
+              collDel(clients, clientId);
+              callListeners(clientIdListeners, [pathId], clientId, -1);
+              if (collIsEmpty(clients)) {
+                stopServerClient(serverClient);
+                collDel(serverClientsByPath, pathId);
+                collDel(clientsByPath, pathId);
+                callListeners(pathIdListeners, void 0, pathId, -1);
+              }
+            });
+          },
+        ),
       ),
     );
     const getWebSocketServer = () => webSocketServer;