tinybase 5.1.0-beta.0 → 5.1.0-beta.1

package/synchronizers/synchronizer-ws-server/index.js

@@ -1,6 +1,11 @@
 const EMPTY_STRING = '';
 const UTF8 = 'utf8';
+const UNDEFINED = '\uFFFC';
 
+const promise = Promise;
+const GLOBAL = globalThis;
+const math = Math;
+const mathFloor = math.floor;
 const isUndefined = (thing) => thing == void 0;
 const ifNotUndefined = (value, then, otherwise) =>
   isUndefined(value) ? otherwise?.() : then(value);
@@ -8,14 +13,46 @@ const isArray = (thing) => Array.isArray(thing);
 const slice = (arrayOrString, start, end) => arrayOrString.slice(start, end);
 const size = (arrayOrString) => arrayOrString.length;
 const test = (regex, subject) => regex.test(subject);
+const promiseNew = (resolver) => new promise(resolver);
+const errorNew = (message) => {
+  throw new Error(message);
+};
 
 const arrayForEach = (array, cb) => array.forEach(cb);
+const arrayMap = (array, cb) => array.map(cb);
 const arrayReduce = (array, cb, initial) => array.reduce(cb, initial);
+const arrayClear = (array, to) => array.splice(0, to);
 const arrayPush = (array, ...values) => array.push(...values);
 const arrayShift = (array) => array.shift();
 
 const object = Object;
+const getPrototypeOf = (obj) => object.getPrototypeOf(obj);
+const objEntries = object.entries;
+const isObject = (obj) =>
+  !isUndefined(obj) &&
+  ifNotUndefined(
+    getPrototypeOf(obj),
+    (objPrototype) =>
+      objPrototype == object.prototype ||
+      isUndefined(getPrototypeOf(objPrototype)),
+
+    /* istanbul ignore next */
+    () => true,
+  );
+const objIds = object.keys;
 const objFreeze = object.freeze;
+const objNew = (entries = []) => object.fromEntries(entries);
+const objHas = (obj, id) => id in obj;
+const objForEach = (obj, cb) =>
+  arrayForEach(objEntries(obj), ([id, value]) => cb(value, id));
+const objSize = (obj) => size(objIds(obj));
+const objIsEmpty = (obj) => isObject(obj) && objSize(obj) == 0;
+const objEnsure = (obj, id, getDefaultValue) => {
+  if (!objHas(obj, id)) {
+    obj[id] = getDefaultValue();
+  }
+  return obj[id];
+};
 
 const collSizeN = (collSizer) => (coll) =>
   arrayReduce(collValues(coll), (total, coll2) => total + collSizer(coll2), 0);
@@ -65,7 +102,413 @@ const visitTree = (node, path, ensureLeaf, pruneLeaf, p = 0) =>
     },
   );
 
+const jsonStringWithUndefined = (obj) =>
+  JSON.stringify(obj, (_key, value) => (value === void 0 ? UNDEFINED : value));
+const jsonParseWithUndefined = (str) =>
+  JSON.parse(str, (_key, value) => (value === UNDEFINED ? void 0 : value));
+
 const MESSAGE_SEPARATOR = '\n';
+const ifPayloadValid = (payload, then) => {
+  const splitAt = payload.indexOf(MESSAGE_SEPARATOR);
+  splitAt !== -1
+    ? then(slice(payload, 0, splitAt), slice(payload, splitAt + 1))
+    : 0;
+};
+const receivePayload = (payload, receive) =>
+  ifPayloadValid(payload, (fromClientId, remainder) =>
+    receive(fromClientId, ...jsonParseWithUndefined(remainder)),
+  );
+const createPayload = (toClientId, ...args) =>
+  createRawPayload(toClientId ?? EMPTY_STRING, jsonStringWithUndefined(args));
+const createRawPayload = (toClientId, remainder) =>
+  toClientId + MESSAGE_SEPARATOR + remainder;
+
+new GLOBAL.TextEncoder();
+
+const newStamp = (value, time) => (time ? [value, time] : [value]);
+const getLatestTime = (time1, time2) =>
+  ((time1 ?? '') > (time2 ?? '') ? time1 : time2) ?? '';
+const stampNewObj = (time = EMPTY_STRING) => newStamp(objNew(), time);
+
+const scheduleRunning = mapNew();
+const scheduleActions = mapNew();
+const getStoreFunctions = (persist = 1 /* StoreOnly */, store) =>
+  persist != 1 /* StoreOnly */ && store.isMergeable()
+    ? [
+        1,
+        store.getMergeableContent,
+        store.getTransactionMergeableChanges,
+        ([[changedTables], [changedValues]]) =>
+          !objIsEmpty(changedTables) || !objIsEmpty(changedValues),
+        store.setDefaultContent,
+      ]
+    : persist != 2 /* MergeableStoreOnly */
+      ? [
+          0,
+          store.getContent,
+          store.getTransactionChanges,
+          ([changedTables, changedValues]) =>
+            !objIsEmpty(changedTables) || !objIsEmpty(changedValues),
+          store.setContent,
+        ]
+      : errorNew('Store type not supported by this Persister');
+const createCustomPersister = (
+  store,
+  getPersisted,
+  setPersisted,
+  addPersisterListener,
+  delPersisterListener,
+  onIgnoredError,
+  persist,
+  extra = {},
+  scheduleId = [],
+) => {
+  let loadSave = 0;
+  let loads = 0;
+  let saves = 0;
+  let action;
+  let autoLoadHandle;
+  let autoSaveListenerId;
+  mapEnsure(scheduleRunning, scheduleId, () => 0);
+  mapEnsure(scheduleActions, scheduleId, () => []);
+  const [
+    isMergeableStore,
+    getContent,
+    getChanges,
+    hasChanges,
+    setDefaultContent,
+  ] = getStoreFunctions(persist, store);
+  const run = async () => {
+    /* istanbul ignore else */
+    if (!mapGet(scheduleRunning, scheduleId)) {
+      mapSet(scheduleRunning, scheduleId, 1);
+      while (
+        !isUndefined((action = arrayShift(mapGet(scheduleActions, scheduleId))))
+      ) {
+        try {
+          await action();
+        } catch (error) {}
+      }
+      mapSet(scheduleRunning, scheduleId, 0);
+    }
+  };
+  const setContentOrChanges = (contentOrChanges) => {
+    (isMergeableStore && isArray(contentOrChanges?.[0])
+      ? contentOrChanges?.[2] === 1
+        ? store.applyMergeableChanges
+        : store.setMergeableContent
+      : contentOrChanges?.[2] === 1
+        ? store.applyChanges
+        : store.setContent)(contentOrChanges);
+  };
+  const load = async (initialContent) => {
+    /* istanbul ignore else */
+    if (loadSave != 2) {
+      loadSave = 1;
+      loads++;
+      await schedule(async () => {
+        try {
+          setContentOrChanges(await getPersisted());
+        } catch (error) {
+          if (initialContent) {
+            setDefaultContent(initialContent);
+          }
+        }
+        loadSave = 0;
+      });
+    }
+    return persister;
+  };
+  const startAutoLoad = async (initialContent) => {
+    await stopAutoLoad().load(initialContent);
+    try {
+      autoLoadHandle = addPersisterListener(async (content, changes) => {
+        if (changes || content) {
+          /* istanbul ignore else */
+          if (loadSave != 2) {
+            loadSave = 1;
+            loads++;
+            setContentOrChanges(changes ?? content);
+            loadSave = 0;
+          }
+        } else {
+          await load();
+        }
+      });
+    } catch (error) {}
+    return persister;
+  };
+  const stopAutoLoad = () => {
+    if (autoLoadHandle) {
+      delPersisterListener(autoLoadHandle);
+      autoLoadHandle = void 0;
+    }
+    return persister;
+  };
+  const isAutoLoading = () => !isUndefined(autoLoadHandle);
+  const save = async (changes) => {
+    /* istanbul ignore else */
+    if (loadSave != 1) {
+      loadSave = 2;
+      saves++;
+      await schedule(async () => {
+        try {
+          await setPersisted(getContent, changes);
+        } catch (error) {}
+        loadSave = 0;
+      });
+    }
+    return persister;
+  };
+  const startAutoSave = async () => {
+    await stopAutoSave().save();
+    autoSaveListenerId = store.addDidFinishTransactionListener(() => {
+      const changes = getChanges();
+      if (hasChanges(changes)) {
+        save(changes);
+      }
+    });
+    return persister;
+  };
+  const stopAutoSave = () => {
+    ifNotUndefined(autoSaveListenerId, store.delListener);
+    autoSaveListenerId = void 0;
+    return persister;
+  };
+  const isAutoSaving = () => !isUndefined(autoSaveListenerId);
+  const schedule = async (...actions) => {
+    arrayPush(mapGet(scheduleActions, scheduleId), ...actions);
+    await run();
+    return persister;
+  };
+  const getStore = () => store;
+  const destroy = () => {
+    arrayClear(mapGet(scheduleActions, scheduleId));
+    return stopAutoLoad().stopAutoSave();
+  };
+  const getStats = () => ({loads, saves});
+  const persister = {
+    load,
+    startAutoLoad,
+    stopAutoLoad,
+    isAutoLoading,
+    save,
+    startAutoSave,
+    stopAutoSave,
+    isAutoSaving,
+    schedule,
+    getStore,
+    destroy,
+    getStats,
+    ...extra,
+  };
+  return objFreeze(persister);
+};
+
+const MASK6 = 63;
+const ENCODE =
+  '-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'.split(
+    EMPTY_STRING,
+  );
+mapNew(arrayMap(ENCODE, (char, index) => [char, index]));
+const encode = (num) => ENCODE[num & MASK6];
+
+const getRandomValues = GLOBAL.crypto
+  ? (array) => GLOBAL.crypto.getRandomValues(array)
+  : /* istanbul ignore next */
+    (array) => arrayMap(array, () => mathFloor(math.random() * 256));
+const getUniqueId = (length = 16) =>
+  arrayReduce(
+    getRandomValues(new Uint8Array(length)),
+    (uniqueId, number) => uniqueId + encode(number),
+    '',
+  );
+
+const createCustomSynchronizer = (
+  store,
+  send,
+  registerReceive,
+  destroyImpl,
+  requestTimeoutSeconds,
+  onSend,
+  onReceive,
+  onIgnoredError,
+  extra = {},
+) => {
+  let persisterListener;
+  let sends = 0;
+  let receives = 0;
+  const pendingRequests = mapNew();
+  const sendImpl = (toClientId, requestId, message, body) => {
+    sends++;
+    send(toClientId, requestId, message, body);
+  };
+  const request = async (toClientId, message, body = EMPTY_STRING) =>
+    promiseNew((resolve, reject) => {
+      const requestId = getUniqueId();
+      const timeout = setTimeout(() => {
+        collDel(pendingRequests, requestId);
+        reject(
+          `No response from ${toClientId ?? 'anyone'} to ${requestId}, ` +
+            message,
+        );
+      }, requestTimeoutSeconds * 1e3);
+      mapSet(pendingRequests, requestId, [
+        toClientId,
+        (response, fromClientId) => {
+          clearTimeout(timeout);
+          collDel(pendingRequests, requestId);
+          resolve([response, fromClientId]);
+        },
+      ]);
+      sendImpl(toClientId, requestId, message, body);
+    });
+  const mergeTablesStamps = (tablesStamp, [tableStamps2, tablesTime2]) => {
+    objForEach(tableStamps2, ([rowStamps2, tableTime2], tableId) => {
+      const tableStamp = objEnsure(tablesStamp[0], tableId, stampNewObj);
+      objForEach(rowStamps2, ([cellStamps2, rowTime2], rowId) => {
+        const rowStamp = objEnsure(tableStamp[0], rowId, stampNewObj);
+        objForEach(
+          cellStamps2,
+          ([cell2, cellTime2], cellId) =>
+            (rowStamp[0][cellId] = newStamp(cell2, cellTime2)),
+        );
+        rowStamp[1] = getLatestTime(rowStamp[1], rowTime2);
+      });
+      tableStamp[1] = getLatestTime(tableStamp[1], tableTime2);
+    });
+    tablesStamp[1] = getLatestTime(tablesStamp[1], tablesTime2);
+  };
+  const getChangesFromOtherStore = async (
+    otherClientId = null,
+    otherContentHashes,
+  ) => {
+    if (isUndefined(otherContentHashes)) {
+      [otherContentHashes, otherClientId] = await request(
+        otherClientId,
+        1 /* GetContentHashes */,
+      );
+    }
+    const [otherTablesHash, otherValuesHash] = otherContentHashes;
+    const [tablesHash, valuesHash] = store.getMergeableContentHashes();
+    let tablesChanges = stampNewObj();
+    if (tablesHash != otherTablesHash) {
+      const [newTables, differentTableHashes] = (
+        await request(
+          otherClientId,
+          4 /* GetTableDiff */,
+          store.getMergeableTableHashes(),
+        )
+      )[0];
+      tablesChanges = newTables;
+      if (!objIsEmpty(differentTableHashes)) {
+        const [newRows, differentRowHashes] = (
+          await request(
+            otherClientId,
+            5 /* GetRowDiff */,
+            store.getMergeableRowHashes(differentTableHashes),
+          )
+        )[0];
+        mergeTablesStamps(tablesChanges, newRows);
+        if (!objIsEmpty(differentRowHashes)) {
+          const newCells = (
+            await request(
+              otherClientId,
+              6 /* GetCellDiff */,
+              store.getMergeableCellHashes(differentRowHashes),
+            )
+          )[0];
+          mergeTablesStamps(tablesChanges, newCells);
+        }
+      }
+    }
+    return [
+      tablesChanges,
+      valuesHash == otherValuesHash
+        ? stampNewObj()
+        : (
+            await request(
+              otherClientId,
+              7 /* GetValueDiff */,
+              store.getMergeableValueHashes(),
+            )
+          )[0],
+      1,
+    ];
+  };
+  const getPersisted = async () => {
+    const changes = await getChangesFromOtherStore();
+    return !objIsEmpty(changes[0][0]) || !objIsEmpty(changes[1][0])
+      ? changes
+      : void 0;
+  };
+  const setPersisted = async (_getContent, changes) =>
+    changes
+      ? sendImpl(null, null, 3 /* ContentDiff */, changes)
+      : sendImpl(
+          null,
+          null,
+          2 /* ContentHashes */,
+          store.getMergeableContentHashes(),
+        );
+  const addPersisterListener = (listener) => (persisterListener = listener);
+  const delPersisterListener = () => (persisterListener = void 0);
+  const startSync = async (initialContent) =>
+    await (await persister.startAutoLoad(initialContent)).startAutoSave();
+  const stopSync = () => persister.stopAutoLoad().stopAutoSave();
+  const destroy = () => {
+    return persister.stopSync();
+  };
+  const getSynchronizerStats = () => ({sends, receives});
+  const persister = createCustomPersister(
+    store,
+    getPersisted,
+    setPersisted,
+    addPersisterListener,
+    delPersisterListener,
+    onIgnoredError,
+    2,
+    // MergeableStoreOnly
+    {startSync, stopSync, destroy, getSynchronizerStats, ...extra},
+  );
+  registerReceive((fromClientId, requestId, message, body) => {
+    receives++;
+    if (message == 0 /* Response */) {
+      ifNotUndefined(
+        mapGet(pendingRequests, requestId),
+        ([toClientId, handleResponse]) =>
+          isUndefined(toClientId) || toClientId == fromClientId
+            ? handleResponse(body, fromClientId)
+            : /* istanbul ignore next */
+              0,
+      );
+    } else if (message == 2 /* ContentHashes */ && persister.isAutoLoading()) {
+      getChangesFromOtherStore(fromClientId, body).then((changes) => {
+        persisterListener?.(void 0, changes);
+      });
+    } else if (message == 3 /* ContentDiff */ && persister.isAutoLoading()) {
+      persisterListener?.(void 0, body);
+    } else {
+      ifNotUndefined(
+        message == 1 /* GetContentHashes */ && persister.isAutoSaving()
+          ? store.getMergeableContentHashes()
+          : message == 4 /* GetTableDiff */
+            ? store.getMergeableTableDiff(body)
+            : message == 5 /* GetRowDiff */
+              ? store.getMergeableRowDiff(body)
+              : message == 6 /* GetCellDiff */
+                ? store.getMergeableCellDiff(body)
+                : message == 7 /* GetValueDiff */
+                  ? store.getMergeableValueDiff(body)
+                  : void 0,
+        (response) => {
+          sendImpl(fromClientId, requestId, 0 /* Response */, response);
+        },
+      );
+    }
+  });
+  return persister;
+};
 
 const setNew = (entryOrEntries) =>
   new Set(
@@ -160,47 +603,104 @@ const getListenerFunctions = (getThing) => {
 };
 
 const PATH_REGEX = /\/([^?]*)/;
-const createWsServer = (webSocketServer) => {
+const SERVER_CLIENT_ID = 'S';
+const createWsServer = (webSocketServer, createPersisterForPath) => {
   const pathIdListeners = mapNew();
   const clientIdListeners = mapNew();
   const clientsByPath = mapNew();
+  const serverClientsByPath = mapNew();
   const [addListener, callListeners, delListenerImpl] = getListenerFunctions(
     () => wsServer,
   );
+  const configureServerClient = (serverClient, pathId, clients) =>
+    ifNotUndefined(createPersisterForPath?.(pathId), (persister) => {
+      serverClient[0 /* State */] = 1;
+      serverClient[1 /* Persister */] = persister;
+      const messageHandler = getMessageHandler(
+        SERVER_CLIENT_ID,
+        clients,
+        serverClient,
+      );
+      serverClient[2 /* Synchronizer */] = createCustomSynchronizer(
+        persister.getStore(),
+        (toClientId, requestId, message, body) =>
+          messageHandler(createPayload(toClientId, requestId, message, body)),
+        (receive) =>
+          (serverClient[3 /* Send */] = (payload) =>
+            receivePayload(payload, receive)),
+        () => {},
+        1,
+      );
+      serverClient[4 /* Buffer */] = [];
+    });
+  const startServerClient = async (serverClient) => {
+    serverClient[0 /* State */] = 2 /* Starting */;
+    await serverClient[1 /* Persister */].schedule(
+      serverClient[1 /* Persister */].startAutoLoad,
+      serverClient[1 /* Persister */].startAutoSave,
+      serverClient[2 /* Synchronizer */].startSync,
+    );
+    serverClient[0 /* State */] = 0 /* Ready */;
+  };
+  const stopServerClient = (serverClient) => {
+    serverClient[1 /* Persister */]?.destroy();
+    serverClient[2 /* Synchronizer */]?.destroy();
+  };
+  const getMessageHandler = (clientId, clients, serverClient) => (payload) =>
+    ifPayloadValid(payload, (toClientId, remainder) => {
+      const forwardedPayload = createRawPayload(clientId, remainder);
+      if (toClientId === EMPTY_STRING) {
+        clientId !== SERVER_CLIENT_ID
+          ? serverClient[3 /* Send */]?.(forwardedPayload)
+          : 0;
+        mapForEach(clients, (otherClientId, otherWebSocket) =>
+          otherClientId !== clientId
+            ? otherWebSocket.send(forwardedPayload)
+            : 0,
+        );
+      } else {
+        toClientId === SERVER_CLIENT_ID
+          ? serverClient[3 /* Send */]?.(forwardedPayload)
+          : mapGet(clients, toClientId)?.send(forwardedPayload);
+      }
+    });
   webSocketServer.on('connection', (webSocket, request) =>
     ifNotUndefined(request.url?.match(PATH_REGEX), ([, pathId]) =>
-      ifNotUndefined(request.headers['sec-websocket-key'], (clientId) => {
+      ifNotUndefined(request.headers['sec-websocket-key'], async (clientId) => {
         const clients = mapEnsure(clientsByPath, pathId, mapNew);
-        mapSet(clients, clientId, webSocket);
-        if (clients.size == 1) {
-          callListeners(pathIdListeners, void 0, () => ({[pathId]: 1}));
+        const serverClient = mapEnsure(serverClientsByPath, pathId, () => [
+          0 /* Ready */,
+        ]);
+        const messageHandler = getMessageHandler(
+          clientId,
+          clients,
+          serverClient,
+        );
+        if (collIsEmpty(clients)) {
+          callListeners(pathIdListeners, void 0, pathId, 1);
+          configureServerClient(serverClient, pathId, clients);
         }
-        callListeners(clientIdListeners, [pathId], () => ({[clientId]: 1}));
+        mapSet(clients, clientId, webSocket);
+        callListeners(clientIdListeners, [pathId], clientId, 1);
         webSocket.on('message', (data) => {
           const payload = data.toString(UTF8);
-          const splitAt = payload.indexOf(MESSAGE_SEPARATOR);
-          if (splitAt !== -1) {
-            const toClientId = slice(payload, 0, splitAt);
-            const message = slice(payload, splitAt + 1);
-            toClientId === EMPTY_STRING
-              ? mapForEach(clients, (otherClientId, otherWebSocket) =>
-                  otherClientId != clientId
-                    ? otherWebSocket.send(
-                        clientId + MESSAGE_SEPARATOR + message,
-                      )
-                    : 0,
-                )
-              : mapGet(clients, toClientId)?.send(
-                  clientId + MESSAGE_SEPARATOR + message,
-                );
-          }
+          serverClient[0 /* State */] == 0 /* Ready */
+            ? messageHandler(payload)
+            : arrayPush(serverClient[4 /* Buffer */], payload);
         });
+        if (serverClient[0 /* State */] == 1 /* Configured */) {
+          await startServerClient(serverClient);
+          arrayForEach(serverClient[4 /* Buffer */], messageHandler);
+          serverClient[4 /* Buffer */] = [];
+        }
         webSocket.on('close', () => {
           collDel(clients, clientId);
-          callListeners(clientIdListeners, [pathId], () => ({[clientId]: -1}));
+          callListeners(clientIdListeners, [pathId], clientId, -1);
           if (collIsEmpty(clients)) {
+            stopServerClient(serverClient);
+            collDel(serverClientsByPath, pathId);
             collDel(clientsByPath, pathId);
-            callListeners(pathIdListeners, void 0, () => ({[pathId]: -1}));
+            callListeners(pathIdListeners, void 0, pathId, -1);
           }
         });
       }),