tinybase 9.4.0-beta.1 → 9.4.0

package/synchronizers/synchronizer-ws-server/with-schemas/index.js

@@ -113,6 +113,8 @@ const tryCatch = async (action, onError) => {
     onError?.(error);
   }
 };
+const tryCatchIgnore = (action, onIgnoredError) =>
+  tryCatch(action, (error) => void tryCatch(() => onIgnoredError?.(error)));
 const tryCatchSync = (action, onError) => {
   try {
     return action();
@@ -434,6 +436,9 @@ const FRAGMENT_PAYLOAD = /^[^\n]*\n[-0-9A-Z_a-z]{16}\n\d+\n\d+\n/;
 const INVALID_CHANNEL_ID_CHARACTERS = /[\n\r?#]/;
 const MAX_FRAGMENT_BUFFERS = 100;
 const MAX_FRAGMENT_COUNT = 1e3;
+const MAX_MULTIPLE_CHANNELS = 100;
+const MAX_MULTIPLE_CHANNEL_RESOURCES = MAX_MULTIPLE_CHANNELS * 2;
+const MAX_MULTIPLE_CHANNEL_ID_SIZE = 1024;
 const MAX_PENDING_REQUESTS = 100;
 const MAX_WEBSOCKET_BUFFER_SIZE = 16777216;
 const MAX_WEBSOCKET_QUEUE_SIZE = 1e3;
@@ -566,21 +571,30 @@ const createPayloadDecoder = (
   receive,
   fragmentTimeoutSeconds = 1,
   onInvalid,
+  sharedBuffer,
 ) => {
   const buffer = mapNew();
+  const payloadBuffer = sharedBuffer ?? [0, 0];
   let bufferedSize = 0;
   let valid = true;
   const delPending = (bufferKey, pending) => {
     stopTimeout(pending[4]);
     bufferedSize -= pending[5];
+    payloadBuffer[0]--;
+    payloadBuffer[1] -= pending[5];
     collDel(buffer, bufferKey);
   };
+  const clear = () => {
+    mapForEach(buffer, (_bufferKey, pending) => stopTimeout(pending[4]));
+    payloadBuffer[0] -= collSize(buffer);
+    payloadBuffer[1] -= bufferedSize;
+    collClear(buffer);
+    bufferedSize = 0;
+  };
   const invalid = (error = errorNew(ERROR_SYNC_MESSAGE)) => {
     if (valid) {
       valid = false;
-      mapForEach(buffer, (_bufferKey, pending) => stopTimeout(pending[4]));
-      collClear(buffer);
-      bufferedSize = 0;
+      clear();
       onInvalid?.(error);
     }
   };
@@ -592,7 +606,7 @@ const createPayloadDecoder = (
       invalid();
     }
   };
-  return (payload) => {
+  const decode = (payload) => {
     if (!valid) {
       return;
     }
@@ -625,42 +639,52 @@ const createPayloadDecoder = (
             const bufferKey = clientId + MESSAGE_SEPARATOR + messageId;
             let pending = mapGet(buffer, bufferKey);
             if (!pending) {
-              if (collSize(buffer) >= MAX_FRAGMENT_BUFFERS) {
+              if (payloadBuffer[0] >= MAX_FRAGMENT_BUFFERS) {
                 invalid(errorNew(ERROR_SYNC_OVERFLOW, 'fragments'));
                 return;
               }
-              pending = [
-                [],
-                [],
-                total,
-                total,
-                startTimeout(() => {
-                  const timedOut = mapGet(buffer, bufferKey);
-                  if (timedOut) {
-                    delPending(bufferKey, timedOut);
-                  }
-                }, fragmentTimeoutSeconds),
-                0,
-              ];
-              mapSet(buffer, bufferKey, pending);
             }
-            const [fragments, remainders] = pending;
-            if (total == pending[3] && isUndefined(fragments[index])) {
-              const fragmentLength = getWebSocketPayloadSize(fragment);
-              if (bufferedSize + fragmentLength > MAX_WEBSOCKET_BUFFER_SIZE) {
+            if (
+              (isUndefined(pending) || total == pending[3]) &&
+              isUndefined(pending?.[0][index])
+            ) {
+              const retainedSize =
+                getWebSocketPayloadSize(remainder) +
+                (pending ? 0 : getWebSocketPayloadSize(bufferKey));
+              if (payloadBuffer[1] + retainedSize > MAX_WEBSOCKET_BUFFER_SIZE) {
                 invalid(errorNew(ERROR_SYNC_OVERFLOW, 'fragments'));
                 return;
               }
+              if (!pending) {
+                pending = [
+                  [],
+                  [],
+                  total,
+                  total,
+                  startTimeout(() => {
+                    const timedOut = mapGet(buffer, bufferKey);
+                    if (timedOut) {
+                      delPending(bufferKey, timedOut);
+                    }
+                  }, fragmentTimeoutSeconds),
+                  0,
+                ];
+                mapSet(buffer, bufferKey, pending);
+                payloadBuffer[0]++;
+              }
+              const [fragments, remainders] = pending;
               fragments[index] = fragment;
               remainders[index] = remainder;
               pending[2]--;
-              pending[5] += fragmentLength;
-              bufferedSize += fragmentLength;
+              pending[5] += retainedSize;
+              bufferedSize += retainedSize;
+              payloadBuffer[1] += retainedSize;
             } else {
               invalid();
               return;
             }
             if (pending[2] == 0) {
+              const [fragments, remainders] = pending;
               delPending(bufferKey, pending);
               receiveRemainder(clientId, remainders, arrayJoin(fragments));
             }
@@ -673,6 +697,7 @@ const createPayloadDecoder = (
       invalid();
     }
   };
+  return [decode, clear];
 };
 const getPayloadCoalesceKey = (clientId, payload) => {
   let key;
@@ -689,12 +714,14 @@ const createPayloadReceiver = (
   receive,
   fragmentTimeoutSeconds = 1,
   onInvalid,
+  sharedBuffer,
 ) =>
   createPayloadDecoder(
     (fromClientId, _remainders, requestId, message, body) =>
       receive(fromClientId, requestId, message, body),
     fragmentTimeoutSeconds,
     onInvalid,
+    sharedBuffer,
   );
 const createRawPayload = (clientId, remainder) =>
   clientId + MESSAGE_SEPARATOR + remainder;
@@ -727,12 +754,15 @@ const getFragments = (remainder, maxFragmentSize) => {
 };
 const createMultiplePayload = (channelId, payload) =>
   createRawPayload(MULTIPLE_CLIENT_ID, createRawPayload(channelId, payload));
-const ifMultiplePayloadValid = (payload, then) => {
+const ifMultiplePayloadValid = (payload, then, onOversized) => {
   let valid = false;
   ifPayloadValid(payload, (multipleClientId, remainder) => {
     if (multipleClientId == MULTIPLE_CLIENT_ID) {
       ifPayloadValid(remainder, (channelId, channelPayload) => {
-        if (isMultipleChannelIdValid(channelId)) {
+        if (isMultipleChannelIdTooLarge(channelId)) {
+          valid = true;
+          onOversized?.();
+        } else if (isMultipleChannelIdStructureValid(channelId)) {
           valid = true;
           then(channelId, channelPayload);
         }
@@ -746,6 +776,15 @@ const createMultipleControlPayload = (requestId, control, body) =>
     SERVER_CLIENT_ID,
     jsonStringWithUndefined([requestId, MULTIPLE_MESSAGE, [control, body]]),
   );
+const isMultipleChannelIdTooLarge = (channelId) =>
+  getWebSocketPayloadSize(channelId) > MAX_MULTIPLE_CHANNEL_ID_SIZE;
+const isMultipleChannelIdStructureValid = (channelId) =>
+  !isEmpty(channelId) &&
+  !INVALID_CHANNEL_ID_CHARACTERS.test(channelId) &&
+  arrayEvery(
+    strSplit(channelId, '/'),
+    (part) => !isEmpty(part) && part != '.' && part != '..',
+  );
 const ifMultipleControlPayloadValid = (payload, then) => {
   let valid = false;
   ifPayloadValid(payload, (serverClientId, remainder) => {
@@ -763,11 +802,13 @@ const ifMultipleControlPayloadValid = (payload, then) => {
           : control == 1 /* Subscribe */
             ? isString(requestId) &&
               isString(body) &&
-              isMultipleChannelIdValid(body)
+              (isMultipleChannelIdTooLarge(body) ||
+                isMultipleChannelIdStructureValid(body))
             : control == 2 /* Unsubscribe */ &&
               isNull(requestId) &&
               isString(body) &&
-              isMultipleChannelIdValid(body))
+              (isMultipleChannelIdTooLarge(body) ||
+                isMultipleChannelIdStructureValid(body)))
       ) {
         valid = true;
         then(requestId, control, body);
@@ -776,13 +817,6 @@ const ifMultipleControlPayloadValid = (payload, then) => {
   });
   return valid;
 };
-const isMultipleChannelIdValid = (channelId) =>
-  !isEmpty(channelId) &&
-  !INVALID_CHANNEL_ID_CHARACTERS.test(channelId) &&
-  arrayEvery(
-    strSplit(channelId, '/'),
-    (part) => !isEmpty(part) && part != '.' && part != '..',
-  );
 const createMultipleServerClient = (
   basePathId,
   addChannel,
@@ -795,26 +829,87 @@ const createMultipleServerClient = (
 ) => {
   const channels = mapNew();
   const decoders = mapNew();
+  const payloadBuffer = [0, 0];
+  const resources = setNew();
+  let resourceCount = 0;
   let negotiated = false;
+  let valid = true;
   const sendControl = (requestId, control, body) =>
     send(createMultipleControlPayload(requestId, control, body));
   const delChannelAndDecoder = (channelId) => {
     collDel(channels, channelId);
+    mapGet(decoders, channelId)?.[1]();
     collDel(decoders, channelId);
   };
+  const clearDecoders = () => {
+    mapForEach(decoders, (_channelId, decoder) => decoder[1]());
+    collClear(decoders);
+  };
+  const invalidate = (error) => {
+    if (valid) {
+      valid = false;
+      clearDecoders();
+      invalid(error);
+    }
+  };
+  const releaseChannel = (channel) => {
+    if (!channel[2 /* Released */]) {
+      channel[2 /* Released */] = 1;
+      resourceCount--;
+      collDel(resources, channel);
+    }
+  };
+  const teardownChannel = (channel) =>
+    (channel[1 /* Teardown */] ??= (async () => {
+      try {
+        const teardown = delChannel(channel[0 /* Channel */]);
+        if (isUndefined(teardown)) {
+          releaseChannel(channel);
+        } else {
+          await teardown;
+        }
+      } finally {
+        releaseChannel(channel);
+      }
+    })());
   const handleControl = (requestId, control, body) => {
     if (control == 0 /* Hello */) {
       negotiated = true;
       sendControl(requestId, control, body);
     } else if (negotiated && control == 1 /* Subscribe */) {
-      if (!collHas(channels, body)) {
+      const subscribed = collHas(channels, body);
+      if (
+        isMultipleChannelIdTooLarge(body) ||
+        (!subscribed &&
+          (collSize(channels) >= MAX_MULTIPLE_CHANNELS ||
+            resourceCount >= MAX_MULTIPLE_CHANNEL_RESOURCES))
+      ) {
+        invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'channels'));
+        return;
+      }
+      if (!subscribed) {
         const pathId = basePathId + (basePathId ? '/' : EMPTY_STRING) + body;
-        const [channel, ready] = addChannel(pathId, body);
+        resourceCount++;
+        let channelAndReady;
+        try {
+          channelAndReady = addChannel(pathId, body);
+        } catch (error) {
+          resourceCount--;
+          throw error;
+        }
+        const [addedChannel, ready] = channelAndReady;
+        const channel = [addedChannel];
+        setAdd(resources, channel);
         mapSet(channels, body, channel);
         sendControl(requestId, control, body);
         if (ready) {
           return ready.catch((error) => {
-            delChannelAndDecoder(body);
+            if (mapGet(channels, body) === channel) {
+              delChannelAndDecoder(body);
+            }
+            if (!channel[1 /* Teardown */]) {
+              releaseChannel(channel);
+            }
             throw error;
           });
         }
@@ -822,16 +917,21 @@ const createMultipleServerClient = (
       }
       sendControl(requestId, control, body);
     } else if (negotiated && control == 2 /* Unsubscribe */) {
+      if (isMultipleChannelIdTooLarge(body)) {
+        invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'channels'));
+        return;
+      }
       const channel = mapGet(channels, body);
       delChannelAndDecoder(body);
-      if (!isUndefined(channel)) {
-        return (async () => delChannel(channel))();
-      }
+      return isUndefined(channel) ? void 0 : teardownChannel(channel);
     }
   };
   const handlePayload = (payload) => {
+    if (!valid) {
+      return;
+    }
     if (isWebSocketPayloadTooLarge(getWebSocketPayloadSize(payload))) {
-      invalid(errorNew(ERROR_SYNC_OVERFLOW, 'socket'));
+      invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'socket'));
       return;
     }
     const control = ifMultipleControlPayloadValid(
@@ -846,32 +946,54 @@ const createMultipleServerClient = (
       (channelId, channelPayload) => {
         const channel2 = negotiated ? mapGet(channels, channelId) : void 0;
         if (isUndefined(channel2)) {
-          invalid(errorNew(ERROR_SYNC_MESSAGE));
+          invalidate(errorNew(ERROR_SYNC_MESSAGE));
         } else {
           mapEnsure(decoders, channelId, () =>
             createPayloadDecoder(
               (toClientId, remainders) =>
-                receive(channel2, toClientId, remainders),
+                receive(channel2[0 /* Channel */], toClientId, remainders),
               fragmentTimeoutSeconds,
-              invalid,
+              invalidate,
+              payloadBuffer,
             ),
-          )(channelPayload);
+          )[0](channelPayload);
         }
       },
+      () => invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'channels')),
     );
     if (!control && !channel) {
-      invalid(errorNew(ERROR_SYNC_MESSAGE));
+      invalidate(errorNew(ERROR_SYNC_MESSAGE));
     }
   };
-  const destroy = () =>
-    tryFinallyAsync(
-      () =>
-        promiseAll(mapMap(channels, async (channel) => delChannel(channel))),
+  const destroy = () => {
+    valid = false;
+    clearDecoders();
+    return tryFinallyAsync(
+      async () => {
+        let errorToThrow;
+        let failed = false;
+        await promiseAll(
+          arrayMap(collValues(resources), async (channel) => {
+            try {
+              await teardownChannel(channel);
+            } catch (error) {
+              if (!failed) {
+                errorToThrow = error;
+                failed = true;
+              }
+            }
+          }),
+        );
+        if (failed) {
+          throw errorToThrow;
+        }
+      },
       () => {
         collClear(channels);
-        collClear(decoders);
+        clearDecoders();
       },
     );
+  };
   return [handlePayload, destroy];
 };
 const createPayloads = (toClientId, requestId, message, body, fragmentSize) => {
@@ -1233,11 +1355,35 @@ const createCustomPersister = (
     initialContent,
     startSaveFirst = false,
   ) => {
-    const [call1, call2] = startSaveFirst
-      ? [startAutoSave, startAutoLoad]
-      : [startAutoLoad, startAutoSave];
-    await call1(initialContent);
-    await call2(initialContent);
+    const [call1, call2, stop1, stop2] = startSaveFirst
+      ? [startAutoSave, startAutoLoad, stopAutoSave, stopAutoLoad]
+      : [startAutoLoad, startAutoSave, stopAutoLoad, stopAutoSave];
+    const start1 = call1(initialContent);
+    const generation1 = startSaveFirst
+      ? autoSaveGeneration
+      : autoLoadGeneration;
+    await start1;
+    const start2 = call2(initialContent);
+    const generation2 = startSaveFirst
+      ? autoLoadGeneration
+      : autoSaveGeneration;
+    try {
+      await start2;
+    } catch (error) {
+      if (
+        generation2 ==
+        (startSaveFirst ? autoLoadGeneration : autoSaveGeneration)
+      ) {
+        await tryCatchIgnore(stop2);
+      }
+      if (
+        generation1 ==
+        (startSaveFirst ? autoSaveGeneration : autoLoadGeneration)
+      ) {
+        await tryCatchIgnore(stop1);
+      }
+      throw error;
+    }
     return persister;
   };
   const stopAutoPersisting = async (stopSaveFirst = false) => {
@@ -1630,6 +1776,7 @@ const createWsServer = (
   const clientIdListeners = mapNew();
   const paths = mapNew();
   const webSockets = setNew();
+  const buffersByWebSocket = mapNew();
   const removeServerListeners = [];
   const removeListenersByWebSocket = mapNew();
   let destroying;
@@ -1679,7 +1826,7 @@ const createWsServer = (
         serverClient[3 /* Send */] = createPayloadReceiver(
           receive,
           requestTimeoutSeconds,
-        );
+        )[0];
       },
       noop,
       requestTimeoutSeconds,
@@ -1743,18 +1890,37 @@ const createWsServer = (
       client.send(outgoingPayload);
     }
   };
+  const updateWebSocketBuffer = (webSocket, count, size2) => {
+    const webSocketBuffer = mapEnsure(buffersByWebSocket, webSocket, () => [
+      0, 0,
+    ]);
+    webSocketBuffer[0] += count;
+    webSocketBuffer[1] += size2;
+    if (!webSocketBuffer[0]) {
+      mapSet(buffersByWebSocket, webSocket);
+    }
+  };
+  const delBufferedFromWebSocket = ([, payload, , webSocket]) =>
+    updateWebSocketBuffer(webSocket, -1, -getWebSocketPayloadSize(payload));
   const clearServerBuffer = (serverClient) => {
     if (serverClient[7 /* BufferTimeout */]) {
       stopTimeout(serverClient[7 /* BufferTimeout */]);
       serverClient[7 /* BufferTimeout */] = void 0;
     }
+    arrayForEach(serverClient[4 /* Buffer */], delBufferedFromWebSocket);
     serverClient[4 /* Buffer */] = [];
     serverClient[6 /* BufferSize */] = 0;
   };
   const delBufferedClient = (serverClient, clientId) => {
     serverClient[4 /* Buffer */] = arrayFilter(
       serverClient[4 /* Buffer */],
-      ([bufferedClientId]) => bufferedClientId != clientId,
+      (buffered) => {
+        if (buffered[0] == clientId) {
+          delBufferedFromWebSocket(buffered);
+          return false;
+        }
+        return true;
+      },
     );
     serverClient[6 /* BufferSize */] = arrayReduce(
       serverClient[4 /* Buffer */],
@@ -1768,10 +1934,8 @@ const createWsServer = (
   const expireServerBuffer = (path) => {
     const serverClient = path[1 /* ServerClient */];
     const clients = setNew();
-    arrayForEach(serverClient[4 /* Buffer */], ([clientId]) =>
-      ifNotUndefined(mapGet(path[0 /* Clients */], clientId), ([client]) =>
-        setAdd(clients, client),
-      ),
+    arrayForEach(serverClient[4 /* Buffer */], ([, , , client]) =>
+      setAdd(clients, client),
     );
     clearServerBuffer(serverClient);
     arrayForEach(collValues(clients), (client) =>
@@ -1781,6 +1945,10 @@ const createWsServer = (
   const bufferMessage = (path, clientId, payload) => {
     const serverClient = path[1 /* ServerClient */];
     const buffer = serverClient[4 /* Buffer */];
+    const webSocket = mapGet(path[0 /* Clients */], clientId)?.[0];
+    if (isUndefined(webSocket)) {
+      return;
+    }
     const coalesceKey = getPayloadCoalesceKey(clientId, payload);
     const coalesceIndex = coalesceKey
       ? arrayFindIndex(
@@ -1788,14 +1956,22 @@ const createWsServer = (
           ([, , bufferedKey]) => bufferedKey == coalesceKey,
         )
       : -1;
-    const replacedSize =
-      coalesceIndex > -1
-        ? getWebSocketPayloadSize(buffer[coalesceIndex][1])
-        : 0;
+    const replaced = coalesceIndex > -1 ? buffer[coalesceIndex] : void 0;
+    const replacedSize = isUndefined(replaced)
+      ? 0
+      : getWebSocketPayloadSize(replaced[1]);
     const payloadSize = getWebSocketPayloadSize(payload);
+    const webSocketBuffer = mapGet(buffersByWebSocket, webSocket);
+    const sameWebSocket = replaced?.[3] === webSocket;
     if (
       (coalesceIndex < 0 && size(buffer) >= MAX_WEBSOCKET_QUEUE_SIZE) ||
       serverClient[6 /* BufferSize */] + payloadSize - replacedSize >
+        MAX_WEBSOCKET_BUFFER_SIZE ||
+      (!sameWebSocket &&
+        (webSocketBuffer?.[0] ?? 0) >= MAX_WEBSOCKET_QUEUE_SIZE) ||
+      (webSocketBuffer?.[1] ?? 0) +
+        payloadSize -
+        (sameWebSocket ? replacedSize : 0) >
         MAX_WEBSOCKET_BUFFER_SIZE
     ) {
       delBufferedClient(serverClient, clientId);
@@ -1804,12 +1980,14 @@ const createWsServer = (
       );
       return;
     }
-    const buffered = [clientId, payload, coalesceKey];
+    const buffered = [clientId, payload, coalesceKey, webSocket];
     if (coalesceIndex > -1) {
+      delBufferedFromWebSocket(buffer[coalesceIndex]);
       buffer[coalesceIndex] = buffered;
     } else {
       arrayPush(buffer, buffered);
     }
+    updateWebSocketBuffer(webSocket, 1, payloadSize);
     serverClient[6 /* BufferSize */] += payloadSize - replacedSize;
     serverClient[7 /* BufferTimeout */] ??= startTimeout(
       () => expireServerBuffer(path),
@@ -1968,7 +2146,7 @@ const createWsServer = (
   };
   const addLegacyClient = async (client, clientId, pathId) => {
     const [path, ready] = addClientToPath(pathId, clientId, [client]);
-    const decode = createPayloadDecoder(
+    const [decode, clearDecoder] = createPayloadDecoder(
       (toClientId, remainders) =>
         handleDecodedMessage(path, clientId, toClientId, remainders),
       requestTimeoutSeconds,
@@ -1977,9 +2155,10 @@ const createWsServer = (
     addWebSocketListener(client, MESSAGE, (data) =>
       decode(data.toString(UTF8)),
     );
-    addWebSocketListener(client, CLOSE, () =>
-      delClientFromPath(pathId, path, clientId).catch(handleError),
-    );
+    addWebSocketListener(client, CLOSE, () => {
+      clearDecoder();
+      delClientFromPath(pathId, path, clientId).catch(handleError);
+    });
     await ready;
   };
   const addMultipleClient = (client, clientId, basePathId) => {
@@ -2124,6 +2303,7 @@ const createWsServer = (
           arrayForEach(removers, (remove) => tryReturn(remove)),
         );
         collClear(removeListenersByWebSocket);
+        collClear(buffersByWebSocket);
         collClear(paths);
         collClear(webSockets);
         const failure = serverFailure[0]