tinybase 9.4.0-beta.1 → 9.4.0

package/synchronizers/synchronizer-ws-client/with-schemas/index.js

@@ -228,6 +228,8 @@ const tryCatch = async (action, onError) => {
     onError?.(error);
   }
 };
+const tryCatchIgnore = (action, onIgnoredError) =>
+  tryCatch(action, (error) => void tryCatch(() => onIgnoredError?.(error)));
 const tryCatchSync = (action, onError) => {
   try {
     return action();
@@ -321,6 +323,8 @@ const FRAGMENT_PAYLOAD = /^[^\n]*\n[-0-9A-Z_a-z]{16}\n\d+\n\d+\n/;
 const INVALID_CHANNEL_ID_CHARACTERS = /[\n\r?#]/;
 const MAX_FRAGMENT_BUFFERS = 100;
 const MAX_FRAGMENT_COUNT = 1e3;
+const MAX_MULTIPLE_CHANNELS = 100;
+const MAX_MULTIPLE_CHANNEL_ID_SIZE = 1024;
 const MAX_PENDING_REQUESTS = 100;
 const MAX_WEBSOCKET_BUFFER_SIZE = 16777216;
 const MAX_WEBSOCKET_QUEUE_SIZE = 1e3;
@@ -458,21 +462,30 @@ const createPayloadDecoder = (
   receive,
   fragmentTimeoutSeconds = 1,
   onInvalid,
+  sharedBuffer,
 ) => {
   const buffer = mapNew();
+  const payloadBuffer = sharedBuffer ?? [0, 0];
   let bufferedSize = 0;
   let valid = true;
   const delPending = (bufferKey, pending) => {
     stopTimeout(pending[4]);
     bufferedSize -= pending[5];
+    payloadBuffer[0]--;
+    payloadBuffer[1] -= pending[5];
     collDel(buffer, bufferKey);
   };
+  const clear = () => {
+    mapForEach(buffer, (_bufferKey, pending) => stopTimeout(pending[4]));
+    payloadBuffer[0] -= collSize(buffer);
+    payloadBuffer[1] -= bufferedSize;
+    collClear(buffer);
+    bufferedSize = 0;
+  };
   const invalid = (error = errorNew(ERROR_SYNC_MESSAGE)) => {
     if (valid) {
       valid = false;
-      mapForEach(buffer, (_bufferKey, pending) => stopTimeout(pending[4]));
-      collClear(buffer);
-      bufferedSize = 0;
+      clear();
       onInvalid?.(error);
     }
   };
@@ -484,7 +497,7 @@ const createPayloadDecoder = (
       invalid();
     }
   };
-  return (payload) => {
+  const decode = (payload) => {
     if (!valid) {
       return;
     }
@@ -517,42 +530,52 @@ const createPayloadDecoder = (
             const bufferKey = clientId + MESSAGE_SEPARATOR + messageId;
             let pending = mapGet(buffer, bufferKey);
             if (!pending) {
-              if (collSize(buffer) >= MAX_FRAGMENT_BUFFERS) {
+              if (payloadBuffer[0] >= MAX_FRAGMENT_BUFFERS) {
                 invalid(errorNew(ERROR_SYNC_OVERFLOW, 'fragments'));
                 return;
               }
-              pending = [
-                [],
-                [],
-                total,
-                total,
-                startTimeout(() => {
-                  const timedOut = mapGet(buffer, bufferKey);
-                  if (timedOut) {
-                    delPending(bufferKey, timedOut);
-                  }
-                }, fragmentTimeoutSeconds),
-                0,
-              ];
-              mapSet(buffer, bufferKey, pending);
             }
-            const [fragments, remainders] = pending;
-            if (total == pending[3] && isUndefined(fragments[index])) {
-              const fragmentLength = getWebSocketPayloadSize(fragment);
-              if (bufferedSize + fragmentLength > MAX_WEBSOCKET_BUFFER_SIZE) {
+            if (
+              (isUndefined(pending) || total == pending[3]) &&
+              isUndefined(pending?.[0][index])
+            ) {
+              const retainedSize =
+                getWebSocketPayloadSize(remainder) +
+                (pending ? 0 : getWebSocketPayloadSize(bufferKey));
+              if (payloadBuffer[1] + retainedSize > MAX_WEBSOCKET_BUFFER_SIZE) {
                 invalid(errorNew(ERROR_SYNC_OVERFLOW, 'fragments'));
                 return;
               }
+              if (!pending) {
+                pending = [
+                  [],
+                  [],
+                  total,
+                  total,
+                  startTimeout(() => {
+                    const timedOut = mapGet(buffer, bufferKey);
+                    if (timedOut) {
+                      delPending(bufferKey, timedOut);
+                    }
+                  }, fragmentTimeoutSeconds),
+                  0,
+                ];
+                mapSet(buffer, bufferKey, pending);
+                payloadBuffer[0]++;
+              }
+              const [fragments, remainders] = pending;
               fragments[index] = fragment;
               remainders[index] = remainder;
               pending[2]--;
-              pending[5] += fragmentLength;
-              bufferedSize += fragmentLength;
+              pending[5] += retainedSize;
+              bufferedSize += retainedSize;
+              payloadBuffer[1] += retainedSize;
             } else {
               invalid();
               return;
             }
             if (pending[2] == 0) {
+              const [fragments, remainders] = pending;
               delPending(bufferKey, pending);
               receiveRemainder(clientId, remainders, arrayJoin(fragments));
             }
@@ -565,6 +588,7 @@ const createPayloadDecoder = (
       invalid();
     }
   };
+  return [decode, clear];
 };
 const isWebSocketBackpressured = (webSocket, payloadSize) =>
   (webSocket.bufferedAmount ?? 0) + payloadSize > MAX_WEBSOCKET_BUFFER_SIZE;
@@ -572,12 +596,14 @@ const createPayloadReceiver = (
   receive,
   fragmentTimeoutSeconds = 1,
   onInvalid,
+  sharedBuffer,
 ) =>
   createPayloadDecoder(
     (fromClientId, _remainders, requestId, message, body) =>
       receive(fromClientId, requestId, message, body),
     fragmentTimeoutSeconds,
     onInvalid,
+    sharedBuffer,
   );
 const createRawPayload = (clientId, remainder) =>
   clientId + MESSAGE_SEPARATOR + remainder;
@@ -610,12 +636,15 @@ const getFragments = (remainder, maxFragmentSize) => {
 };
 const createMultiplePayload = (channelId, payload) =>
   createRawPayload(MULTIPLE_CLIENT_ID, createRawPayload(channelId, payload));
-const ifMultiplePayloadValid = (payload, then) => {
+const ifMultiplePayloadValid = (payload, then, onOversized) => {
   let valid = false;
   ifPayloadValid(payload, (multipleClientId, remainder) => {
     if (multipleClientId == MULTIPLE_CLIENT_ID) {
       ifPayloadValid(remainder, (channelId, channelPayload) => {
-        if (isMultipleChannelIdValid(channelId)) {
+        if (isMultipleChannelIdTooLarge(channelId)) {
+          valid = true;
+          onOversized?.();
+        } else if (isMultipleChannelIdStructureValid(channelId)) {
           valid = true;
           then(channelId, channelPayload);
         }
@@ -629,6 +658,15 @@ const createMultipleControlPayload = (requestId, control, body) =>
     SERVER_CLIENT_ID,
     jsonStringWithUndefined([requestId, MULTIPLE_MESSAGE, [control, body]]),
   );
+const isMultipleChannelIdTooLarge = (channelId) =>
+  getWebSocketPayloadSize(channelId) > MAX_MULTIPLE_CHANNEL_ID_SIZE;
+const isMultipleChannelIdStructureValid = (channelId) =>
+  !isEmpty(channelId) &&
+  !INVALID_CHANNEL_ID_CHARACTERS.test(channelId) &&
+  arrayEvery(
+    strSplit(channelId, '/'),
+    (part) => !isEmpty(part) && part != '.' && part != '..',
+  );
 const ifMultipleControlPayloadValid = (payload, then) => {
   let valid = false;
   ifPayloadValid(payload, (serverClientId, remainder) => {
@@ -646,11 +684,13 @@ const ifMultipleControlPayloadValid = (payload, then) => {
           : control == 1 /* Subscribe */
             ? isString(requestId) &&
               isString(body) &&
-              isMultipleChannelIdValid(body)
+              (isMultipleChannelIdTooLarge(body) ||
+                isMultipleChannelIdStructureValid(body))
             : control == 2 /* Unsubscribe */ &&
               isNull(requestId) &&
               isString(body) &&
-              isMultipleChannelIdValid(body))
+              (isMultipleChannelIdTooLarge(body) ||
+                isMultipleChannelIdStructureValid(body)))
       ) {
         valid = true;
         then(requestId, control, body);
@@ -660,12 +700,8 @@ const ifMultipleControlPayloadValid = (payload, then) => {
   return valid;
 };
 const isMultipleChannelIdValid = (channelId) =>
-  !isEmpty(channelId) &&
-  !INVALID_CHANNEL_ID_CHARACTERS.test(channelId) &&
-  arrayEvery(
-    strSplit(channelId, '/'),
-    (part) => !isEmpty(part) && part != '.' && part != '..',
-  );
+  !isMultipleChannelIdTooLarge(channelId) &&
+  isMultipleChannelIdStructureValid(channelId);
 const createPayloads = (toClientId, requestId, message, body, fragmentSize) => {
   const clientId = toClientId ?? EMPTY_STRING;
   const remainder = jsonStringWithUndefined([requestId, message, body]);
@@ -1138,11 +1174,35 @@ const createCustomPersister = (
     initialContent,
     startSaveFirst = false,
   ) => {
-    const [call1, call2] = startSaveFirst
-      ? [startAutoSave, startAutoLoad]
-      : [startAutoLoad, startAutoSave];
-    await call1(initialContent);
-    await call2(initialContent);
+    const [call1, call2, stop1, stop2] = startSaveFirst
+      ? [startAutoSave, startAutoLoad, stopAutoSave, stopAutoLoad]
+      : [startAutoLoad, startAutoSave, stopAutoLoad, stopAutoSave];
+    const start1 = call1(initialContent);
+    const generation1 = startSaveFirst
+      ? autoSaveGeneration
+      : autoLoadGeneration;
+    await start1;
+    const start2 = call2(initialContent);
+    const generation2 = startSaveFirst
+      ? autoLoadGeneration
+      : autoSaveGeneration;
+    try {
+      await start2;
+    } catch (error) {
+      if (
+        generation2 ==
+        (startSaveFirst ? autoLoadGeneration : autoSaveGeneration)
+      ) {
+        await tryCatchIgnore(stop2);
+      }
+      if (
+        generation1 ==
+        (startSaveFirst ? autoSaveGeneration : autoLoadGeneration)
+      ) {
+        await tryCatchIgnore(stop1);
+      }
+      throw error;
+    }
     return persister;
   };
   const stopAutoPersisting = async (stopSaveFirst = false) => {
@@ -1543,6 +1603,7 @@ const createConnection = () => {
 const createMultipleState = (webSocket) => {
   const channels = mapNew();
   const pendingControls = mapNew();
+  const payloadBuffer = [0, 0];
   const removeListeners = [];
   let connection = createConnection();
   let connected = false;
@@ -1554,6 +1615,7 @@ const createMultipleState = (webSocket) => {
   let overflowing = false;
   let queuedCount = 0;
   let queuedSize = 0;
+  let receiving = true;
   const notifyChannelIgnoredError = (channel, error) =>
     tryReturn(() => channel[6 /* OnIgnoredError */]?.(error));
   const notifyIgnoredError = (error) =>
@@ -1565,11 +1627,10 @@ const createMultipleState = (webSocket) => {
     disconnect();
     tryReturn(() => webSocket.close());
   };
-  const invalid = createInvalidPayloadHandler(webSocket, notifyIgnoredError);
   const addWebSocketListener = (event, handler) =>
     arrayPush(removeListeners, addEventListener(webSocket, event, handler));
   const rejectPendingControls = (error) =>
-    mapForEach(pendingControls, (requestId, [, , reject, timeout]) => {
+    mapForEach(pendingControls, (requestId, [, , , reject, timeout]) => {
       stopTimeout(timeout);
       collDel(pendingControls, requestId);
       reject(error);
@@ -1600,11 +1661,43 @@ const createMultipleState = (webSocket) => {
       channel[7 /* Dirty */] = void 0;
     }
   };
+  const clearReceive = (channel) => channel[0 /* Receive */]?.[1]();
+  const clearReceives = () =>
+    mapForEach(channels, (_channelId, channel) => clearReceive(channel));
+  const handleInvalid = createInvalidPayloadHandler(
+    webSocket,
+    notifyIgnoredError,
+  );
+  const invalid = (error) => {
+    if (receiving) {
+      receiving = false;
+      disconnected = true;
+      connected = false;
+      clearReceives();
+      connection[2](error);
+      rejectPendingControls(error);
+      failChannels(error);
+      mapForEach(channels, (_channelId, channel) => {
+        clearIncoming(channel);
+        clearOutgoing(channel, true);
+        channel[3 /* Subscribed */] = false;
+        channel[4 /* Subscribing */] = void 0;
+        channel[12 /* SubscribeConnection */] = void 0;
+      });
+      if (flushTimeout) {
+        stopTimeout(flushTimeout);
+        flushTimeout = void 0;
+      }
+      handleInvalid(error);
+    }
+  };
   const overflow = (details) => {
     const error = errorNew(ERROR_SYNC_OVERFLOW, details);
     if (!overflowing && !destroyed) {
       overflowing = true;
+      receiving = false;
       connected = false;
+      clearReceives();
       notifyIgnoredError(error);
       rejectPendingControls(error);
       failChannels(error);
@@ -1613,6 +1706,7 @@ const createMultipleState = (webSocket) => {
         clearOutgoing(channel, true);
         channel[3 /* Subscribed */] = false;
         channel[4 /* Subscribing */] = void 0;
+        channel[12 /* SubscribeConnection */] = void 0;
       });
       if (flushTimeout) {
         stopTimeout(flushTimeout);
@@ -1634,7 +1728,7 @@ const createMultipleState = (webSocket) => {
     });
     return timeoutSeconds;
   };
-  const sendControl = (control, body, timeoutSeconds) => {
+  const sendControl = (control, body, timeoutSeconds, onSent) => {
     const requestId = getUniqueId();
     return promiseNew((resolve, reject) => {
       if (collSize(pendingControls) >= MAX_PENDING_REQUESTS) {
@@ -1654,9 +1748,16 @@ const createMultipleState = (webSocket) => {
         collDel(pendingControls, requestId);
         reject(errorNew(ERROR_MULTIPLEX_RESPONSE, control));
       }, timeoutSeconds);
-      mapSet(pendingControls, requestId, [control, resolve, reject, timeout]);
+      mapSet(pendingControls, requestId, [
+        control,
+        body,
+        resolve,
+        reject,
+        timeout,
+      ]);
       try {
         webSocket.send(payload);
+        onSent?.();
       } catch (error) {
         stopTimeout(timeout);
         collDel(pendingControls, requestId);
@@ -1839,23 +1940,45 @@ const createMultipleState = (webSocket) => {
   const subscribe = async (channelId, channel, timeoutSeconds) => {
     if (!channel[3 /* Subscribed */]) {
       let subscribing = channel[4 /* Subscribing */];
-      if (!subscribing) {
-        subscribing = (async () => {
-          await connection[0];
+      const subscribingConnection = connection;
+      if (subscribing?.[0] !== subscribingConnection) {
+        const subscribingPromise = (async () => {
+          await subscribingConnection[0];
+          if (
+            mapGet(channels, channelId) !== channel ||
+            connection !== subscribingConnection ||
+            !connected
+          ) {
+            return;
+          }
           await sendControl(
             MultipleControl.Subscribe,
             channelId,
             timeoutSeconds,
+            () => {
+              if (
+                mapGet(channels, channelId) === channel &&
+                connection === subscribingConnection &&
+                connected
+              ) {
+                channel[12 /* SubscribeConnection */] = subscribingConnection;
+              }
+            },
           );
-          if (mapGet(channels, channelId) === channel && connected) {
+          if (
+            mapGet(channels, channelId) === channel &&
+            connection === subscribingConnection &&
+            connected
+          ) {
             channel[3 /* Subscribed */] = true;
             flushOutgoing(channelId, channel);
           }
         })();
+        subscribing = [subscribingConnection, subscribingPromise];
         channel[4 /* Subscribing */] = subscribing;
       }
       try {
-        await subscribing;
+        await subscribing[1];
       } finally {
         if (channel[4 /* Subscribing */] === subscribing) {
           channel[4 /* Subscribing */] = void 0;
@@ -1864,7 +1987,7 @@ const createMultipleState = (webSocket) => {
     }
   };
   const onOpen = () => {
-    if (destroyed || opening?.[0] === connection) {
+    if (destroyed || !receiving || connected || opening?.[0] === connection) {
       return;
     }
     disconnected = false;
@@ -1877,7 +2000,7 @@ const createMultipleState = (webSocket) => {
           MULTIPLE_VERSION,
           getConnectionTimeoutSeconds(),
         );
-        if (connection === openingConnection) {
+        if (connection === openingConnection && receiving && !destroyed) {
           connected = true;
           openingConnection[1]();
           mapForEach(channels, (channelId, channel) =>
@@ -1925,13 +2048,18 @@ const createMultipleState = (webSocket) => {
       }
       connection = createConnection();
       mapForEach(channels, (_channelId, channel) => {
+        clearReceive(channel);
         channel[3 /* Subscribed */] = false;
         channel[4 /* Subscribing */] = void 0;
+        channel[12 /* SubscribeConnection */] = void 0;
       });
     }
   };
   const onClose = disconnect;
   addWebSocketListener(MESSAGE, ({data}) => {
+    if (!receiving) {
+      return;
+    }
     const payload = data.toString(UTF8);
     if (isWebSocketPayloadTooLarge(getWebSocketPayloadSize(payload))) {
       invalid(errorNew(ERROR_SYNC_OVERFLOW, 'socket'));
@@ -1939,14 +2067,23 @@ const createMultipleState = (webSocket) => {
     }
     const control = ifMultipleControlPayloadValid(
       payload,
-      (requestId, control2, _body) => {
+      (requestId, control2, body) => {
+        if (
+          (control2 == MultipleControl.Subscribe ||
+            control2 == MultipleControl.Unsubscribe) &&
+          isString(body) &&
+          !isMultipleChannelIdValid(body)
+        ) {
+          overflow('channels');
+          return;
+        }
         const pendingControl = requestId
           ? mapGet(pendingControls, requestId)
           : void 0;
-        if (pendingControl?.[0] == control2) {
-          stopTimeout(pendingControl[3]);
+        if (pendingControl?.[0] == control2 && pendingControl[1] === body) {
+          stopTimeout(pendingControl[4]);
           collDel(pendingControls, requestId);
-          pendingControl[1]();
+          pendingControl[2]();
         }
       },
     );
@@ -1956,7 +2093,7 @@ const createMultipleState = (webSocket) => {
         const channel2 = mapGet(channels, channelId);
         if (channel2) {
           if (channel2[0 /* Receive */]) {
-            channel2[0 /* Receive */](channelPayload);
+            channel2[0 /* Receive */][0](channelPayload);
           } else {
             queueIncoming(channel2, channelPayload);
           }
@@ -1964,6 +2101,7 @@ const createMultipleState = (webSocket) => {
           invalid(errorNew(ERROR_SYNC_MESSAGE));
         }
       },
+      () => overflow('channels'),
     );
     if (!control && !channel) {
       invalid(errorNew(ERROR_SYNC_MESSAGE));
@@ -1976,12 +2114,18 @@ const createMultipleState = (webSocket) => {
     failChannels(errorNew(ERROR_MULTIPLEX_SOCKET));
   });
   const addChannel = async (channelId, timeoutSeconds, onIgnoredError) => {
+    if (!receiving || destroyed) {
+      errorThrow(ERROR_MULTIPLEX_SOCKET);
+    }
     if (!isMultipleChannelIdValid(channelId)) {
       errorThrow(ERROR_MULTIPLEX_CHANNEL, channelId);
     }
     if (collHas(channels, channelId)) {
       errorThrow(ERROR_MULTIPLEX_CHANNEL_DUPLICATE, channelId);
     }
+    if (collSize(channels) >= MAX_MULTIPLE_CHANNELS) {
+      errorThrow(ERROR_SYNC_OVERFLOW, 'channels');
+    }
     const channel = [
       void 0,
       [],
@@ -1995,6 +2139,7 @@ const createMultipleState = (webSocket) => {
       0,
       0,
       void 0,
+      void 0,
     ];
     mapSet(channels, channelId, channel);
     if (webSocket.readyState == webSocket.OPEN) {
@@ -2028,7 +2173,7 @@ const createMultipleState = (webSocket) => {
             queuedCount--;
             queuedSize -= payloadSize;
             incomingIndex++;
-            receive(payload);
+            receive[0](payload);
           }
         },
         () => arrayClear(incoming, incomingIndex),
@@ -2036,6 +2181,9 @@ const createMultipleState = (webSocket) => {
     }
   };
   const send = (channelId, payloads, coalesce) => {
+    if (!receiving) {
+      return;
+    }
     const channel = mapGet(channels, channelId);
     if (channel) {
       if (
@@ -2056,6 +2204,9 @@ const createMultipleState = (webSocket) => {
   const destroyIfEmpty = () => {
     if (collIsEmpty(channels) && !destroyed) {
       destroyed = true;
+      const closeWebSocket = receiving && !overflowing;
+      receiving = false;
+      clearReceives();
       const error = errorNew(ERROR_MULTIPLEX_DESTROYED);
       rejectPendingControls(error);
       if (flushTimeout) {
@@ -2067,7 +2218,7 @@ const createMultipleState = (webSocket) => {
       );
       arrayClear(removeListeners);
       multipleStates.delete(webSocket);
-      if (!overflowing) {
+      if (closeWebSocket) {
         tryReturn(() => webSocket.close());
       }
     }
@@ -2076,9 +2227,10 @@ const createMultipleState = (webSocket) => {
     const channel = mapGet(channels, channelId);
     if (channel) {
       try {
+        clearReceive(channel);
         clearIncoming(channel);
         clearOutgoing(channel, true);
-        if (connected && channel[3 /* Subscribed */]) {
+        if (connected && channel[12 /* SubscribeConnection */] === connection) {
           const payload = createMultipleControlPayload(
             null,
             MultipleControl.Unsubscribe,
@@ -2107,6 +2259,7 @@ const createMultipleState = (webSocket) => {
     delChannel,
     destroyIfEmpty,
     invalid,
+    payloadBuffer,
     registerReceive,
     send,
   };
@@ -2162,7 +2315,12 @@ const createMultipleWsSynchronizer = async (
       (receive, fail) =>
         state.registerReceive(
           channelId,
-          createPayloadReceiver(receive, requestTimeoutSeconds, state.invalid),
+          createPayloadReceiver(
+            receive,
+            requestTimeoutSeconds,
+            state.invalid,
+            state.payloadBuffer,
+          ),
           fail,
         ),
       () => state.delChannel(channelId),
@@ -2204,13 +2362,14 @@ const createLegacyWsSynchronizer = async (
   };
   const registerReceive = (receive, fail) => {
     const invalid = createInvalidPayloadHandler(webSocket, notifyIgnoredError);
-    const receivePayload = createPayloadReceiver(
+    const [receivePayload, clearReceivePayload] = createPayloadReceiver(
       receive,
       requestTimeoutSeconds,
       invalid,
     );
     let removeListeners = [];
     removeTransportListeners = () => {
+      clearReceivePayload();
       arrayForEach(removeListeners, (removeListener) =>
         tryReturn(removeListener),
       );