tinybase 9.4.0-beta.1 → 9.4.0

package/synchronizers/synchronizer-ws-server-simple/index.js

@@ -62,6 +62,7 @@ const arrayPush = (array, ...values) => array.push(...values);
 const collSize = (coll) => coll?.size ?? 0;
 const collHas = (coll, keyOrValue) => coll?.has(keyOrValue) ?? false;
 const collIsEmpty = (coll) => isUndefined(coll) || collSize(coll) == 0;
+const collValues = (coll) => [...(coll?.values() ?? [])];
 const collClear = (coll) => coll.clear();
 const collForEach = (coll, cb) => coll?.forEach(cb);
 const collDel = (coll, keyOrValue) => coll?.delete(keyOrValue);
@@ -153,11 +154,6 @@ const mapKeys = (map2) => [...(map2?.keys() ?? [])];
 const mapGet = (map2, key) => map2?.get(key);
 const mapForEach = (map2, cb) =>
   collForEach(map2, (value, key) => cb(key, value));
-const mapMap = (coll, cb) => {
-  const mapped = [];
-  mapForEach(coll, (key, value) => arrayPush(mapped, cb(value, key)));
-  return mapped;
-};
 const mapSet = (map2, key, value) =>
   isUndefined(value) ? (collDel(map2, key), map2) : map2?.set(key, value);
 const mapEnsure = (map2, key, getDefaultValue, hadExistingValue) => {
@@ -231,12 +227,23 @@ const isHlc = (hlc, maxLogicalTime) =>
   hlc == EMPTY_STRING ||
   (isString(hlc) && test(HLC, hlc) && decodeHlcTime(hlc) <= maxLogicalTime);
 
+const setNew = (entryOrEntries) =>
+  new Set(
+    isArray(entryOrEntries) || isUndefined(entryOrEntries)
+      ? entryOrEntries
+      : [entryOrEntries],
+  );
+const setAdd = (set, value) => set?.add(value);
+
 const MESSAGE_SEPARATOR = '\n';
 const FRAGMENT = /^([-0-9A-Z_a-z]{16})\n(\d+)\n(\d+)\n([\s\S]*)$/;
 const FRAGMENT_PAYLOAD = /^[^\n]*\n[-0-9A-Z_a-z]{16}\n\d+\n\d+\n/;
 const INVALID_CHANNEL_ID_CHARACTERS = /[\n\r?#]/;
 const MAX_FRAGMENT_BUFFERS = 100;
 const MAX_FRAGMENT_COUNT = 1e3;
+const MAX_MULTIPLE_CHANNELS = 100;
+const MAX_MULTIPLE_CHANNEL_RESOURCES = MAX_MULTIPLE_CHANNELS * 2;
+const MAX_MULTIPLE_CHANNEL_ID_SIZE = 1024;
 const MAX_WEBSOCKET_BUFFER_SIZE = 16777216;
 const getWebSocketPayloadSize = (value) => {
   let byteSize = 0;
@@ -367,21 +374,30 @@ const createPayloadDecoder = (
   receive,
   fragmentTimeoutSeconds = 1,
   onInvalid,
+  sharedBuffer,
 ) => {
   const buffer = mapNew();
+  const payloadBuffer = sharedBuffer ?? [0, 0];
   let bufferedSize = 0;
   let valid = true;
   const delPending = (bufferKey, pending) => {
     stopTimeout(pending[4]);
     bufferedSize -= pending[5];
+    payloadBuffer[0]--;
+    payloadBuffer[1] -= pending[5];
     collDel(buffer, bufferKey);
   };
+  const clear = () => {
+    mapForEach(buffer, (_bufferKey, pending) => stopTimeout(pending[4]));
+    payloadBuffer[0] -= collSize(buffer);
+    payloadBuffer[1] -= bufferedSize;
+    collClear(buffer);
+    bufferedSize = 0;
+  };
   const invalid = (error = errorNew(ERROR_SYNC_MESSAGE)) => {
     if (valid) {
       valid = false;
-      mapForEach(buffer, (_bufferKey, pending) => stopTimeout(pending[4]));
-      collClear(buffer);
-      bufferedSize = 0;
+      clear();
       onInvalid?.(error);
     }
   };
@@ -393,7 +409,7 @@ const createPayloadDecoder = (
       invalid();
     }
   };
-  return (payload) => {
+  const decode = (payload) => {
     if (!valid) {
       return;
     }
@@ -426,42 +442,52 @@ const createPayloadDecoder = (
             const bufferKey = clientId + MESSAGE_SEPARATOR + messageId;
             let pending = mapGet(buffer, bufferKey);
             if (!pending) {
-              if (collSize(buffer) >= MAX_FRAGMENT_BUFFERS) {
+              if (payloadBuffer[0] >= MAX_FRAGMENT_BUFFERS) {
                 invalid(errorNew(ERROR_SYNC_OVERFLOW, 'fragments'));
                 return;
               }
-              pending = [
-                [],
-                [],
-                total,
-                total,
-                startTimeout(() => {
-                  const timedOut = mapGet(buffer, bufferKey);
-                  if (timedOut) {
-                    delPending(bufferKey, timedOut);
-                  }
-                }, fragmentTimeoutSeconds),
-                0,
-              ];
-              mapSet(buffer, bufferKey, pending);
             }
-            const [fragments, remainders] = pending;
-            if (total == pending[3] && isUndefined(fragments[index])) {
-              const fragmentLength = getWebSocketPayloadSize(fragment);
-              if (bufferedSize + fragmentLength > MAX_WEBSOCKET_BUFFER_SIZE) {
+            if (
+              (isUndefined(pending) || total == pending[3]) &&
+              isUndefined(pending?.[0][index])
+            ) {
+              const retainedSize =
+                getWebSocketPayloadSize(remainder) +
+                (pending ? 0 : getWebSocketPayloadSize(bufferKey));
+              if (payloadBuffer[1] + retainedSize > MAX_WEBSOCKET_BUFFER_SIZE) {
                 invalid(errorNew(ERROR_SYNC_OVERFLOW, 'fragments'));
                 return;
               }
+              if (!pending) {
+                pending = [
+                  [],
+                  [],
+                  total,
+                  total,
+                  startTimeout(() => {
+                    const timedOut = mapGet(buffer, bufferKey);
+                    if (timedOut) {
+                      delPending(bufferKey, timedOut);
+                    }
+                  }, fragmentTimeoutSeconds),
+                  0,
+                ];
+                mapSet(buffer, bufferKey, pending);
+                payloadBuffer[0]++;
+              }
+              const [fragments, remainders] = pending;
               fragments[index] = fragment;
               remainders[index] = remainder;
               pending[2]--;
-              pending[5] += fragmentLength;
-              bufferedSize += fragmentLength;
+              pending[5] += retainedSize;
+              bufferedSize += retainedSize;
+              payloadBuffer[1] += retainedSize;
             } else {
               invalid();
               return;
             }
             if (pending[2] == 0) {
+              const [fragments, remainders] = pending;
               delPending(bufferKey, pending);
               receiveRemainder(clientId, remainders, arrayJoin(fragments));
             }
@@ -474,6 +500,7 @@ const createPayloadDecoder = (
       invalid();
     }
   };
+  return [decode, clear];
 };
 const isWebSocketBackpressured = (webSocket, payloadSize) =>
   (webSocket.bufferedAmount ?? 0) + payloadSize > MAX_WEBSOCKET_BUFFER_SIZE;
@@ -481,12 +508,15 @@ const createRawPayload = (clientId, remainder) =>
   clientId + MESSAGE_SEPARATOR + remainder;
 const createMultiplePayload = (channelId, payload) =>
   createRawPayload(MULTIPLE_CLIENT_ID, createRawPayload(channelId, payload));
-const ifMultiplePayloadValid = (payload, then) => {
+const ifMultiplePayloadValid = (payload, then, onOversized) => {
   let valid = false;
   ifPayloadValid(payload, (multipleClientId, remainder) => {
     if (multipleClientId == MULTIPLE_CLIENT_ID) {
       ifPayloadValid(remainder, (channelId, channelPayload) => {
-        if (isMultipleChannelIdValid(channelId)) {
+        if (isMultipleChannelIdTooLarge(channelId)) {
+          valid = true;
+          onOversized?.();
+        } else if (isMultipleChannelIdStructureValid(channelId)) {
           valid = true;
           then(channelId, channelPayload);
         }
@@ -500,6 +530,15 @@ const createMultipleControlPayload = (requestId, control, body) =>
     SERVER_CLIENT_ID,
     jsonStringWithUndefined([requestId, MULTIPLE_MESSAGE, [control, body]]),
   );
+const isMultipleChannelIdTooLarge = (channelId) =>
+  getWebSocketPayloadSize(channelId) > MAX_MULTIPLE_CHANNEL_ID_SIZE;
+const isMultipleChannelIdStructureValid = (channelId) =>
+  !isEmpty(channelId) &&
+  !INVALID_CHANNEL_ID_CHARACTERS.test(channelId) &&
+  arrayEvery(
+    strSplit(channelId, '/'),
+    (part) => !isEmpty(part) && part != '.' && part != '..',
+  );
 const ifMultipleControlPayloadValid = (payload, then) => {
   let valid = false;
   ifPayloadValid(payload, (serverClientId, remainder) => {
@@ -517,11 +556,13 @@ const ifMultipleControlPayloadValid = (payload, then) => {
           : control == 1 /* Subscribe */
             ? isString(requestId) &&
               isString(body) &&
-              isMultipleChannelIdValid(body)
+              (isMultipleChannelIdTooLarge(body) ||
+                isMultipleChannelIdStructureValid(body))
             : control == 2 /* Unsubscribe */ &&
               isNull(requestId) &&
               isString(body) &&
-              isMultipleChannelIdValid(body))
+              (isMultipleChannelIdTooLarge(body) ||
+                isMultipleChannelIdStructureValid(body)))
       ) {
         valid = true;
         then(requestId, control, body);
@@ -530,13 +571,6 @@ const ifMultipleControlPayloadValid = (payload, then) => {
   });
   return valid;
 };
-const isMultipleChannelIdValid = (channelId) =>
-  !isEmpty(channelId) &&
-  !INVALID_CHANNEL_ID_CHARACTERS.test(channelId) &&
-  arrayEvery(
-    strSplit(channelId, '/'),
-    (part) => !isEmpty(part) && part != '.' && part != '..',
-  );
 const createMultipleServerClient = (
   basePathId,
   addChannel,
@@ -549,26 +583,87 @@ const createMultipleServerClient = (
 ) => {
   const channels = mapNew();
   const decoders = mapNew();
+  const payloadBuffer = [0, 0];
+  const resources = setNew();
+  let resourceCount = 0;
   let negotiated = false;
+  let valid = true;
   const sendControl = (requestId, control, body) =>
     send(createMultipleControlPayload(requestId, control, body));
   const delChannelAndDecoder = (channelId) => {
     collDel(channels, channelId);
+    mapGet(decoders, channelId)?.[1]();
     collDel(decoders, channelId);
   };
+  const clearDecoders = () => {
+    mapForEach(decoders, (_channelId, decoder) => decoder[1]());
+    collClear(decoders);
+  };
+  const invalidate = (error) => {
+    if (valid) {
+      valid = false;
+      clearDecoders();
+      invalid(error);
+    }
+  };
+  const releaseChannel = (channel) => {
+    if (!channel[2 /* Released */]) {
+      channel[2 /* Released */] = 1;
+      resourceCount--;
+      collDel(resources, channel);
+    }
+  };
+  const teardownChannel = (channel) =>
+    (channel[1 /* Teardown */] ??= (async () => {
+      try {
+        const teardown = delChannel(channel[0 /* Channel */]);
+        if (isUndefined(teardown)) {
+          releaseChannel(channel);
+        } else {
+          await teardown;
+        }
+      } finally {
+        releaseChannel(channel);
+      }
+    })());
   const handleControl = (requestId, control, body) => {
     if (control == 0 /* Hello */) {
       negotiated = true;
       sendControl(requestId, control, body);
     } else if (negotiated && control == 1 /* Subscribe */) {
-      if (!collHas(channels, body)) {
+      const subscribed = collHas(channels, body);
+      if (
+        isMultipleChannelIdTooLarge(body) ||
+        (!subscribed &&
+          (collSize(channels) >= MAX_MULTIPLE_CHANNELS ||
+            resourceCount >= MAX_MULTIPLE_CHANNEL_RESOURCES))
+      ) {
+        invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'channels'));
+        return;
+      }
+      if (!subscribed) {
         const pathId = basePathId + (basePathId ? '/' : EMPTY_STRING) + body;
-        const [channel, ready] = addChannel(pathId, body);
+        resourceCount++;
+        let channelAndReady;
+        try {
+          channelAndReady = addChannel(pathId, body);
+        } catch (error) {
+          resourceCount--;
+          throw error;
+        }
+        const [addedChannel, ready] = channelAndReady;
+        const channel = [addedChannel];
+        setAdd(resources, channel);
         mapSet(channels, body, channel);
         sendControl(requestId, control, body);
         if (ready) {
           return ready.catch((error) => {
-            delChannelAndDecoder(body);
+            if (mapGet(channels, body) === channel) {
+              delChannelAndDecoder(body);
+            }
+            if (!channel[1 /* Teardown */]) {
+              releaseChannel(channel);
+            }
             throw error;
           });
         }
@@ -576,16 +671,21 @@ const createMultipleServerClient = (
       }
       sendControl(requestId, control, body);
     } else if (negotiated && control == 2 /* Unsubscribe */) {
+      if (isMultipleChannelIdTooLarge(body)) {
+        invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'channels'));
+        return;
+      }
       const channel = mapGet(channels, body);
       delChannelAndDecoder(body);
-      if (!isUndefined(channel)) {
-        return (async () => delChannel(channel))();
-      }
+      return isUndefined(channel) ? void 0 : teardownChannel(channel);
     }
   };
   const handlePayload = (payload) => {
+    if (!valid) {
+      return;
+    }
     if (isWebSocketPayloadTooLarge(getWebSocketPayloadSize(payload))) {
-      invalid(errorNew(ERROR_SYNC_OVERFLOW, 'socket'));
+      invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'socket'));
       return;
     }
     const control = ifMultipleControlPayloadValid(
@@ -600,32 +700,54 @@ const createMultipleServerClient = (
       (channelId, channelPayload) => {
         const channel2 = negotiated ? mapGet(channels, channelId) : void 0;
         if (isUndefined(channel2)) {
-          invalid(errorNew(ERROR_SYNC_MESSAGE));
+          invalidate(errorNew(ERROR_SYNC_MESSAGE));
         } else {
           mapEnsure(decoders, channelId, () =>
             createPayloadDecoder(
               (toClientId, remainders) =>
-                receive(channel2, toClientId, remainders),
+                receive(channel2[0 /* Channel */], toClientId, remainders),
               fragmentTimeoutSeconds,
-              invalid,
+              invalidate,
+              payloadBuffer,
             ),
-          )(channelPayload);
+          )[0](channelPayload);
         }
       },
+      () => invalidate(errorNew(ERROR_SYNC_OVERFLOW, 'channels')),
     );
     if (!control && !channel) {
-      invalid(errorNew(ERROR_SYNC_MESSAGE));
+      invalidate(errorNew(ERROR_SYNC_MESSAGE));
     }
   };
-  const destroy = () =>
-    tryFinallyAsync(
-      () =>
-        promiseAll(mapMap(channels, async (channel) => delChannel(channel))),
+  const destroy = () => {
+    valid = false;
+    clearDecoders();
+    return tryFinallyAsync(
+      async () => {
+        let errorToThrow;
+        let failed = false;
+        await promiseAll(
+          arrayMap(collValues(resources), async (channel) => {
+            try {
+              await teardownChannel(channel);
+            } catch (error) {
+              if (!failed) {
+                errorToThrow = error;
+                failed = true;
+              }
+            }
+          }),
+        );
+        if (failed) {
+          throw errorToThrow;
+        }
+      },
       () => {
         collClear(channels);
-        collClear(decoders);
+        clearDecoders();
       },
     );
+  };
   return [handlePayload, destroy];
 };
 
@@ -714,7 +836,7 @@ const createWsServerSimple = (webSocketServer) => {
   };
   const addLegacyClient = (client, clientId, pathId) => {
     addClientToPath(pathId, clientId, [client]);
-    const decode = createPayloadDecoder(
+    const [decode, clearDecoder] = createPayloadDecoder(
       (toClientId, remainders) =>
         handleDecodedMessage(pathId, clientId, toClientId, remainders),
       1,
@@ -723,8 +845,10 @@ const createWsServerSimple = (webSocketServer) => {
     addWebSocketListener(client, MESSAGE, (data) =>
       decode(data.toString(UTF8)),
     );
-    mapGet(webSocketStates, client)[1] = () =>
+    mapGet(webSocketStates, client)[1] = () => {
+      clearDecoder();
       delClientFromPath(pathId, clientId);
+    };
   };
   const addMultipleClient = (client, clientId, basePathId) => {
     const invalid = createInvalidPayloadHandler(client);