wslink 1.12.4__py3-none-any.whl → 2.0.0__py3-none-any.whl

wslink/chunking.py

@@ -0,0 +1,214 @@
+import sys
+import secrets
+import msgpack
+from typing import Dict, Tuple, Union
+if sys.version_info >= (3, 8):
+    from typing import TypedDict  # pylint: disable=no-name-in-module
+else:
+    from typing_extensions import TypedDict
+
+UINT32_LENGTH = 4
+ID_LOCATION = 0
+ID_LENGTH = UINT32_LENGTH
+MESSAGE_OFFSET_LOCATION = ID_LOCATION + ID_LENGTH
+MESSAGE_OFFSET_LENGTH = UINT32_LENGTH
+MESSAGE_SIZE_LOCATION = MESSAGE_OFFSET_LOCATION + MESSAGE_OFFSET_LENGTH
+MESSAGE_SIZE_LENGTH = UINT32_LENGTH
+
+HEADER_LENGTH = ID_LENGTH + MESSAGE_OFFSET_LENGTH + MESSAGE_SIZE_LENGTH
+
+
+def _encode_header(id: bytes, offset: int, size: int) -> bytes:
+    return (
+        id
+        + offset.to_bytes(MESSAGE_OFFSET_LENGTH, "little", signed=False)
+        + size.to_bytes(MESSAGE_SIZE_LENGTH, "little", signed=False)
+    )
+
+
+def _decode_header(header: bytes) -> Tuple[bytes, int, int]:
+    id = header[ID_LOCATION:ID_LENGTH]
+    offset = int.from_bytes(
+        header[
+            MESSAGE_OFFSET_LOCATION : MESSAGE_OFFSET_LOCATION + MESSAGE_OFFSET_LENGTH
+        ],
+        "little",
+        signed=False,
+    )
+    size = int.from_bytes(
+        header[MESSAGE_SIZE_LOCATION : MESSAGE_SIZE_LOCATION + MESSAGE_SIZE_LENGTH],
+        "little",
+        signed=False,
+    )
+    return id, offset, size
+
+
+def generate_chunks(message: bytes, max_size: int):
+    total_size = len(message)
+
+    if max_size == 0:
+        max_content_size = total_size
+    else:
+        max_content_size = max(max_size - HEADER_LENGTH, 1)
+
+    id = secrets.token_bytes(ID_LENGTH)
+
+    offset = 0
+
+    while offset < total_size:
+        header = _encode_header(id, offset, total_size)
+        chunk_content = message[offset : offset + max_content_size]
+
+        yield header + chunk_content
+
+        offset += max_content_size
+
+    return
+
+
+class PendingMessage(TypedDict):
+    received_size: int
+    content: bytearray
+
+
+# This un-chunker is vulnerable to DOS.
+# If it receives a message with a header claiming a large incoming message
+# it will allocate the memory blindly even without actually receiving the content
+# Chunks for a given message can come in any order
+# Chunks across messages can be interleaved.
+class UnChunker:
+    pending_messages: Dict[bytes, PendingMessage]
+    max_message_size: int
+
+    def __init__(self):
+        self.pending_messages = {}
+        self.max_message_size = 512
+
+    def set_max_message_size(self, size):
+        self.max_message_size = size
+
+    def release_pending_messages(self):
+        self.pending_messages = {}
+
+    def process_chunk(self, chunk: bytes) -> Union[bytes, None]:
+        header, chunk_content = chunk[:HEADER_LENGTH], chunk[HEADER_LENGTH:]
+        id, offset, total_size = _decode_header(header)
+
+        pending_message = self.pending_messages.get(id, None)
+
+        if pending_message is None:
+            if total_size > self.max_message_size:
+                raise ValueError(
+                    f"""Total size for message {id} exceeds the allocation limit allowed.
+    Maximum size = {self.max_message_size},
+    Received size = {total_size}."""
+                )
+
+            pending_message = PendingMessage(
+                received_size=0, content=bytearray(total_size)
+            )
+            self.pending_messages[id] = pending_message
+
+        # This should never happen, but still check it
+        if total_size != len(pending_message["content"]):
+            del self.pending_messages[id]
+            raise ValueError(
+                f"Total size in chunk header for message {id} does not match total size declared by previous chunk."
+            )
+
+        content_size = len(chunk_content)
+        content_view = memoryview(pending_message["content"])
+        content_view[offset : offset + content_size] = chunk_content
+        pending_message["received_size"] += content_size
+
+        if pending_message["received_size"] >= total_size:
+            full_message = pending_message["content"]
+            del self.pending_messages[id]
+            return msgpack.unpackb(bytes(full_message))
+
+        return None
+
+
+class StreamPendingMessage(TypedDict):
+    received_size: int
+    total_size: int
+    unpacker: msgpack.Unpacker
+
+
+# This un-chunker is more memory efficient
+# (each chunk is passed immediately to msgpack)
+# and it will only allocate memory when it receives content.
+# Chunks for a given message are expected to come sequentially
+# Chunks across messages can be interleaved.
+class StreamUnChunker:
+    pending_messages: Dict[bytes, StreamPendingMessage]
+
+    def __init__(self):
+        self.pending_messages = {}
+
+    def set_max_message_size(self, _size):
+        pass
+
+    def release_pending_messages(self):
+        self.pending_messages = {}
+
+    def process_chunk(self, chunk: bytes) -> Union[bytes, None]:
+        header, chunk_content = chunk[:HEADER_LENGTH], chunk[HEADER_LENGTH:]
+        id, offset, total_size = _decode_header(header)
+
+        pending_message = self.pending_messages.get(id, None)
+
+        if pending_message is None:
+            pending_message = StreamPendingMessage(
+                received_size=0,
+                total_size=total_size,
+                unpacker=msgpack.Unpacker(max_buffer_size=total_size),
+            )
+            self.pending_messages[id] = pending_message
+
+        # This should never happen, but still check it
+        if offset != pending_message["received_size"]:
+            del self.pending_messages[id]
+            raise ValueError(
+                f"""Received an unexpected chunk for message {id}.
+    Expected offset = {pending_message['received_size']},
+    Received offset = {offset}."""
+            )
+
+        # This should never happen, but still check it
+        if total_size != pending_message["total_size"]:
+            del self.pending_messages[id]
+            raise ValueError(
+                f"""Received an unexpected total size in chunk header for message {id}.
+    Expected size = {pending_message['total_size']},
+    Received size = {total_size}."""
+            )
+
+        content_size = len(chunk_content)
+        pending_message["received_size"] += content_size
+
+        unpacker = pending_message["unpacker"]
+        unpacker.feed(chunk_content)
+
+        full_message = None
+
+        try:
+            full_message = unpacker.unpack()
+        except msgpack.OutOfData:
+            pass  # message is incomplete, keep ingesting chunks
+
+        if full_message is not None:
+            del self.pending_messages[id]
+
+            if pending_message["received_size"] < total_size:
+                # In principle feeding a stream to the unpacker could yield multiple outputs
+                # for example unpacker.feed(b'0123') would yield b'0', b'1', ect
+                # or concatenated packed payloads would yield two or more unpacked objects
+                # but in our use case we expect a full message to be mapped to a single object
+                raise ValueError(
+                    f"""Received a parsable payload shorter than expected for message {id}.
+    Expected size = {total_size},
+    Received size = {pending_message['received_size']}."""
+                )
+
+        return full_message

wslink/protocol.py

@@ -1,13 +1,14 @@
 import asyncio
 import copy
 import inspect
-import json
 import logging
-import re
+import msgpack
+import os
 import traceback
 
 from wslink import schedule_coroutine
 from wslink.publish import PublishManager
+from wslink.chunking import generate_chunks, UnChunker
 
 # from http://www.jsonrpc.org/specification, section 5.1
 METHOD_NOT_FOUND = -32601
@@ -17,6 +18,9 @@ RESULT_SERIALIZE_ERROR = -32002
 # used in client JS code:
 CLIENT_ERROR = -32099
 
+# 4MB is the default inside aiohttp
+MAX_MSG_SIZE = int(os.environ.get("WSLINK_MAX_MSG_SIZE", 4194304))
+
 logger = logging.getLogger(__name__)
 
 
@@ -147,6 +151,7 @@ class WslinkHandler(object):
         self.authentified_client_ids = set()
         self.attachment_atomic = asyncio.Lock()
         self.pub_manager = PublishManager()
+        self.unchunkers = {}
 
         # Build the rpc method dictionary, assuming we were given a serverprotocol
         if self.getServerProtocol():
@@ -183,6 +188,8 @@ class WslinkHandler(object):
         return "reverse_connection_client_id"
 
     async def onConnect(self, request, client_id):
+        self.unchunkers[client_id] = UnChunker()
+
         if not self.serverProtocol:
             return
         if hasattr(self.serverProtocol, "onConnect"):
@@ -192,6 +199,8 @@ class WslinkHandler(object):
                 linkProtocol.onConnect(request, client_id)
 
     async def onClose(self, client_id):
+        del self.unchunkers[client_id]
+
         if not self.serverProtocol:
             return
         if hasattr(self.serverProtocol, "onClose"):
@@ -212,9 +221,16 @@ class WslinkHandler(object):
                     and await self.validateToken(args[0]["secret"], client_id)
                 ):
                     self.authentified_client_ids.add(client_id)
+                    # Once a client is authenticated let the unchunker allocate memory unrestricted
+                    self.unchunkers[client_id].set_max_message_size(
+                        4 * 1024 * 1024 * 1024
+                    )  # 4GB
                     await self.sendWrappedMessage(
                         rpcid,
-                        {"clientID": "c{0}".format(client_id)},
+                        {
+                            "clientID": "c{0}".format(client_id),
+                            "maxMsgSize": MAX_MSG_SIZE,
+                        },
                         client_id=client_id,
                     )
                 else:
@@ -235,34 +251,16 @@ class WslinkHandler(object):
         return False
 
     async def onMessage(self, is_binary, msg, client_id):
-        payload = msg.data
-
-        if is_binary:
-            if self.isClientAuthenticated(client_id):
-                # assume all binary messages are attachments
-                try:
-                    key = self.attachmentsRecvQueue.pop(0)
-                    self.attachmentsReceived[key] = payload
-                except:
-                    pass
-                return
+        if not is_binary:
+            return
 
-        # handles issue https://bugs.python.org/issue10976
-        # `payload` is type bytes in Python 3. Unfortunately, json.loads
-        # doesn't support taking bytes until Python 3.6.
-        if type(payload) is bytes:
-            payload = payload.decode("utf-8")
+        full_message = self.unchunkers[client_id].process_chunk(msg.data)
+        if full_message is not None:
+            await self.onCompleteMessage(full_message, client_id)
 
-        rpc = json.loads(payload)
+    async def onCompleteMessage(self, rpc, client_id):
         logger.debug("wslink incoming msg %s", self.payloadWithSecretStripped(rpc))
         if "id" not in rpc:
-            # should be a binary attachment header
-            if rpc.get("method") == "wslink.binary.attachment":
-                keys = rpc.get("args", [])
-                if isinstance(keys, list):
-                    for k in keys:
-                        # wait for an attachment by it's order
-                        self.attachmentsRecvQueue.append(k)
             return
 
         # TODO validate
@@ -303,73 +301,37 @@ class WslinkHandler(object):
             return
 
         obj, func = self.functionMap[methodName]
+        args.insert(0, obj)
+
         try:
-            # get any attachments
-            def findAttachments(o):
-                if (
-                    isinstance(o, str)
-                    and re.match(r"^wslink_bin\d+$", o)
-                    and o in self.attachmentsReceived
-                ):
-                    attachment = self.attachmentsReceived[o]
-                    del self.attachmentsReceived[o]
-                    return attachment
-                elif isinstance(o, list):
-                    for i, v in enumerate(o):
-                        o[i] = findAttachments(v)
-                elif isinstance(o, dict):
-                    for k in o:
-                        o[k] = findAttachments(o[k])
-                return o
-
-            args = findAttachments(args)
-            kwargs = findAttachments(kwargs)
-
-            args.insert(0, obj)
-
-            try:
-                self.web_app.last_active_client_id = client_id
-                results = func(*args, **kwargs)
-                if inspect.isawaitable(results):
-                    results = await results
-
-                if self.connections[client_id].closed:
-                    # Connection was closed during RPC call.
-                    return
-
-                await self.sendWrappedMessage(
-                    rpcid, results, method=methodName, client_id=client_id
-                )
-            except Exception as e_inst:
-                captured_trace = traceback.format_exc()
-                logger.error("Exception raised")
-                logger.error(repr(e_inst))
-                logger.error(captured_trace)
-                await self.sendWrappedError(
-                    rpcid,
-                    EXCEPTION_ERROR,
-                    "Exception raised",
-                    {
-                        "method": methodName,
-                        "exception": repr(e_inst),
-                        "trace": captured_trace,
-                    },
-                    client_id=client_id,
-                )
+            self.web_app.last_active_client_id = client_id
+            results = func(*args, **kwargs)
+            if inspect.isawaitable(results):
+                results = await results
 
-        except Exception as e:
+            if self.connections[client_id].closed:
+                # Connection was closed during RPC call.
+                return
+
+            await self.sendWrappedMessage(
+                rpcid, results, method=methodName, client_id=client_id
+            )
+        except Exception as e_inst:
+            captured_trace = traceback.format_exc()
+            logger.error("Exception raised")
+            logger.error(repr(e_inst))
+            logger.error(captured_trace)
             await self.sendWrappedError(
                 rpcid,
                 EXCEPTION_ERROR,
                 "Exception raised",
                 {
                     "method": methodName,
-                    "exception": repr(e),
-                    "trace": traceback.format_exc(),
+                    "exception": repr(e_inst),
+                    "trace": captured_trace,
                 },
                 client_id=client_id,
             )
-            return
 
     def payloadWithSecretStripped(self, payload):
         payload = copy.deepcopy(payload)
@@ -428,9 +390,10 @@ class WslinkHandler(object):
             "id": rpcid,
             "result": content,
         }
+
         try:
-            encMsg = json.dumps(wrapper, ensure_ascii=False)
-        except TypeError as e:
+            packed_wrapper = msgpack.packb(wrapper)
+        except Exception:
             # the content which is not serializable might be arbitrarily large, don't include.
             # repr(content) would do that...
             await self.sendWrappedError(
@@ -444,47 +407,14 @@ class WslinkHandler(object):
 
         websockets = self.getAuthenticatedWebsockets(client_id, skip_last_active_client)
 
-        # Check if any attachments in the map go with this message
-        attachments = self.pub_manager.getAttachmentMap()
-        found_keys = []
-        if attachments:
-            for key in attachments:
-                # string match the encoded attachment key
-                if key in encMsg:
-                    if key not in found_keys:
-                        found_keys.append(key)
-                    # increment  for key
-                    self.pub_manager.registerAttachment(key)
-
-            for key in found_keys:
-                # send header
-                header = {
-                    "wslink": "1.0",
-                    "method": "wslink.binary.attachment",
-                    "args": [key],
-                }
-                json_header = json.dumps(header, ensure_ascii=False)
-
-                # aiohttp can not handle pending ws.send_bytes()
-                # tried with semaphore but got exception with >1
-                # https://github.com/aio-libs/aiohttp/issues/2934
-                async with self.attachment_atomic:
-                    for ws in websockets:
-                        if ws is not None:
-                            # Send binary header
-                            await ws.send_str(json_header)
-                            # Send binary message
-                            await ws.send_bytes(attachments[key])
-
-                # decrement for key
-                self.pub_manager.unregisterAttachment(key)
-
-        for ws in websockets:
-            if ws is not None:
-                await ws.send_str(encMsg)
-
-        loop = asyncio.get_event_loop()
-        loop.call_soon(self.pub_manager.freeAttachments, found_keys)
+        # aiohttp can not handle pending ws.send_bytes()
+        # tried with semaphore but got exception with >1
+        # https://github.com/aio-libs/aiohttp/issues/2934
+        async with self.attachment_atomic:
+            for chunk in generate_chunks(packed_wrapper, MAX_MSG_SIZE):
+                for ws in websockets:
+                    if ws is not None:
+                        await ws.send_bytes(chunk)
 
     async def sendWrappedError(self, rpcid, code, message, data=None, client_id=None):
         wrapper = {
@@ -497,15 +427,26 @@ class WslinkHandler(object):
         }
         if data:
             wrapper["error"]["data"] = data
-        encMsg = json.dumps(wrapper, ensure_ascii=False)
+
+        try:
+            packed_wrapper = msgpack.packb(wrapper)
+        except Exception:
+            del wrapper["error"]["data"]
+            packed_wrapper = msgpack.packb(wrapper)
+
         websockets = (
             [self.connections[client_id]]
             if client_id
             else [self.connections[c] for c in self.connections]
         )
-        for ws in websockets:
-            if ws is not None:
-                await ws.send_str(encMsg)
+        # aiohttp can not handle pending ws.send_bytes()
+        # tried with semaphore but got exception with >1
+        # https://github.com/aio-libs/aiohttp/issues/2934
+        async with self.attachment_atomic:
+            for chunk in generate_chunks(packed_wrapper, MAX_MSG_SIZE):
+                for ws in websockets:
+                    if ws is not None:
+                        await ws.send_bytes(chunk)
 
     def publish(self, topic, data, client_id=None, skip_last_active_client=False):
         client_list = [client_id] if client_id else [c_id for c_id in self.connections]

wslink/publish.py

@@ -7,9 +7,6 @@ from . import schedule_coroutine
 class PublishManager(object):
     def __init__(self):
         self.protocols = []
-        self.attachmentMap = {}
-        self.attachmentRefCounts = {}  # keyed same as attachment map
-        self.attachmentId = 0
         self.publishCount = 0
 
     def registerProtocol(self, protocol):
@@ -19,38 +16,12 @@ class PublishManager(object):
         if protocol in self.protocols:
             self.protocols.remove(protocol)
 
-    def getAttachmentMap(self):
-        return self.attachmentMap
-
-    def clearAttachmentMap(self):
-        self.attachmentMap.clear()
-
-    def registerAttachment(self, attachKey):
-        self.attachmentRefCounts[attachKey] += 1
-
-    def unregisterAttachment(self, attachKey):
-        self.attachmentRefCounts[attachKey] -= 1
-
-    def freeAttachments(self, keys=None):
-        keys_to_delete = []
-        keys_to_check = keys if keys is not None else [k for k in self.attachmentMap]
-
-        for key in keys_to_check:
-            if self.attachmentRefCounts.get(key) == 0:
-                keys_to_delete.append(key)
-
-        for key in keys_to_delete:
-            self.attachmentMap.pop(key)
-            self.attachmentRefCounts.pop(key)
-
     def addAttachment(self, payload):
-        # print("attachment", self, self.attachmentId)
-        # use a string flag in place of the binary attachment.
-        binaryId = "wslink_bin{0}".format(self.attachmentId)
-        self.attachmentMap[binaryId] = payload
-        self.attachmentRefCounts[binaryId] = 0
-        self.attachmentId += 1
-        return binaryId
+        """Deprecated method, keeping it to avoid breaking compatibility
+        Now that we use msgpack to pack/unpack messages,
+        We can have binary data directly in the object itself,
+        without needing to transfer it separately from the rest."""
+        return payload
 
     def publish(self, topic, data, client_id=None, skip_last_active_client=False):
         for protocol in self.protocols:

{wslink-1.12.4.dist-info → wslink-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wslink
-Version: 1.12.4
+Version: 2.0.0
 Summary: Python/JavaScript library for communicating over WebSocket
 Home-page: https://github.com/kitware/wslink
 Author: Kitware, Inc.
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: aiohttp <4
+Requires-Dist: msgpack <2,>=1
 Provides-Extra: ssl
 Requires-Dist: cryptography ; extra == 'ssl'
 

{wslink-1.12.4.dist-info → wslink-2.0.0.dist-info}/RECORD

@@ -1,8 +1,9 @@
 wslink/LICENSE,sha256=I44UH7kDVqxDLnnlOWw_hFL2Fz7RjQ_4vPzZv9NYgTU,1483
 wslink/__init__.py,sha256=AbEm-sUSoGL-uLpnbK1rSSjHSvyW-bMsGHWie7FgMHw,2708
+wslink/chunking.py,sha256=BZZ0YAlh6PNI8rQe80NfdxU8pAvn_Klxew47AkvUJow,7130
 wslink/launcher.py,sha256=8VMs3juObLkyGYQFNLjMoo4qFpKIcxWz0kS-af-DKO4,21170
-wslink/protocol.py,sha256=lI_xPZl3DsEbqdeyHzvHIvdfK-AlpiUqzc5vN8t6Ak8,18164
-wslink/publish.py,sha256=6xLr7tGkvbq5LbXbAvPQ6OplEBbs192WR-uhiTJMPdg,2354
+wslink/protocol.py,sha256=zdf4QthFHpAgEw3hTUyyaOuN76jzHeOJBpvekPbk7aY,15886
+wslink/publish.py,sha256=9G5TXqyGr-LCo_LwHYhzif6lhG2iXDvEBmEgwR8fh1M,1437
 wslink/relay.py,sha256=E8Lzu2Ay7KbOheN1-ArAZawo8lLqdDgJXOZSBuMknYs,86
 wslink/server.py,sha256=FKSJAKHDyfkNVM45-M-y1Zn8hh2TTYto1hTCIJx1pp8,9440
 wslink/ssl_context.py,sha256=hNOJJCdrStws1Qf6vPvY4vTk9Bf8J5d90W3fS0cRv8o,2290
@@ -18,7 +19,7 @@ wslink/backends/jupyter/__init__.py,sha256=Qu65gWsd2xCSsxybnDtEDI5vMjHN-F5jgPZOy
 wslink/backends/jupyter/core.py,sha256=H73IEEHyom3TsbhkyI5O88bFBbUIDzHVuvqbIF6PAIM,3858
 wslink/backends/tornado/__init__.py,sha256=Qu65gWsd2xCSsxybnDtEDI5vMjHN-F5jgPZOyNIxnGs,112
 wslink/backends/tornado/core.py,sha256=tPMkkhWuO_ovkisVim0zcegwZKEAG4IRUdd_O_0a_R0,2157
-wslink-1.12.4.dist-info/METADATA,sha256=nd9iR5_IF8-flORecGE1-Gwibs3no1fPwj_1qaLPkdE,3016
-wslink-1.12.4.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-wslink-1.12.4.dist-info/top_level.txt,sha256=N0d8eqvhwhfW1p1yPTmvxlbzhjz7ZyhBfysNvaFqpQY,7
-wslink-1.12.4.dist-info/RECORD,,
+wslink-2.0.0.dist-info/METADATA,sha256=XWnXD2kxZEtIuOy4Y-_8HZ8anq9hYFL7axDQhv96GoE,3045
+wslink-2.0.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+wslink-2.0.0.dist-info/top_level.txt,sha256=N0d8eqvhwhfW1p1yPTmvxlbzhjz7ZyhBfysNvaFqpQY,7
+wslink-2.0.0.dist-info/RECORD,,

{wslink-1.12.4.dist-info → wslink-2.0.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any