intersect-sdk-common 0.9.2__tar.gz → 0.9.4__tar.gz

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: intersect-sdk-common
-Version: 0.9.2
+Version: 0.9.4
 Summary: Python SDK to interact with INTERSECT
 Keywords: intersect
 Author: Lance Drane, Marshall McDonnell, Seth Hitefield, Andrew Ayres, Gregory Cage, Jesse McGaha, Robert Smith, Gavin Wiggins, Michael Brim, Rick Archibald, Addi Malviya Thakur
@@ -48,5 +48,5 @@ The INTERSECT-SDK MUST follow semantic versioning, and MUST update its semantic
 ```bash
 uv venv .venv
 source .venv/bin/activate
-uv sync --locked --all-extras --all-groups
+uv sync --all-extras --all-groups
 ```

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/README.md

@@ -28,5 +28,5 @@ The INTERSECT-SDK MUST follow semantic versioning, and MUST update its semantic
 ```bash
 uv venv .venv
 source .venv/bin/activate
-uv sync --locked --all-extras --all-groups
+uv sync --all-extras --all-groups
 ```

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/pyproject.toml

@@ -18,7 +18,7 @@ readme = "README.md"
 license = { text = "BSD-3-Clause" }
 requires-python = ">=3.10,<4.0"
 keywords = ["intersect"]
-version = "0.9.2"
+version = "0.9.4"
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
@@ -34,13 +34,15 @@ dependencies = [
 ]
 
 [dependency-groups]
-lint = [
+dev = [
     "pre-commit>=3.3.1",
     "ruff==0.12.7",
     "mypy>=1.10.0",
     "codespell>=2.3.0",
+    "pytest>=7.3.2",
+    "pytest-cov>=4.1.0",
+    "httpretty>=1.1.4",
 ]
-test = ["pytest>=7.3.2", "pytest-cov>=4.1.0", "httpretty>=1.1.4"]
 
 [build-system]
 requires = ["uv_build>=0.10.2,<0.11.0"]

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/src/intersect_sdk_common/control_plane/brokers/amqp_client.py

@@ -23,14 +23,12 @@ from ...utils.multi_flag_thread_event import MultiFlagThreadEvent
 from .broker_client import BrokerClient
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
-
     from pika.channel import Channel
     from pika.frame import Frame
     from pika.spec import Basic, BasicProperties
 
+    from ..control_plane_manager import ControlPlaneManager
     from ..definitions import MessageCallback
-    from ..topic_handler import TopicHandler
 
 
 _AMQP_MAX_RETRIES = 10
@@ -95,8 +93,7 @@ class AMQPClient(BrokerClient):
         port: int,
         username: str,
         password: str,
-        topics_to_handlers: Callable[[], dict[str, TopicHandler]],
-        find_wildcard_handler: Callable[[str], TopicHandler | None],
+        control_plane_manager: ControlPlaneManager,
         is_root: bool,
     ) -> None:
         """The default constructor.
@@ -106,8 +103,7 @@ class AMQPClient(BrokerClient):
             port: port number of AMQP broker
             username: username credentials for AMQP broker
             password: password credentials for AMQP broker
-            topics_to_handlers: callback function which gets the topic to handler map from the channel manager
-            find_wildcard_handler: callback function which gets the wildcard topic handler from the channel manager
+            control_plane_manager: reference to the ControlPlaneManager instance, remember to ONLY use functions which do not mutate state
             is_root: Whether or not the client can configure exchanges and queues themselves (core services), or if this must be delegated to a Core Service (SDK Clients/Services)
         """
         self._connection_params = pika.ConnectionParameters(
@@ -132,11 +128,11 @@ class AMQPClient(BrokerClient):
         self._thread: threading.Thread | None = None
 
         # Callback to the topics_to_handler list inside of
-        self._topics_to_handlers = topics_to_handlers
-        self._find_wildcard_handler = find_wildcard_handler
+        self._control_plane_manager = control_plane_manager
 
         # mapping of topics to callables which can unsubscribe from the topic
         self._topics_to_consumer_tags: dict[str, _ConsumerTagInfo] = {}
+        """NOTE: separators for topics in this object is '.' , not '/' """
         self._consumer_tags_to_threads: dict[str, threading.Thread] = {}
 
         self._should_disconnect = False
@@ -227,17 +223,22 @@ class AMQPClient(BrokerClient):
         else:
             logger.error('Unable to publish message %s on topic %s', payload, topic)
 
-    def subscribe(self, topic: str, persist: bool) -> None:
+    def subscribe(self, topic: str, persist: bool, queue_name: str) -> None:
         """Subscribe to a topic.
 
         topic: system-of-system hierarchy. In AMQP parlance this gets translated to the routing key.
         persist: If True, we will create an idempotent queue name which should persist
           even on broker or application shutdown. If False, we will allow the server to create a unique
           queue name, and the queue will be destroyed once the associated channel is closed.
+        queue_name: The name of the queue to subscribe to.
         """
         topic = _hierarchy_2_amqp(topic)
         cb = functools.partial(
-            self._create_queue, channel=self._channel_in, topic=topic, persist=persist
+            self._create_queue,
+            channel=self._channel_in,
+            topic=topic,
+            persist=persist,
+            queue_name=queue_name,
         )
         self._connection.ioloop.add_callback_threadsafe(cb)
 
@@ -390,7 +391,7 @@ class AMQPClient(BrokerClient):
     def _on_input_channel_open(self, channel: Channel) -> None:
         channel_num = 1
         self._channel_in = channel
-        # consumer channel flag can be set immediately
+        # consumer channel flag can be set immediately (NOTE: this means that code which simultaneously creates Clients and Services will need to manually wait, to ensure that each application has made the basic_consume call to the broker before messages are published)
         self._channel_flags.set_nth_flag(channel_num)
         cb_1 = functools.partial(self._on_channel_closed, channel_num=channel_num)
         self._channel_in.add_on_close_callback(cb_1)
@@ -403,17 +404,7 @@ class AMQPClient(BrokerClient):
                 callback=cb_2,
             )
         else:
-            for topic, topic_handler in self._topics_to_handlers().items():
-                amqp_topic = _hierarchy_2_amqp(topic)
-                cb = functools.partial(
-                    self._setup_basic_qos,
-                    None,
-                    channel=channel,
-                    topic=amqp_topic,
-                    persist=topic_handler.topic_persist,
-                    queue_name=topic_handler.queue_name,
-                )
-                self._connection.ioloop.add_callback_threadsafe(cb)
+            self._create_all_queues(channel)
 
     def _on_exchange_declareok(self, _unused_frame: Frame, channel: Channel) -> None:
         """Create a queue on the broker (called from AMQP).
@@ -425,7 +416,17 @@ class AMQPClient(BrokerClient):
             _unused_frame: response from declaring the exchange on the broker (irrelevant).
             channel: The Channel being instantiated.
         """
-        for topic, topic_handler in self._topics_to_handlers().items():
+        self._create_all_queues(channel)
+
+    def _create_all_queues(self, channel: Channel) -> None:
+        """Create queues for all subscription channels on the broker.
+
+        This should only be called once we've verified that the exchange exists.
+
+        Args:
+            channel: The Channel being instantiated.
+        """
+        for topic, topic_handler in self._control_plane_manager.get_all_subscription_channels():
             amqp_topic = _hierarchy_2_amqp(topic)
             cb = functools.partial(
                 self._create_queue,
@@ -456,6 +457,7 @@ class AMQPClient(BrokerClient):
             queue=queue_name
             if persist
             else '',  # if we're transient, let the broker generate a name for us
+            passive=not self._is_root,
             durable=persist,
             exclusive=not persist,  # transient queues can be exclusive
             callback=cb,
@@ -597,17 +599,25 @@ class AMQPClient(BrokerClient):
             channel.basic_ack(basic_deliver.delivery_tag)
             return
 
-        tth_key = _amqp_2_hierarchy(basic_deliver.routing_key)
-        topic_handler = self._topics_to_handlers().get(tth_key)
+        amqp_routing_key_rep = basic_deliver.routing_key
+        manager_routing_key_rep = _amqp_2_hierarchy(basic_deliver.routing_key)
+        topic_handler = self._control_plane_manager.get_non_wildcard_subscription_channels().get(
+            manager_routing_key_rep
+        )
         if not topic_handler:
             # we may have included the topic in one of our wildcard handlers
-            topic_handler = self._find_wildcard_handler(tth_key)
+            wildcard_result = self._control_plane_manager.get_wildcard_topic_and_topic_handler(
+                manager_routing_key_rep
+            )
+            if wildcard_result:
+                manager_routing_key_rep, topic_handler = wildcard_result
+                amqp_routing_key_rep = _hierarchy_2_amqp(manager_routing_key_rep)
         if topic_handler:
-            consumer_tag_info = self._topics_to_consumer_tags.get(basic_deliver.routing_key)
+            consumer_tag_info = self._topics_to_consumer_tags.get(amqp_routing_key_rep)
             if not consumer_tag_info:
                 logger.error(
                     'Could not fetch consumer tag for topic %s, please inform an INTERSECT-SDK developer that you saw this message',
-                    tth_key,
+                    amqp_routing_key_rep,
                 )
                 return
             while not consumer_tag_info.consumer_tag_obtained():

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/src/intersect_sdk_common/control_plane/brokers/broker_client.py

@@ -55,7 +55,7 @@ class BrokerClient(Protocol):
         """
         ...
 
-    def subscribe(self, topic: str, persist: bool) -> None:
+    def subscribe(self, topic: str, persist: bool, queue_name: str) -> None:
         """Subscribe to a topic over the pre-existing connection (via connect()).
 
         This function should ALSO be called by reconnect handlers, and not just directly.
@@ -64,6 +64,7 @@ class BrokerClient(Protocol):
         Args:
             topic: Topic to subscribe to.
             persist: Whether or not the queue subscribed to is intended to be long-lived.
+            queue_name: The name of the queue to subscribe to. Used or ignored depending on the protocol.
         """
         ...
 

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/src/intersect_sdk_common/control_plane/brokers/mqtt_client.py

@@ -22,12 +22,10 @@ from ...logger import logger
 from .broker_client import BrokerClient
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
-
     from paho.mqtt.client import DisconnectFlags
     from paho.mqtt.reasoncodes import ReasonCode
 
-    from ..topic_handler import TopicHandler
+    from ..control_plane_manager import ControlPlaneManager
 
 
 _MQTT_MAX_RETRIES = 10
@@ -37,7 +35,7 @@ _MQTT_MAX_RETRIES = 10
 # this will be a breaking change, so only add it when ready to break
 def _hierarchy_2_mqtt(hierarchy: str) -> str:
     """Take the hierarchy string format saved in the Service and map it to the MQTT topic format. Currently just covers wildcards."""
-    return hierarchy.replace('#', '+')
+    return hierarchy.replace('*', '+')
 
 
 class MQTTClient(BrokerClient):
@@ -61,8 +59,7 @@ class MQTTClient(BrokerClient):
         port: int,
         username: str,
         password: str,
-        topics_to_handlers: Callable[[], dict[str, TopicHandler]],
-        find_wildcard_handler: Callable[[str], TopicHandler | None],
+        control_plane_manager: ControlPlaneManager,
         uid: str | None = None,
     ) -> None:
         """The default constructor.
@@ -72,8 +69,7 @@ class MQTTClient(BrokerClient):
             port: port number of MQTT broker
             username: username credentials for MQTT broker
             password: password credentials for MQTT broker
-            topics_to_handlers: callback function which gets the topic to handler map from the channel manager
-            find_wildcard_handler: callback function which gets the wildcard topic handler from the channel manager
+            control_plane_manager: reference to the ControlPlaneManager instance, remember to ONLY use functions which do not mutate state
             uid: A string representing the unique id to identify the client.
         """
         # Unique id for the MQTT broker to associate this client with
@@ -97,8 +93,7 @@ class MQTTClient(BrokerClient):
         self._connected_flag = threading.Event()
 
         # ConnectionManager callable state
-        self._topics_to_handlers = topics_to_handlers
-        self._find_wildcard_handler = find_wildcard_handler
+        self._control_plane_manager = control_plane_manager
 
         # MQTT v3.1.1 automatically downgrades a QOS which is too high (good), but MQTT v5 will terminate the connection (bad)
         # see https://github.com/rabbitmq/rabbitmq-server/discussions/11842
@@ -170,12 +165,13 @@ class MQTTClient(BrokerClient):
             resolved_topic, payload, qos=self._max_supported_qos if persist else 0, properties=props
         )
 
-    def subscribe(self, topic: str, persist: bool) -> None:
+    def subscribe(self, topic: str, persist: bool, queue_name: str) -> None:  # noqa: ARG002
         """Subscribe to a topic over the pre-existing connection (via connect()).
 
         Args:
             topic: Topic to subscribe to.
             persist: Determine if the associated message queue of the topic is long-lived (True) or not (False)
+            queue_name: The name of the queue to subscribe to. Ignored with current MQTT implementation.
         """
         resolved_topic = _hierarchy_2_mqtt(topic)
         # NOTE: RabbitMQ only works with QOS of 1 and 0, and seems to convert QOS2 to QOS1
@@ -204,10 +200,17 @@ class MQTTClient(BrokerClient):
           message: MQTT message
         """
         # NOTE: should not need to convert the MQTT topic to the protocol agnostic representation, as we only change the wildcard format (which won't show up as the message topic)
-        topic_handler = self._topics_to_handlers().get(message.topic)
+        inmem_topic = message.topic
+        topic_handler = self._control_plane_manager.get_non_wildcard_subscription_channels().get(
+            inmem_topic
+        )
         if not topic_handler:
             # we may have included the topic in one of our wildcard handlers
-            topic_handler = self._find_wildcard_handler(message.topic)
+            wildcard_result = self._control_plane_manager.get_wildcard_topic_and_topic_handler(
+                inmem_topic
+            )
+            if wildcard_result:
+                _, topic_handler = wildcard_result
         # Note that if we return prior to the callback, there will be no reply message
         if not topic_handler:
             logger.warning('Incompatible message topic %s, rejecting message', message.topic)
@@ -298,8 +301,8 @@ class MQTTClient(BrokerClient):
                 self._max_supported_qos = properties.MaximumQoS
 
             self._connected_flag.set()
-            for topic, topic_handler in self._topics_to_handlers().items():
-                self.subscribe(topic, topic_handler.topic_persist)
+            for topic, topic_handler in self._control_plane_manager.get_all_subscription_channels():
+                self.subscribe(topic, topic_handler.topic_persist, topic_handler.queue_name)
         else:
             # This will generally suggest a misconfiguration
             self._connected = False

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/src/intersect_sdk_common/control_plane/control_plane_manager.py

@@ -1,7 +1,7 @@
 """Wrapper around interacting with the control plane, which may be multiple brokers."""
 
+import itertools
 import re
-from collections.abc import Callable
 
 from ..config import ControlPlaneConfig
 from ..exceptions import IntersectSetupError
@@ -23,8 +23,7 @@ _CHANNEL_REGEX = re.compile(r'^[a-zA-Z0-9*/-]+[a-zA-Z0-9*#/-]$')
 
 def _create_control_provider(
     config: ControlPlaneConfig,
-    topic_handler_callback: Callable[[], dict[str, TopicHandler]],
-    find_wildcard_handler: Callable[[str], TopicHandler | None],
+    control_plane_manager: 'ControlPlaneManager',
 ) -> BrokerClient:
     if config.protocol == 'amqp0.9.1':
         from .brokers.amqp_client import (  # noqa: PLC0415 (lazy load all AMQP modules)
@@ -36,8 +35,7 @@ def _create_control_provider(
             port=config.port or 5672,
             username=config.username,
             password=config.password,
-            topics_to_handlers=topic_handler_callback,
-            find_wildcard_handler=find_wildcard_handler,
+            control_plane_manager=control_plane_manager,
             is_root=config.is_root,
         )
 
@@ -49,8 +47,7 @@ def _create_control_provider(
         port=config.port or 1883,
         username=config.username,
         password=config.password,
-        topics_to_handlers=topic_handler_callback,
-        find_wildcard_handler=find_wildcard_handler,
+        control_plane_manager=control_plane_manager,
     )
 
 
@@ -68,10 +65,7 @@ class ControlPlaneManager:
         queue_name_generator should be a hardcoded value for Core Services, the SDK should provide its own function to generate queue names.
         """
         self._control_providers = [
-            _create_control_provider(
-                config, self.get_subscription_channels, self.get_wildcard_topic_handler
-            )
-            for config in control_configs
+            _create_control_provider(config, self) for config in control_configs
         ]
 
         # flag which indicates if we SHOULD be connected.
@@ -121,7 +115,7 @@ class ControlPlaneManager:
             topic_handler.queue_name = queue_name
         if self.is_connected():
             for provider in self._control_providers:
-                provider.subscribe(channel, persist)
+                provider.subscribe(channel, persist, queue_name)
 
     def remove_subscription_channel(self, channel: str) -> bool:
         """Stop subscribing to a channel on all configured brokers.
@@ -142,29 +136,47 @@ class ControlPlaneManager:
                 provider.unsubscribe(channel)
         return True
 
-    def get_subscription_channels(self) -> dict[str, TopicHandler]:
-        """Get the subscription channels.
+    def get_non_wildcard_subscription_channels(self) -> dict[str, TopicHandler]:
+        """Get primary subscription channels.
 
-        These channels cannot be wildcards, and incoming topics must match the channel topics exactly. Note that this function gets accessed as a callback from the direct broker implementations.
+        These channels cannot be wildcards, and incoming topics must match the channel topics exactly.
+
+        This function is safe to call from the broker clients, as it does not mutate state.
 
         Returns:
           the dictionary of topics to topic information
         """
         return self._topics_to_handlers
 
-    def get_wildcard_topic_handler(self, topic: str) -> TopicHandler | None:
+    def get_wildcard_topic_and_topic_handler(self, topic: str) -> tuple[str, TopicHandler] | None:
         """Get the wildcard topic handler for a given topic, if it exists.
 
         This is an inefficient lookup, so should only be used if we fail to find a non-wildcard match for an incoming topic. Note that this function gets accessed as a callback from the direct broker implementations.
 
+        This function is safe to call from the broker clients, as it does not mutate state.
+
+        Params:
+          topic: the topic for an incoming message (this will not have any wildcards in it)
+
         Returns:
-          the topic handler associated with the topic, if it exists; None otherwise
+          if a match found: the wildcard string configured application-side to match the topic parameter, and the topic handler associated with the topic
+          if no match found: None
         """
-        for topic_handler in self._wildcards.values():
+        for wildcard, topic_handler in self._wildcards.items():
             if topic_handler.does_topic_match(topic):
-                return topic_handler
+                return wildcard, topic_handler
         return None
 
+    def get_all_subscription_channels(self) -> itertools.chain[tuple[str, TopicHandler]]:
+        """Get all subscription channels, including wildcard channels.
+
+        This function is safe to call from the broker clients, as it does not mutate state.
+
+        Returns:
+            an iterator which yields tuples of (channel, topic handler) for all channels, including wildcards. Note that wildcard channels will be returned in no particular order, and may be interspersed with non-wildcard channels.
+        """
+        return itertools.chain(self._topics_to_handlers.items(), self._wildcards.items())
+
     def connect(self) -> None:
         """Connect to all configured brokers.
 

intersect_sdk_common-0.9.4/src/intersect_sdk_common/control_plane/custom_binary_protocol.py

@@ -0,0 +1,77 @@
+"""This module contains the binary specification for an INTERSECT message IF the protocol does not provide any built-in header support. If it does, use the built-in functionality of the protocol.
+
+Some protocols do not have any built-in header support, leaving it up to us to define the binary structure of the message. We want to avoid "chunking" messages into multiple parts, so a message should be guaranteed to include all metadata.
+"""
+
+from intersect_sdk_common import IntersectApplicationError
+
+_HEADER_KV_SEPARATOR = b'\x02'
+"""Indicates end of a header key and start of a header value.
+
+Used because applications should generally assign no special meaning to this byte, and this byte has no reason to appear in header keys or values.
+"""
+_HEADER_VALUE_SEPARATOR = b'\x03'
+"""Indicates end of a header value and start of the next header key.
+
+Used because applications should generally assign no special meaning to this byte, and this byte has no reason to appear in header keys or values.
+"""
+_PAYLOAD_SEPARATOR = b'\x01'
+"""Indicates end of headers and start of the payload.
+
+Used because applications should generally assign no special meaning to this byte, and this byte has no reason to appear in header keys or values.
+"""
+
+_TOTAL_REASONABLE_HEADER_BYTES = 131072
+"""The total number of bytes that should be reasonably expected to be used for header keys, values, and header separators combined. This provides applications with some level of DOS protection.
+
+It is rare in practice for ANY application to use this number of bytes for the total amount of headers, for example http2_max_header_size in NGINX is rarely set above 128KB
+"""
+
+CONTENT_TYPE_HEADER_KEY = 'content_type'
+CONTENT_TYPE_HEADER_KEY_BYTES = CONTENT_TYPE_HEADER_KEY.encode()
+
+
+def create_binary_message(body: bytes, content_type: str, headers: dict[str, str]) -> bytes:
+    """Create a binary message from the body, headers, and content type."""
+    return b''.join(
+        [
+            # content-type 'header' first (this is generally handled separately from other headers in many protocols)
+            CONTENT_TYPE_HEADER_KEY_BYTES,
+            _HEADER_KV_SEPARATOR,
+            content_type.encode(),
+            _HEADER_VALUE_SEPARATOR if len(headers) else b'',
+            # headers
+            _HEADER_VALUE_SEPARATOR.join(
+                _HEADER_KV_SEPARATOR.join([key.encode(), value.encode()])
+                for key, value in headers.items()
+            ),
+            # end of headers
+            _PAYLOAD_SEPARATOR,
+            # body
+            body,
+        ]
+    )
+
+
+def parse_binary_message(message: bytes) -> tuple[bytes, str, dict[str, str]]:
+    """Parse a binary message into its body, content type, and headers."""
+    # IMPORTANT!!! ----- Total length of header keys and values combined should be limited to first several bytes, terminate header search early if headers aren't a reasonable length.
+    payload_sep_location = message.find(_PAYLOAD_SEPARATOR, 0, _TOTAL_REASONABLE_HEADER_BYTES)
+    if payload_sep_location == -1:
+        msg = 'Probable malformed message: no payload separator found in first expected bytes, .'
+        raise IntersectApplicationError(msg)
+    header_string = message[:payload_sep_location]
+    headers = {
+        key.decode(): value.decode()
+        for key, value in (
+            header.split(_HEADER_KV_SEPARATOR)
+            for header in header_string.split(_HEADER_VALUE_SEPARATOR)
+        )
+    }
+    try:
+        content_type = headers.pop(CONTENT_TYPE_HEADER_KEY)
+    except KeyError as e:
+        msg = 'Probable malformed message: no content_type header found in message, discarding it.'
+        raise IntersectApplicationError(msg) from e
+
+    return message[payload_sep_location + 1 :], content_type, headers

{intersect_sdk_common-0.9.2 → intersect_sdk_common-0.9.4}/src/intersect_sdk_common/control_plane/topic_handler.py

@@ -46,8 +46,8 @@ class TopicHandler:
                     # match any sequence of 0 or more words (make sure to allow for the separator)
                     regex_builder.append('[a-zA-Z0-9/-]*')
                 elif char == '/':
-                    # in-memory topics use '/' as the separator, but on the broker '.' is used
-                    regex_builder.append('\\.')
+                    # in-memory topics use '/' as the separator, it is the protocol handler's responsibility to convert topics to use '/' as the separator
+                    regex_builder.append('/')
                 else:
                     # ordinary character
                     regex_builder.append(char)