intersect-sdk 0.6.1a1__py3-none-any.whl

intersect_sdk/__init__.py

@@ -0,0 +1,64 @@
+"""The root module contains the intended public API for users of the INTERSECT-SDK.
+
+Users should not need to import anything outside of the root.
+
+In general, most breaking changes on version updates will relate to:
+  - Configuration classes (both adding and removing new config models). These configuration classes are relevant to the next point.
+  - When a new data service is integrated into INTERSECT, ALL adapters will need to update to support this data service, which will include new dependencies.
+"""
+
+from .app_lifecycle import default_intersect_lifecycle_loop
+from .capability.base import IntersectBaseCapabilityImplementation
+from .client import IntersectClient
+from .client_callback_definitions import (
+    INTERSECT_CLIENT_EVENT_CALLBACK_TYPE,
+    INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE,
+    INTERSECT_JSON_VALUE,
+    IntersectClientCallback,
+    IntersectClientMessageParams,
+)
+from .config.client import IntersectClientConfig
+from .config.service import IntersectServiceConfig
+from .config.shared import (
+    ControlPlaneConfig,
+    DataStoreConfig,
+    DataStoreConfigMap,
+    HierarchyConfig,
+)
+from .core_definitions import IntersectDataHandler, IntersectMimeType
+from .schema import get_schema_from_capability_implementation
+from .service import IntersectService
+from .service_definitions import (
+    IntersectEventDefinition,
+    intersect_event,
+    intersect_message,
+    intersect_status,
+)
+from .version import __version__, version_info
+
+__all__ = [
+    'IntersectDataHandler',
+    'IntersectEventDefinition',
+    'IntersectMimeType',
+    'intersect_event',
+    'intersect_message',
+    'intersect_status',
+    'get_schema_from_capability_implementation',
+    'IntersectService',
+    'IntersectClient',
+    'IntersectClientCallback',
+    'IntersectClientMessageParams',
+    'INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE',
+    'INTERSECT_CLIENT_EVENT_CALLBACK_TYPE',
+    'INTERSECT_JSON_VALUE',
+    'IntersectBaseCapabilityImplementation',
+    'default_intersect_lifecycle_loop',
+    'IntersectClientConfig',
+    'IntersectServiceConfig',
+    'HierarchyConfig',
+    'ControlPlaneConfig',
+    'DataStoreConfig',
+    'DataStoreConfigMap',
+    '__version__',
+    'version_info',
+]

intersect_sdk/_internal/__init__.py

@@ -0,0 +1,4 @@
+"""NOTE: It's not advised for users to import anything in this module. These are internal definitions essential for INTERSECT's functions.
+
+THERE IS NO GUARANTEE ANY API DEFINED HERE REMAINS STABLE.
+"""

intersect_sdk/_internal/compression.py.tmp

@@ -0,0 +1,38 @@
+"""Compression logic used inside INTERSECT.
+
+Our current compression algorithm is Brotli - see https://stackoverflow.com/a/59255343 for a fairly comprehensive overview of serialization formats compatible with generic JSON.
+Zstd is worth considering for instances where data isn't being persisted for long, as it is generally faster than Brotli (especially for decompression) - however, Zstd requires multithreading to maximize its potential.
+
+Currently, we will ALWAYS compress the following:
+  - The schemas sent as part of the lifecycle message
+  - The PAYLOADS of UserspaceMessages *if the data itself is being sent in UserspaceMessages* 
+
+We do NOT want to compress message headers, though.  
+
+MINIO should always handle compression itself, see https://min.io/docs/minio/linux/administration/object-management/data-compression.html for details.
+
+TODO - how should the data API handle compression?
+
+TODO - We should consider NOT compressing audio, video, image, or any data which is already compressed.
+"""
+
+import brotli
+
+
+def intersect_compress(message: bytes) -> bytes:
+    """Compress MESSAGE using the compression algorithm, and return compression result.
+    
+    Current compression algorithm is Brotli with the highest level quality.
+    """
+    return brotli.compress(message, quality=11) # type: ignore[no-any-return]
+
+def intersect_decompress(compressed: bytes) -> bytes:
+    """Decompress COMPRESSED using the decompression algorithm, and return decompressed message.
+    
+    Current decompression algorithm is Brotli.
+    """
+    try:
+        return brotli.decompress(compressed) # type: ignore[no-any-return]
+    except brotli.error:
+        # if the parameters weren't compressed with Brotli, just return the input
+        return compressed

intersect_sdk/_internal/constants.py

@@ -0,0 +1,10 @@
+BASE_RESPONSE_ATTR = '__is_intersect_response__'
+BASE_STATUS_ATTR = '__is_intersect_status__'
+BASE_EVENT_ATTR = '__is_intersect_event__'
+# in theory, as long as the next attributes are unique, they can be any string
+REQUEST_CONTENT = '__request_content_type__'
+RESPONSE_CONTENT = '__response_content_type__'
+RESPONSE_DATA = '__response_data_transfer_handler__'
+STRICT_VALIDATION = '__strict_validation__'
+SHUTDOWN_KEYS = '__ignore_message__'
+EVENT_ATTR_KEY = '__intersect_sdk_events__'

intersect_sdk/_internal/control_plane/brokers/amqp_client.py

@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import functools
+import threading
+import uuid
+from typing import TYPE_CHECKING, Callable
+
+import pika
+from retrying import retry
+
+from .broker_client import BrokerClient
+
+if TYPE_CHECKING:
+    from collections import defaultdict
+
+    from pika.channel import Channel
+    from pika.frame import Frame
+    from pika.spec import Basic, BasicProperties
+
+
+class AMQPClient(BrokerClient):
+    """Client for performing broker actions backed by a AMQP broker.
+
+    NOTE: Currently, thread safety has been attempted, but may not be guaranteed
+
+    Attributes:
+        id: A string representation of the client's UUID.
+        _connection_params: connection information to the AMQP broker (includes credentials)
+        _publish_connection: AMQP connection dedicated to publishing messages
+        _consume_connection: AMQP connection dedicated to consuming messages
+        _topics_to_handlers: Dictionary of string topic names to lists of
+            Callables to invoke for messages on that topic.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        username: str,
+        password: str,
+        topics_to_handlers: Callable[[], defaultdict[str, set[Callable[[bytes], None]]]],
+        uid: str | None = None,
+    ) -> None:
+        """The default constructor.
+
+        Args:
+            host: String for hostname of AMQP broker
+            port: port number of AMQP broker
+            username: username credentials for AMQP broker
+            password: password credentials for AMQP broker
+            topics_to_handlers: callback function which gets the topic to handler map from the channel manager
+            uid: String for the client's UUID.
+        """
+        self.uid = uid if uid else str(uuid.uuid4())
+
+        self._connection_params = pika.ConnectionParameters(
+            host=host,
+            port=port,
+            virtual_host='/',
+            credentials=pika.PlainCredentials(username, password),
+            blocked_connection_timeout=1,
+        )
+
+        # The pika connection to the broker
+        self._publish_connection: pika.adapters.BlockingConnection = None
+        self._consume_connection: pika.adapters.SelectConnection = None
+        self._consume_connection_ready_event = threading.Event()
+        self._consume_subscription_ready_event = threading.Event()
+
+        # Callback to the topics_to_handler list inside of
+        self._topics_to_handlers = topics_to_handlers
+        # mapping of topics to callables which can unsubscribe from the topic
+        self._topics_to_channel_cancel_callbacks: dict[str, Callable[[], None]] = {}
+        self._consumer_thread = None
+
+    @retry(
+        stop_max_attempt_number=5,
+        wait_exponential_multiplier=1000,
+        wait_exponential_max=60000,
+    )
+    def connect(self) -> None:
+        """Connect to the defined broker.
+
+        Try to connect to the broker, performing exponential backoff if connection fails.
+        """
+        # need deamon=True otherwise if tests fails it hangs trying to acquire lock
+        self.thread = threading.Thread(target=self._start_consuming, daemon=True)
+        self.thread.start()
+        self._publish_connection = pika.adapters.BlockingConnection(self._connection_params)
+        self._consume_connection_ready_event.wait(timeout=5)
+
+    def disconnect(self) -> None:
+        """Close all connections.
+
+        Close both the public and consume connections and stop the consuming thread.
+        """
+        self._publish_connection.close()
+        self._publish_connection = None
+        self._consume_connection.ioloop.add_callback_threadsafe(self._close_consume_connection)
+        # as soon as connection is closed, the ioloop.stop will be
+        # called which in turn will terminate the consuming thread
+        self.thread.join(5)
+        self._consume_connection = None
+
+    def is_connected(self) -> bool:
+        """Check if there is an active connection to the broker.
+
+        Returns:
+            A boolean. True if there is a connection, False if not.
+        """
+        # We are connected to the broker if either the publish or consume connections is open
+        return (self._publish_connection is not None and self._publish_connection.is_open) or (
+            self._consume_connection is not None and self._consume_connection.is_open
+        )
+
+    def publish(self, topic: str, payload: bytes) -> None:
+        """Publish the given message.
+
+        Publish payload with the pre-existing connection (via connect()) on topic.
+
+        Args:
+            topic: The topic on which to publish the message as a string
+            payload: The message to publish, as raw bytes.
+        """
+        channel = self._publish_connection.channel()
+        channel.exchange_declare(topic, exchange_type='fanout', durable=True)
+        # this will send the message to topic exchange and distribute to all
+        # queues that subscribed to it
+        channel.basic_publish(
+            exchange=topic,
+            routing_key=topic,
+            body=payload,
+            properties=pika.BasicProperties(content_type='text/plain'),
+        )
+        channel.close()
+
+    def subscribe(self, topic: str) -> None:
+        self._consume_subscription_ready_event.clear()
+        self._subscribe_to_queue(topic)
+        self._consume_subscription_ready_event.wait()
+
+    def unsubscribe(self, topic: str) -> None:
+        old_channel = self._topics_to_channel_cancel_callbacks.get(topic, None)
+        if old_channel:
+            old_channel()
+            del self._topics_to_channel_cancel_callbacks[topic]
+
+    def _start_consuming(self) -> None:
+        """Start consuming messages from broker.
+
+        Open the consuming connection and start its io loop.
+        """
+        self._consume_connection = pika.adapters.SelectConnection(
+            parameters=self._connection_params,
+            on_close_callback=(
+                lambda _connection, _exception: self._consume_connection.ioloop.stop()
+            ),
+            on_open_callback=lambda _connection: self._consume_connection_ready_event.set(),
+        )
+
+        self._consume_connection.ioloop.start()
+
+    def _subscribe_to_queue(self, topic: str) -> None:
+        """Start consuming from the given topic.
+
+        Declares the correct exchange and queue on the broker if needed, then starts
+        consuming messages on that queue.
+
+        Args:
+            topic: Name of the topic on the broker to subscribe to as a string.
+        """
+        # in consumer thread: open channel-> declare queue (need that if we starte
+        # consuming  before message is published (no queue yet)) -> start consuming
+        cb = functools.partial(self._open_channel, topic=topic)
+        self._consume_connection.ioloop.add_callback_threadsafe(cb)
+
+    def _open_channel(self, topic: str) -> None:
+        """Open a channel for the given topic.
+
+        Args:
+            topic: The topic to open a channel for as a string.
+        """
+        cb = functools.partial(self._on_channel_open, topic=topic)
+        self._consume_connection.channel(on_open_callback=cb)
+
+    def _on_channel_open(self, channel: Channel, topic: str) -> None:
+        """Create an exchange on the broker.
+
+        Used as a listener on channel open.
+
+        Args:
+            channel: The Channel being instantiated.
+            topic: The string name for the Channel on the broker.
+        """
+        cb = functools.partial(self._on_exchange_declareok, channel=channel, topic=topic)
+        channel.exchange_declare(exchange=topic, exchange_type='fanout', durable=True, callback=cb)
+
+    def _on_exchange_declareok(self, _unused_frame: Frame, channel: Channel, topic: str) -> None:
+        """Create a queue on the broker.
+
+        Used as a listener on exchange declaration.
+
+        Args:
+            _unused_frame: Object from pika. Ignored.
+            channel: The Channel being instantiated.
+            topic: The string name for the Channel on the broker.
+        """
+        cb = functools.partial(self._on_queue_declareok, channel=channel, topic=topic)
+        channel.queue_declare(queue=topic + '.' + self.uid, durable=True, callback=cb)
+
+    def _on_queue_declareok(self, _unused_frame: Frame, channel: Channel, topic: str) -> None:
+        """Begins listening on the given queue.
+
+        Used as a listener on queue declaration.
+
+        Args:
+            _unused_frame: Object from pika. Ignored.
+            channel: The Channel being instantiated.
+            topic: The string name for the Channel on the broker.
+        """
+        cb = functools.partial(self._on_queue_bindok, channel=channel, topic=topic)
+        channel.queue_bind(topic + '.' + self.uid, topic, routing_key=topic, callback=cb)
+
+    def _on_queue_bindok(self, _unused_frame: Frame, channel: Channel, topic: str) -> None:
+        """Consumes a message from the given channel.
+
+        Used as a listener on queue binding.
+
+        Args:
+            _unused_frame: Object from pika. Ignored.
+            channel: The Channel being instantiated.
+            topic: The string name for the Channel on the broker.
+        """
+        cb = functools.partial(self._on_consume_ok)
+        consumer_tag = channel.basic_consume(
+            queue=topic + '.' + self.uid,
+            auto_ack=True,
+            on_message_callback=self._consume_message,
+            callback=cb,
+        )
+        self._topics_to_channel_cancel_callbacks[topic] = lambda: channel.basic_cancel(consumer_tag)
+
+    def _on_consume_ok(self, _unused_frame: Frame) -> None:
+        """Sets the consume subscription ready even.
+
+        Used as a listener on consuming an initial message on a channel.
+
+        Args:
+            _unused_frame: Object from pika. Ignored.
+            topic: The string name for the Channel on the broker.
+        """
+        self._consume_subscription_ready_event.set()
+
+    def _consume_message(
+        self,
+        _unused_channel: Channel,
+        basic_deliver: Basic.Deliver,
+        _properties: BasicProperties,
+        body: bytes,
+    ) -> None:
+        """Handles incoming messages.
+
+        Looks up all handlers for the topic and delegates message handling to them.
+
+        Args:
+            _unused_channel: The Pika channel the message was received on. Ignored
+            basic_deliver: Contains internal Pika delivery information - i.e. the routing key.
+            _properties: Object from the Pika call. Ignored.
+            body: the pika message to be handled.
+        """
+        for handler in self._topics_to_handlers().get(basic_deliver.routing_key, []):
+            handler(body)
+
+    def _close_consume_connection(self) -> None:
+        """Closes the consume connection.
+
+        Used as a listener on the connection loop to safely shutdown.
+        """
+        self._consume_connection.close()
+        self._topics_to_channel_cancel_callbacks.clear()

intersect_sdk/_internal/control_plane/brokers/broker_client.py

@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+
+
+class BrokerClient(ABC):
+    """Abstract definition of a Broker Client.
+
+    Will abstractly manage interaction between implementation specific message broker
+    calls and higher level pub/sub features.
+    """
+
+    @abstractmethod
+    def connect(self) -> None:
+        """Connect to the defined broker. Credentials should be cached in the constructor."""
+        ...
+
+    @abstractmethod
+    def disconnect(self) -> None:
+        """Disconnect from the defined broker."""
+        ...
+
+    @abstractmethod
+    def is_connected(self) -> bool:
+        """Checks if there is an active connection to the broker.
+
+        Returns:
+            A boolean. True if there is a connection, False if not.
+        """
+        ...
+
+    @abstractmethod
+    def publish(self, topic: str, payload: bytes) -> None:
+        """Publishes the given message.
+
+        Publish payload with the pre-existing connection (via connect()) on topic.
+
+        Args:
+            topic: The topic on which to publish the message as a string.
+            payload: The message to publish, as raw bytes.
+        """
+        ...
+
+    @abstractmethod
+    def subscribe(self, topic: str) -> None:
+        """Subscribe to a topic over the pre-existing connection (via connect()).
+
+        Args:
+            topic: Topic to subscribe to.
+        """
+        ...
+
+    @abstractmethod
+    def unsubscribe(self, topic: str) -> None:
+        """Unsubscribe from a topic over the pre-existing connection (via connect()).
+
+        Args:
+            topic: Topic to unsubscribe from.
+        """
+        ...

intersect_sdk/_internal/control_plane/brokers/mqtt_client.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING, Any, Callable
+
+import paho.mqtt.client as paho_client
+from retrying import retry
+
+from .broker_client import BrokerClient
+
+if TYPE_CHECKING:
+    from collections import defaultdict
+
+
+class MQTTClient(BrokerClient):
+    """Client for performing broker actions backed by a MQTT broker.
+
+    Note that this class may not be thread safe, see https://github.com/eclipse/paho.mqtt.python/issues/358#issuecomment-1880819505
+
+    Attributes:
+        uid: String defining this client's unique ID in the broker
+        host: hostname of MQTT broker
+        port: port of MQTT broker
+        _connection: Paho Client used to interact with the broker
+        _connected: current state of whether or not we're connected to the broker (boolean)
+        _topics_to_handlers: Dictionary of string topic names to lists of
+            Callables to invoke for messages on that topic.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        username: str,
+        password: str,
+        topics_to_handlers: Callable[[], defaultdict[str, set[Callable[[bytes], None]]]],
+        uid: str | None = None,
+    ) -> None:
+        """The default constructor.
+
+        Args:
+            host: String for hostname of MQTT broker
+            port: port number of MQTT broker
+            username: username credentials for MQTT broker
+            password: password credentials for MQTT broker
+            topics_to_handlers: callback function which gets the topic to handler map from the channel manager
+            uid: A string representing the unique id to identify the client.
+        """
+        # Unique id for the MQTT broker to associate this client with
+        self.uid = uid if uid else str(uuid.uuid4())
+        self.host = host
+        self.port = port
+
+        # Create a client to connect to RabbitMQ
+        # TODO clean_session param is ONLY for MQTT v3 here
+        self._connection = paho_client.Client(client_id=self.uid, clean_session=False)
+        self._connection.username_pw_set(username=username, password=password)
+
+        # Whether the connection is currently active
+        self._connected = False
+        self._topics_to_handlers = topics_to_handlers
+
+        # MQTT callback functions
+        self._connection.on_connect = self._set_connection_status
+        self._connection.on_disconnect = self._handle_disconnect
+        self._connection.on_message = self._on_message
+
+    @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000, wait_exponential_max=60000)
+    def connect(self) -> None:
+        """Connect to the defined broker."""
+        # Create a client to connect to RabbitMQ
+        # TODO MQTT v5 implementations should set clean_start to NEVER here
+        self._connection.connect(self.host, self.port, 60)
+        self._connection.loop_start()
+
+    def disconnect(self) -> None:
+        """Disconnect from the broker."""
+        self._connection.disconnect()
+        self._connection.loop_stop()
+
+    def is_connected(self) -> bool:
+        """Check if there is an active connection to the broker.
+
+        Returns:
+            A boolean. True if there is a connection, False if not.
+        """
+        return self._connected
+
+    def publish(self, topic: str, payload: bytes) -> None:
+        """Publish the given message.
+
+        Publish payload with the pre-existing connection (via connect()) on topic.
+
+        Args:
+            topic: The topic on which to publish the message as a string.
+            payload: The message to publish, as raw bytes.
+        """
+        self._connection.publish(topic, payload, qos=2)
+
+    def subscribe(self, topic: str) -> None:
+        """Subscribe to a topic over the pre-existing connection (via connect()).
+
+        Args:
+            topic: Topic to subscribe to.
+        """
+        self._connection.subscribe(topic, qos=2)
+
+    def unsubscribe(self, topic: str) -> None:
+        """Unsubscribe from a topic over the pre-existing connection.
+
+        Args:
+          topic: Topic to unsubscribe from.
+        """
+        self._connection.unsubscribe(topic)
+
+    def _on_message(
+        self, _client: paho_client.Client, _userdata: Any, message: paho_client.MQTTMessage
+    ) -> None:
+        """Handle a message from the MQTT server.
+
+        Args:
+          _client: the Paho client
+          _userdata: MQTT user data
+          message: MQTT message
+        """
+        for handler in self._topics_to_handlers().get(message.topic, []):
+            handler(message.payload)
+
+    def _handle_disconnect(self, _client: paho_client.Client, _userdata: Any, _rc: int) -> None:
+        """Handle a disconnection from the MQTT server.
+
+        Args:
+            _client: The Paho client.
+            _userdata: MQTT user data.
+            rc: MQTT return code as an integer.
+        """
+        self._connected = False
+
+    def _set_connection_status(
+        self, _client: paho_client.Client, _userdata: Any, _flags: dict[str, Any], rc: int
+    ) -> None:
+        """Set the connection status in response to the result of a Paho connection attempt.
+
+        Args:
+            client: The Paho MQTT client.
+            userdata: The MQTT userdata.
+            flags: List of MQTT connection flags.
+            rc: The MQTT return code as an int.
+        """
+        # Return code 0 means connection was successful
+        if rc == 0:
+            self._connected = True
+        else:
+            self._connected = False