intersect-sdk 0.6.1a1__py3-none-any.whl

intersect_sdk/_internal/control_plane/control_plane_manager.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any, Callable, Literal
+
+from pydantic import TypeAdapter
+
+from ..exceptions import IntersectInvalidBrokerError
+from ..logger import logger
+from .brokers.mqtt_client import MQTTClient
+
+if TYPE_CHECKING:
+    from ...config.shared import ControlPlaneConfig
+    from .brokers.broker_client import BrokerClient
+
+GENERIC_MESSAGE_SERIALIZER = TypeAdapter(Any)
+
+
+def serialize_message(message: Any) -> bytes:
+    """Serialize a message to bytes, in preparation for publishing it on a message broker.
+
+    Works as a generic serializer/deserializer
+    """
+    return GENERIC_MESSAGE_SERIALIZER.dump_json(message, warnings=False)
+
+
+def create_control_provider(
+    config: ControlPlaneConfig,
+    topic_handler_callback: Callable[[], defaultdict[str, set[Callable[[bytes], None]]]],
+) -> BrokerClient:
+    if config.protocol == 'amqp0.9.1':
+        # only try to import the AMQP client if the user is using an AMQP broker
+        try:
+            from .brokers.amqp_client import AMQPClient
+
+            return AMQPClient(
+                host=config.host,
+                port=config.port or 5672,
+                username=config.username,
+                password=config.password,
+                topics_to_handlers=topic_handler_callback,
+            )
+        except ImportError as e:
+            msg = "Configuration includes AMQP broker, but AMQP dependencies were not installed. Install intersect with the 'amqp' optional dependency to use this backend. (i.e. `pip install intersect_sdk[amqp]`)"
+            raise IntersectInvalidBrokerError(msg) from e
+    # MQTT
+    return MQTTClient(
+        host=config.host,
+        port=config.port or 1883,
+        username=config.username,
+        password=config.password,
+        topics_to_handlers=topic_handler_callback,
+    )
+
+
+class ControlPlaneManager:
+    """The ControlPlaneManager class allows for working with multiple brokers from a single function call."""
+
+    def __init__(
+        self,
+        control_configs: list[ControlPlaneConfig] | Literal['discovery'],
+    ) -> None:
+        if control_configs == 'discovery':
+            msg = 'Discovery service not implemented yet'
+            raise ValueError(msg)
+        self._control_providers = [
+            create_control_provider(config, self.get_subscription_channels)
+            for config in control_configs
+        ]
+
+        self._ready = False
+        # topics_to_handlers are managed here and transcend connections/disconnections to the broker
+        self._topics_to_handlers: defaultdict[str, set[Callable[[bytes], None]]] = defaultdict(set)
+
+    def add_subscription_channel(
+        self, channel: str, callbacks: set[Callable[[bytes], None]]
+    ) -> None:
+        """Start listening for userspace messages on a channel on all configured brokers.
+
+        Note that ALL channels listened to will always handle userspace messages.
+
+        Params:
+          channel: string of the channel which we should start listening to
+        """
+        self._topics_to_handlers[channel] |= callbacks
+        if self._ready:
+            for provider in self._control_providers:
+                provider.subscribe(channel)
+
+    def remove_subscription_channel(self, channel: str) -> bool:
+        """Stop subscribing to a channel on all configured brokers.
+
+        Params:
+          channel: string of the channel which should no longer be listened to
+        Returns: True if channel is no longer being listened to, False if the channel wasn't being listened to in the first place
+        """
+        try:
+            del self._topics_to_handlers[channel]
+        except KeyError:
+            return False
+        else:
+            if self._ready:
+                for provider in self._control_providers:
+                    provider.unsubscribe(channel)
+            return True
+
+    def get_subscription_channels(self) -> defaultdict[str, set[Callable[[bytes], None]]]:
+        """Get the subscription channels.
+
+        Note that this function gets accessed as a callback from the direct broker implementations.
+
+        Returns:
+          the dictionary of topics to callback functions
+        """
+        return self._topics_to_handlers
+
+    def connect(self) -> None:
+        """Connect to all configured brokers and subscribe to any channels configured."""
+        # TODO - when implementing discovery service, discovery and connection logic should be applied here
+        for provider in self._control_providers:
+            provider.connect()
+            for channel in self._topics_to_handlers:
+                provider.subscribe(channel)
+        self._ready = True
+
+    def disconnect(self) -> None:
+        """Disconnect from all configured brokers."""
+        self._ready = False
+        for provider in self._control_providers:
+            provider.disconnect()
+
+    def publish_message(self, channel: str, msg: Any) -> None:
+        """Publish message on channel for all brokers."""
+        if self._ready:
+            serialized_message = serialize_message(msg)
+            for provider in self._control_providers:
+                provider.publish(channel, serialized_message)
+        else:
+            logger.error('Cannot send message, providers are not connected')
+
+    def is_connected(self) -> bool:
+        """Check that we are connected to ALL configured brokers.
+
+        Returns:
+          - True if we are currently connected to all brokers we've configured, False if not
+        """
+        return all(control_provider.is_connected() for control_provider in self._control_providers)

intersect_sdk/_internal/control_plane/discovery_service.py

@@ -0,0 +1,40 @@
+"""TODO: This is all old code we currently aren't using."""
+
+from __future__ import annotations
+
+import json
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
+
+
+def discover_broker(address: str, broker_endpoint: str) -> tuple[str, str, int]:
+    """Get the metadata for a broker from the discovery service.
+
+    Args:
+        address: A string containing the address for the discovery service.
+        broker_endpoint: specific API for broker
+    Returns:
+        Three strings. The first is the name of the broker type (as used in
+        _create_broker_client()), the second is the broker's address, and
+        the third is the broker's port number.
+    """
+    url = f'{address}/v0.1/{broker_endpoint}'
+
+    # Get scheme associated with the `url` string
+    scheme = urlparse(url).scheme
+
+    # Only accept `http` and `https` schemes, otherwise raise error
+    if scheme not in ('http', 'https'):
+        msg = f'URL scheme is {scheme}, only http or https schemes are accepted'
+        raise ValueError(msg)
+
+    request = Request(url)  # noqa: S310 (scheme checked earlier)
+    with urlopen(request) as response:  # noqa: S310 (scheme checked earlier)
+        body = response.read()
+
+    broker_info = json.loads(body.decode('utf-8'))
+    endpoint = broker_info['endpoint']
+    backend_name = broker_info['backendName']
+    address, port = endpoint.split(':', 1)
+
+    return backend_name, address, port

intersect_sdk/_internal/data_plane/data_plane_manager.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING
+
+from ...core_definitions import IntersectDataHandler, IntersectMimeType
+from ..exceptions import IntersectError
+from ..logger import logger
+from .minio_utils import MinioPayload, create_minio_store, get_minio_object, send_minio_object
+
+if TYPE_CHECKING:
+    from ...config.shared import DataStoreConfigMap, HierarchyConfig
+    from ..messages.event import EventMessage
+    from ..messages.userspace import UserspaceMessage
+
+
+class DataPlaneManager:
+    """The DataPlaneManager serves as a common interface to the data plane.
+
+    The API supports extensive plug-and-play for different data providers.
+    """
+
+    def __init__(self, hierarchy: HierarchyConfig, data_configs: DataStoreConfigMap) -> None:
+        """Inside the constructor, we verify that all data configuration credentials are correct.
+
+        Params:
+          hierarchy: Hierarchy configuration
+          data_configs: data configuration
+        """
+        self._hierarchy = hierarchy
+        self._minio_providers = list(map(create_minio_store, data_configs.minio))
+
+        # warn users about missing data plane
+        if not self._minio_providers:
+            logger.warn('WARNING: This service cannot support any MINIO instances')
+
+    def incoming_message_data_handler(self, message: UserspaceMessage | EventMessage) -> bytes:
+        """Get data from the request data provider.
+
+        Params:
+          message: the message sent externally to this location
+        Returns:
+          the actual data we want to submit to the user function
+        Raise:
+          IntersectException - if we couldn't get the data
+        """
+        request_data_handler = message['headers']['data_handler']
+        if request_data_handler == IntersectDataHandler.MESSAGE:
+            return message['payload']  # type: ignore[return-value]
+        if request_data_handler == IntersectDataHandler.MINIO:
+            # TODO - we may want to send additional provider information in the payload
+            payload: MinioPayload = message['payload']  # type: ignore[assignment]
+            provider = None
+            for store in self._minio_providers:
+                if store._base_url._url.geturl() == payload['minio_url']:  # noqa: SLF001 (only way to get URL from MINIO API)
+                    provider = store
+                    break
+            if not provider:
+                logger.error(
+                    f"You did not configure listening to MINIO instance '{payload['minio_url']}'. You must fix this to handle this data."
+                )
+                raise IntersectError
+            return get_minio_object(provider, payload)
+        logger.warning(f'Cannot parse data handler {request_data_handler}')
+        raise IntersectError
+
+    def outgoing_message_data_handler(
+        self,
+        function_response: bytes,
+        content_type: IntersectMimeType,
+        data_handler: IntersectDataHandler,
+    ) -> bytes | MinioPayload:
+        """Send the user's response to the appropriate data provider.
+
+        Params:
+          - function_response - the return value from the user's function
+          - content_type - content type of function_response
+          - data_handler - where we're going to send the data off to (i.e. the message, MINIO...)
+
+        Returns:
+          the payload of the message
+        Raise:
+          IntersectException - if there was any error in submitting the response
+        """
+        # TODO - instead of requiring users to specify the data handler themselves, another idea could be to use
+        # sys.getsizeof(function_response) and determine the data handler dynamically
+        # users could perhaps specify T1/T2/T3 data types but not the specific implementation
+        if data_handler == IntersectDataHandler.MESSAGE:
+            return function_response
+        if data_handler == IntersectDataHandler.MINIO:
+            if not self._minio_providers:
+                logger.error(
+                    'No MINIO provider configured, so you cannot set response_data_handler on @intersect_message to equal IntersectDataHandler.MINIO .'
+                )
+                raise IntersectError
+            provider = random.choice(self._minio_providers)  # noqa: S311 (TODO choose a MINIO provider better than at random - this may be determined from external message params)
+            return send_minio_object(function_response, provider, content_type, self._hierarchy)
+
+        logger.error(
+            f'No support implemented for code {data_handler}, please upgrade your intersect-sdk version.'
+        )
+        raise IntersectError

intersect_sdk/_internal/data_plane/minio_utils.py

@@ -0,0 +1,149 @@
+import mimetypes
+from hashlib import sha224
+from io import BytesIO
+from uuid import uuid4
+
+from minio import Minio
+from minio.error import MinioException
+from typing_extensions import TypedDict
+from urllib3.exceptions import MaxRetryError
+from urllib3.util import parse_url
+
+from ...config.shared import DataStoreConfig, HierarchyConfig
+from ...core_definitions import IntersectMimeType
+from ..exceptions import IntersectError
+from ..logger import logger
+from ..utils import die
+
+
+class MinioPayload(TypedDict):
+    """This is a payload which gets sent in the actual userspace message if the data handler is "MINIO"."""
+
+    minio_url: str
+    """
+    The complete URL of the MINIO instance. This is used for finding the correct MINIO instance when retrieving an object.
+    """
+    minio_bucket: str
+    """
+    The name of the bucket where the object is located. Each service should only create objects in one bucket.
+    """
+    minio_object_id: str
+    """
+    The name of the object. This is a random UUID.
+    """
+
+
+def _condense_minio_bucket_name(hierarchy: HierarchyConfig) -> str:
+    """Condense a hierarchy string into a string less than 64 characters.
+
+    This function is needed to handle MINIO bucket names. Collisions should be extremely rare,
+    and it should be fairly straightforward to identify a specific bucket for MINIO admins.
+
+    Bucket name = first characters (up to 6) of service name + hyphen + sha224 of full hierarchy string
+
+    TODO in the future, MINIO calls should be system-level only, so only the system + facility + organization
+    should need to be hashed.
+    Also, potentially come up with a better hashing algorithm (though sha224 is compressed enough, and gives us 56 characters).
+    """
+    return f'{hierarchy.service[:6]}-{sha224(hierarchy.hierarchy_string().encode()).hexdigest()}'
+
+
+def create_minio_store(config: DataStoreConfig) -> Minio:
+    config_uri = parse_url(config.host)
+    if not config_uri.host:
+        die(f'Minio configuration host {config.host} cannot be parsed as a valid hostname.')
+
+    client = Minio(
+        secure=config_uri.scheme == 'https',
+        access_key=config.username,
+        secret_key=config.password,
+        endpoint=config_uri.host if not config.port else f'{config_uri.host}:{config.port}',
+    )
+    try:
+        # it doesn't matter if the bucket exists, we're just checking the exception
+        # this is the fastest way in the public API to perform a HEAD request.
+        client.bucket_exists('qwop')
+    except MinioException:
+        die(f'Invalid credentials for Minio instance: {config}')
+    return client
+
+
+def send_minio_object(
+    data: bytes, provider: Minio, content_type: IntersectMimeType, hierarchy: HierarchyConfig
+) -> MinioPayload:
+    """Core function to save data in MINIO.
+
+    Params:
+      data: user response data, as bytes
+      provider: the Minio client
+      content_type: the content type of the body
+      hierarchy: the hierarchy configuration
+    Returns:
+      The MINIO payload which gets sent in the actual message.
+
+    Raises:
+      IntersectException - if any non-fatal MinIO error is caught
+    """
+    bucket_name = _condense_minio_bucket_name(hierarchy)
+    # mimetypes.guess_extension() is a nice-to-have for MINIO preview, but isn't essential.
+    object_id = str(uuid4()) + (mimetypes.guess_extension(content_type.value) or '')
+    try:
+        if not provider.bucket_exists(bucket_name):
+            provider.make_bucket(bucket_name)
+        buff_data = BytesIO(data)
+        provider.put_object(
+            bucket_name=bucket_name,
+            object_name=object_id,
+            data=buff_data,
+            length=buff_data.getbuffer().nbytes,
+            content_type=content_type.value,
+        )
+        return MinioPayload(
+            minio_url=provider._base_url._url.geturl(),  # noqa: SLF001 (only way to get URL from MINIO API)
+            minio_bucket=bucket_name,
+            minio_object_id=object_id,
+        )
+    except MaxRetryError as e:
+        logger.warning(
+            f'Non-fatal MinIO error when sending object, the server may be under stress but you should double-check your configuration. Details: \n{e}'
+        )
+        raise IntersectError from e
+    except MinioException as e:
+        logger.error(
+            f'Important MinIO error when sending object, this usually indicates a problem with your configuration. Details: \n{e}'
+        )
+        raise IntersectError from e
+
+
+def get_minio_object(provider: Minio, payload: MinioPayload) -> bytes:
+    """Core function to retrieve data from MINIO.
+
+    Params:
+      provider: a pre-cached MinIO provider from the data provider store
+      payload: the payload from the message (at this point, the minio_url should exist)
+
+    Returns:
+      user response data, as raw bytes
+    Raises:
+      IntersectException - if any non-fatal MinIO error is caught
+    """
+    try:
+        response = provider.get_object(
+            bucket_name=payload['minio_bucket'], object_name=payload['minio_object_id']
+        )
+        # TODO - objects should ONLY be removed if they are T1
+        provider.remove_object(
+            bucket_name=payload['minio_bucket'], object_name=payload['minio_object_id']
+        )
+    except MaxRetryError as e:
+        logger.warning(
+            f'Non-fatal MinIO error when retrieving object, the server may be under stress but you should double-check your configuration. Details: \n{e}'
+        )
+        raise IntersectError from e
+    except MinioException as e:
+        logger.error(
+            f'Important MinIO error when retrieving object, this usually indicates a problem with your configuration. Details: \n{e}'
+        )
+        raise IntersectError from e
+    else:
+        return response.data

intersect_sdk/_internal/event_metadata.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, NamedTuple
+
+if TYPE_CHECKING:
+    from pydantic import TypeAdapter
+
+    from ..core_definitions import IntersectDataHandler, IntersectMimeType
+    from ..service_definitions import IntersectEventDefinition
+
+
+class EventMetadata(NamedTuple):
+    """Internal cache of metadata associated with an event.
+
+    NOTE: both this class and all properties in it should remain immutable after creation
+    """
+
+    operations: set[str]
+    """
+    A hash set of operations which advertise this event
+    """
+    type: type
+    """
+    The actual type of the event
+    """
+    type_adapter: TypeAdapter[Any]
+    """
+    The type adapter used for deserializing and validating events
+    """
+    data_transfer_handler: IntersectDataHandler
+    """
+    The data transfer type
+    """
+    content_type: IntersectMimeType
+    """
+    The content type
+    """
+
+
+def definition_metadata_differences(
+    definition: IntersectEventDefinition, metadata: EventMetadata
+) -> list[tuple[str, str, str]]:
+    """Return a list of differences between 'definition' and 'metadata'.
+
+    First tuple value = defintion key
+    Second tuple value = second value
+    Third tuple value = already cached value
+    """
+    differences = []
+    if definition.event_type != metadata.type:
+        differences.append(('event_type', str(definition.event_type), str(metadata.type)))
+    if definition.content_type != metadata.content_type:
+        differences.append(
+            ('content_type', str(definition.content_type), str(metadata.content_type))
+        )
+    if definition.data_handler != metadata.data_transfer_handler:
+        differences.append(
+            ('data_handler', str(definition.data_handler), str(metadata.data_transfer_handler))
+        )
+    return differences

intersect_sdk/_internal/exceptions.py

@@ -0,0 +1,17 @@
+class IntersectError(Exception):
+    """Generic marker for INTERSECT-specific exceptions."""
+
+
+class IntersectApplicationError(IntersectError):
+    """This is a special IntersectException, thrown if user application logic throws ANY kind of Exception.
+
+    In general, validation should be expressed through JSON schema as much as possible; however, JSON schema is NOT a complete prescription for input validation.
+    When this exception is thrown, however, we do not leak any exception information in the error message. On the other hand, if the input fails
+    JSON schema validation, Pydantic will throw a specific ValidationError, and that exception information will deliberately be exposed in the error message.
+
+    This exception should strictly be used for control flow - it should NEVER be a fatal exception
+    """
+
+
+class IntersectInvalidBrokerError(IntersectError):
+    """Exception when invalid broker backend used."""

intersect_sdk/_internal/function_metadata.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, NamedTuple
+
+if TYPE_CHECKING:
+    from pydantic import TypeAdapter
+
+
+class FunctionMetadata(NamedTuple):
+    """Internal cache of public function metadata.
+
+    NOTE: both this class and all properties in it should remain immutable after creation
+    """
+
+    method: Callable[[Any], Any]
+    """
+    The raw method of the function. The function itself is useless and should not be called,
+    but will store user-defined attributes needed for internal handling of data.
+    """
+    request_adapter: TypeAdapter[Any] | None
+    """
+    Type adapter for serializing and validating requests. Should only be null if user did not specify a request parameter.
+    """
+    response_adapter: TypeAdapter[Any]
+    """
+    Type adapter for serializing and validating responses.
+    """

intersect_sdk/_internal/interfaces.py

@@ -0,0 +1,20 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class IntersectEventObserver(ABC):
+    """Abstract definition of an entity which observes an INTERSECT event (i.e. IntersectService).
+
+    Used as the common interface for event emitters (i.e. CapabilityImplementations).
+    """
+
+    @abstractmethod
+    def _on_observe_event(self, event_name: str, event_value: Any, operation: str) -> None:
+        """How to react to an event being fired.
+
+        Args:
+            event_name: The key of the event which is fired.
+            event_value: The value of the event which is fired.
+            operation: The source of the event (generally the function name, not directly invoked by application devs)
+        """
+        ...

intersect_sdk/_internal/logger.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger('intersect-sdk')