intersect-sdk 0.8.3__tar.gz → 0.9.0__tar.gz

{intersect_sdk-0.8.3 → intersect_sdk-0.9.0}/PKG-INFO

@@ -1,28 +1,27 @@
 Metadata-Version: 2.1
 Name: intersect-sdk
-Version: 0.8.3
+Version: 0.9.0
 Summary: Python SDK to interact with INTERSECT
 Keywords: intersect
 Author-Email: Lance Drane <dranelt@ornl.gov>, Marshall McDonnell <mcdonnellmt@ornl.gov>, Seth Hitefield <hitefieldsd@ornl.gov>, Andrew Ayres <ayresaf@ornl.gov>, Gregory Cage <cagege@ornl.gov>, Jesse McGaha <mcgahajr@ornl.gov>, Robert Smith <smithrw@ornl.gov>, Gavin Wiggins <wigginsg@ornl.gov>, Michael Brim <brimmj@ornl.gov>, Rick Archibald <archibaldrk@ornl.gov>, Addi Malviya Thakur <malviyaa@ornl.gov>
 License: BSD-3-Clause
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Project-URL: Homepage, https://github.com/INTERSECT-SDK/python-sdk/
 Project-URL: Changelog, https://github.com/INTERSECT-SDK/python-sdk/blob/main/CHANGELOG.md
 Project-URL: Documentation, https://intersect-python-sdk.readthedocs.io/en/latest/
 Project-URL: Issues, https://github.com/INTERSECT-SDK/python-sdk/issues
-Requires-Python: <4.0,>=3.8.10
+Requires-Python: <4.0,>=3.10
 Requires-Dist: pydantic>=2.7.0
 Requires-Dist: retrying<2.0.0,>=1.3.4
-Requires-Dist: paho-mqtt<2.0.0,>=1.6.1
+Requires-Dist: paho-mqtt<3.0.0,>=2.1.0
+Requires-Dist: pika<2.0.0,>=1.3.2
 Requires-Dist: minio>=7.2.3
 Requires-Dist: jsonschema[format-nongpl]>=4.21.1
-Requires-Dist: eval-type-backport>=0.1.3; python_version < "3.10"
-Provides-Extra: amqp
-Requires-Dist: pika<2.0.0,>=1.3.2; extra == "amqp"
+Requires-Dist: psutil>=7.0.0
 Provides-Extra: docs
 Requires-Dist: sphinx>=5.3.0; extra == "docs"
 Requires-Dist: furo>=2023.3.27; extra == "docs"
@@ -49,7 +48,8 @@ For a high-level overview, please see [the architecture website.](https://inters
 
 - Event-driven architecture
 - Support core interaction types: request/response, events, commands, statuses
-- Borrows several concepts from [AsyncAPI](https://www.asyncapi.com/docs/reference/specification/latest), and intends to support multiple different protocols. Currently, we support MQTT 3.1.1 and AMQP 0.9.1, but other protocols will be supported as well.
+- Borrows several concepts from [AsyncAPI](https://www.asyncapi.com/docs/reference/specification/latest), and intends to support multiple different protocols. Currently, we support MQTT 5.0 and AMQP 0.9.1, but other protocols will be supported as well.
+  - As a general rule, we will not support any protocols which do not support headers, do not allow for asynchronous messaging, or require the microservice itself to "keep alive" multiple connections.
 - Users automatically generate schema from code; schemas are part of the core contract of an INTERSECT microservice, and both external inputs and microservice outputs are required to uphold this contract.
 
 ## Authors

{intersect_sdk-0.8.3 → intersect_sdk-0.9.0}/README.md

@@ -19,7 +19,8 @@ For a high-level overview, please see [the architecture website.](https://inters
 
 - Event-driven architecture
 - Support core interaction types: request/response, events, commands, statuses
-- Borrows several concepts from [AsyncAPI](https://www.asyncapi.com/docs/reference/specification/latest), and intends to support multiple different protocols. Currently, we support MQTT 3.1.1 and AMQP 0.9.1, but other protocols will be supported as well.
+- Borrows several concepts from [AsyncAPI](https://www.asyncapi.com/docs/reference/specification/latest), and intends to support multiple different protocols. Currently, we support MQTT 5.0 and AMQP 0.9.1, but other protocols will be supported as well.
+  - As a general rule, we will not support any protocols which do not support headers, do not allow for asynchronous messaging, or require the microservice itself to "keep alive" multiple connections.
 - Users automatically generate schema from code; schemas are part of the core contract of an INTERSECT microservice, and both external inputs and microservice outputs are required to uphold this contract.
 
 ## Authors

{intersect_sdk-0.8.3 → intersect_sdk-0.9.0}/pyproject.toml

@@ -15,27 +15,28 @@ authors = [
     { name = "Addi Malviya Thakur", email = "malviyaa@ornl.gov" },
 ]
 readme = "README.md"
-requires-python = ">=3.8.10,<4.0"
+requires-python = ">=3.10,<4.0"
 keywords = [
     "intersect",
 ]
 dynamic = []
 classifiers = [
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
     "pydantic>=2.7.0",
     "retrying>=1.3.4,<2.0.0",
-    "paho-mqtt>=1.6.1,<2.0.0",
+    "paho-mqtt>=2.1.0,<3.0.0",
+    "pika>=1.3.2,<2.0.0",
     "minio>=7.2.3",
     "jsonschema[format-nongpl]>=4.21.1",
-    "eval-type-backport>=0.1.3;python_version<'3.10'",
+    "psutil>=7.0.0",
 ]
-version = "0.8.3"
+version = "0.9.0"
 
 [project.license]
 text = "BSD-3-Clause"
@@ -47,9 +48,6 @@ Documentation = "https://intersect-python-sdk.readthedocs.io/en/latest/"
 Issues = "https://github.com/INTERSECT-SDK/python-sdk/issues"
 
 [project.optional-dependencies]
-amqp = [
-    "pika>=1.3.2,<2.0.0",
-]
 docs = [
     "sphinx>=5.3.0",
     "furo>=2023.3.27",
@@ -95,7 +93,7 @@ extend-select = [
     "SLF",
     "SLOT",
     "SIM",
-    "TCH",
+    "TC",
     "ARG",
     "PTH",
     "PGH",
@@ -103,6 +101,7 @@ extend-select = [
     "TRY",
     "FLY",
     "RUF",
+    "INT",
 ]
 ignore = [
     "COM812",
@@ -130,12 +129,23 @@ max-complexity = 20
 [tool.ruff.lint.pylint]
 max-args = 10
 max-branches = 20
-max-returns = 10
+max-returns = 15
 max-statements = 75
 
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = [
+    "pydantic.BaseModel",
+    "intersect_sdk.IntersectBaseCapabilityImplementation",
+]
+runtime-evaluated-decorators = [
+    "pydantic.dataclasses.dataclass",
+    "pydantic.validate_call",
+]
+
 [tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = [
     "F401",
+    "TC004",
 ]
 "docs/*" = [
     "D",
@@ -148,7 +158,6 @@ max-statements = 75
     "D100",
     "D104",
     "TRY002",
-    "FA100",
 ]
 "tests/*" = [
     "S101",
@@ -158,7 +167,7 @@ max-statements = 75
     "ANN",
     "ARG",
     "D",
-    "FA100",
+    "RUF012",
 ]
 
 [tool.mypy]
@@ -196,9 +205,8 @@ exclude_also = [
 [tool.pdm.dev-dependencies]
 lint = [
     "pre-commit>=3.3.1",
-    "ruff==0.9.4",
+    "ruff==0.12.7",
     "mypy>=1.10.0",
-    "types-paho-mqtt>=1.6.0.20240106",
     "codespell>=2.3.0",
 ]
 test = [

intersect_sdk-0.9.0/src/intersect_sdk/__init__.py

@@ -0,0 +1,130 @@
+"""The root module contains the intended public API for users of the INTERSECT-SDK.
+
+Users should not need to import anything outside of the root.
+
+In general, most breaking changes on version updates will relate to:
+  - Configuration classes (both adding and removing new config models). These configuration classes are relevant to the next point.
+  - When a new data service is integrated into INTERSECT, ALL adapters will need to update to support this data service, which will include new dependencies.
+"""
+
+from importlib import import_module
+from typing import TYPE_CHECKING
+
+# import everything eagerly for IDEs/LSPs
+if TYPE_CHECKING:
+    from .app_lifecycle import default_intersect_lifecycle_loop
+    from .capability.base import IntersectBaseCapabilityImplementation
+    from .client import IntersectClient
+    from .client_callback_definitions import (
+        INTERSECT_CLIENT_EVENT_CALLBACK_TYPE,
+        INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE,
+        IntersectClientCallback,
+    )
+    from .config.client import IntersectClientConfig
+    from .config.service import IntersectServiceConfig
+    from .config.shared import (
+        ControlPlaneConfig,
+        ControlProvider,
+        DataStoreConfig,
+        DataStoreConfigMap,
+        HierarchyConfig,
+    )
+    from .core_definitions import IntersectDataHandler, IntersectMimeType
+    from .exceptions import IntersectCapabilityError
+    from .schema import get_schema_from_capability_implementations
+    from .service import IntersectService
+    from .service_callback_definitions import (
+        INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE,
+    )
+    from .service_definitions import (
+        IntersectEventDefinition,
+        intersect_message,
+        intersect_status,
+    )
+    from .shared_callback_definitions import (
+        INTERSECT_JSON_VALUE,
+        INTERSECT_RESPONSE_VALUE,
+        IntersectDirectMessageParams,
+        IntersectEventMessageParams,
+    )
+    from .version import __version__, version_info, version_string
+
+__all__ = (
+    'INTERSECT_CLIENT_EVENT_CALLBACK_TYPE',
+    'INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE',
+    'INTERSECT_JSON_VALUE',
+    'INTERSECT_RESPONSE_VALUE',
+    'INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE',
+    'ControlPlaneConfig',
+    'ControlProvider',
+    'DataStoreConfig',
+    'DataStoreConfigMap',
+    'HierarchyConfig',
+    'IntersectBaseCapabilityImplementation',
+    'IntersectCapabilityError',
+    'IntersectClient',
+    'IntersectClientCallback',
+    'IntersectClientConfig',
+    'IntersectDataHandler',
+    'IntersectDirectMessageParams',
+    'IntersectEventDefinition',
+    'IntersectEventMessageParams',
+    'IntersectMimeType',
+    'IntersectService',
+    'IntersectServiceConfig',
+    '__version__',
+    'default_intersect_lifecycle_loop',
+    'get_schema_from_capability_implementations',
+    'intersect_message',
+    'intersect_status',
+    'version_info',
+    'version_string',
+)
+
+# PEP 562 stuff: do lazy imports for people who just want to import from the top-level module
+
+__lazy_imports = {
+    'INTERSECT_CLIENT_EVENT_CALLBACK_TYPE': '.client_callback_definitions',
+    'INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE': '.client_callback_definitions',
+    'INTERSECT_JSON_VALUE': '.shared_callback_definitions',
+    'INTERSECT_RESPONSE_VALUE': '.shared_callback_definitions',
+    'INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE': '.service_callback_definitions',
+    'ControlPlaneConfig': '.config.shared',
+    'ControlProvider': '.config.shared',
+    'DataStoreConfig': '.config.shared',
+    'DataStoreConfigMap': '.config.shared',
+    'HierarchyConfig': '.config.shared',
+    'IntersectBaseCapabilityImplementation': '.capability.base',
+    'IntersectCapabilityError': '.exceptions',
+    'IntersectClient': '.client',
+    'IntersectClientCallback': '.client_callback_definitions',
+    'IntersectClientConfig': '.config.client',
+    'IntersectDataHandler': '.core_definitions',
+    'IntersectDirectMessageParams': '.shared_callback_definitions',
+    'IntersectEventDefinition': '.service_definitions',
+    'IntersectEventMessageParams': '.shared_callback_definitions',
+    'IntersectMimeType': '.core_definitions',
+    'IntersectService': '.service',
+    'IntersectServiceConfig': '.config.service',
+    '__version__': '.version',
+    'default_intersect_lifecycle_loop': '.app_lifecycle',
+    'get_schema_from_capability_implementations': '.schema',
+    'intersect_message': '.service_definitions',
+    'intersect_status': '.service_definitions',
+    'version_info': '.version',
+    'version_string': '.version',
+}
+
+
+def __getattr__(attr_name: str) -> object:
+    attr_module = __lazy_imports.get(attr_name)
+    if attr_module:
+        module = import_module(attr_module, package=__spec__.parent)
+        return getattr(module, attr_name)
+
+    msg = f'module {__name__!r} has no attribute {attr_name!r}'
+    raise AttributeError(msg)
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

{intersect_sdk-0.8.3 → intersect_sdk-0.9.0}/src/intersect_sdk/_internal/constants.py

@@ -1,10 +1,8 @@
 BASE_RESPONSE_ATTR = '__is_intersect_response__'
 BASE_STATUS_ATTR = '__is_intersect_status__'
-BASE_EVENT_ATTR = '__is_intersect_event__'
 # in theory, as long as the next attributes are unique, they can be any string
 REQUEST_CONTENT = '__request_content_type__'
 RESPONSE_CONTENT = '__response_content_type__'
 RESPONSE_DATA = '__response_data_transfer_handler__'
 STRICT_VALIDATION = '__strict_validation__'
 SHUTDOWN_KEYS = '__ignore_message__'
-EVENT_ATTR_KEY = '__intersect_sdk_events__'

{intersect_sdk-0.8.3 → intersect_sdk-0.9.0}/src/intersect_sdk/_internal/control_plane/brokers/amqp_client.py

@@ -12,7 +12,7 @@ from __future__ import annotations
 import functools
 import threading
 from hashlib import sha384
-from typing import TYPE_CHECKING, Callable
+from typing import TYPE_CHECKING
 
 import pika
 import pika.delivery_mode
@@ -24,10 +24,13 @@ from ...multi_flag_thread_event import MultiFlagThreadEvent
 from .broker_client import BrokerClient
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from pika.channel import Channel
     from pika.frame import Frame
     from pika.spec import Basic, BasicProperties
 
+    from ..definitions import MessageCallback
     from ..topic_handler import TopicHandler
 
 
@@ -191,7 +194,9 @@ class AMQPClient(BrokerClient):
     def considered_unrecoverable(self) -> bool:
         return self._unrecoverable
 
-    def publish(self, topic: str, payload: bytes, persist: bool) -> None:
+    def publish(
+        self, topic: str, payload: bytes, content_type: str, headers: dict[str, str], persist: bool
+    ) -> None:
         """Publish the given message.
 
         Publish payload with the pre-existing connection (via connect()) on topic.
@@ -199,6 +204,8 @@ class AMQPClient(BrokerClient):
         Args:
             topic: The topic on which to publish the message as a string
             payload: The message to publish, as raw bytes.
+            content_type: The content type of the message (if the data plane used is the control plane itself), or the value to be retrieved from the data plane (if the message handler is MINIO/etc.)
+            headers: UTF-8 dictionary which can help parse information about the message
             persist: True if message should persist until consumers available, False if message should be removed immediately.
         """
         topic = _hierarchy_2_amqp(topic)
@@ -210,11 +217,11 @@ class AMQPClient(BrokerClient):
                 routing_key=topic,
                 body=payload,
                 properties=pika.BasicProperties(
-                    content_type='text/plain',
+                    content_type=content_type,
+                    headers=headers,
                     delivery_mode=pika.delivery_mode.DeliveryMode.Persistent
                     if persist
                     else pika.delivery_mode.DeliveryMode.Transient,
-                    # expiration=None if persist else '8640000',
                 ),
             )
         else:
@@ -533,7 +540,7 @@ class AMQPClient(BrokerClient):
         self,
         channel: Channel,
         basic_deliver: Basic.Deliver,
-        _properties: BasicProperties,
+        properties: BasicProperties,
         body: bytes,
         persist: bool,
     ) -> None:
@@ -545,7 +552,7 @@ class AMQPClient(BrokerClient):
         Args:
             channel: The AMQP channel the message was received on. Used to manually acknowledge messages.
             basic_deliver: Contains internal AMQP delivery information - i.e. the routing key.
-            _properties: Object from the AMQP call. Ignored.
+            properties: Object from the AMQP call. Contains various metadata.
             body: the AMQP message to be handled.
             persist: Whether or not our queue should persist on either broker or application shutdown.
         """
@@ -557,6 +564,18 @@ class AMQPClient(BrokerClient):
             channel.basic_reject(basic_deliver.delivery_tag)
             return
 
+        # make sure that we have a content-type and headers, note that this does not publish a "reply" message if we fail here
+        content_type = properties.content_type
+        if not content_type:
+            logger.error('Missing message content type')
+            channel.basic_ack(basic_deliver.delivery_tag)
+            return
+        headers = properties.headers
+        if not headers:
+            logger.error('Missing message headers')
+            channel.basic_ack(basic_deliver.delivery_tag)
+            return
+
         tth_key = _amqp_2_hierarchy(basic_deliver.routing_key)
         topic_handler = self._topics_to_handlers().get(tth_key)
         if topic_handler:
@@ -571,7 +590,15 @@ class AMQPClient(BrokerClient):
                 consumer_tag_info.wait(1.0)
             thrd = threading.Thread(
                 target=self._consume_message_subthread,
-                args=(channel, topic_handler.callbacks, body, basic_deliver.delivery_tag, persist),
+                args=(
+                    channel,
+                    topic_handler.callbacks,
+                    body,
+                    content_type,
+                    headers,
+                    basic_deliver.delivery_tag,
+                    persist,
+                ),
             )
             self._consumer_tags_to_threads[consumer_tag_info.consumer_tag] = thrd
             thrd.start()
@@ -582,14 +609,16 @@ class AMQPClient(BrokerClient):
     def _consume_message_subthread(
         self,
         channel: Channel,
-        callbacks: set[Callable[[bytes], None]],
+        callbacks: set[MessageCallback],
         body: bytes,
+        content_type: str,
+        headers: dict[str, str],
         delivery_tag: int,
         persist: bool,
     ) -> None:
         """This is a subthread which executes the consumer code without blocking the IO loop. Without using a subthread, the AMQP heartbeat checker will be blocked."""
         for cb in callbacks:
-            cb(body)
+            cb(body, content_type, headers)
         # With persistent messages, we only acknowledge the message AFTER we are done processing
         # (this removes the message from the broker queue)
         # this allows us to retry a message if the broker OR this application goes down

{intersect_sdk-0.8.3 → intersect_sdk-0.9.0}/src/intersect_sdk/_internal/control_plane/brokers/broker_client.py

@@ -32,7 +32,9 @@ class BrokerClient(Protocol):
         """
         ...
 
-    def publish(self, topic: str, payload: bytes, persist: bool) -> None:
+    def publish(
+        self, topic: str, payload: bytes, content_type: str, headers: dict[str, str], persist: bool
+    ) -> None:
         """Publishes the given message.
 
         Publish payload with the pre-existing connection (via connect()) on topic.
@@ -40,6 +42,8 @@ class BrokerClient(Protocol):
         Args:
             topic: The topic on which to publish the message as a string.
             payload: The message to publish, as raw bytes.
+            content_type: The content type of the message (if the data plane used is the control plane itself), or the value to be retrieved from the data plane (if the message handler is MINIO/etc.)
+            headers: UTF-8 dictionary which can help parse information about the message
             persist:
                 True = message will persist forever in associated queues until consumers are available (usually used for Userspace messages)
                 False = remove message immediately if no consumers available (usually used for Event messages and Lifecycle messages)