intersect-sdk 0.8.1__tar.gz → 0.8.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/PKG-INFO +5 -1
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/README.md +4 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/pyproject.toml +1 -1
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/amqp_client.py +150 -33
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/mqtt_client.py +4 -2
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/schema.py +2 -2
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/capability/base.py +2 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/version.py +1 -1
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/example_schema.json +3 -11
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_base_capability_implementation.py +13 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_schema_invalids.py +4 -4
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_schema_valid.py +46 -2
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/LICENSE +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/constants.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/broker_client.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/control_plane_manager.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/discovery_service.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/topic_handler.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/data_plane_manager.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/minio_utils.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/event_metadata.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/exceptions.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/function_metadata.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/interfaces.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/logger.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/event.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/lifecycle.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/userspace.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/multi_flag_thread_event.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/pydantic_schema_generator.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/stoppable_thread.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/utils.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/version.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/version_resolver.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/app_lifecycle.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/capability/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/client.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/client_callback_definitions.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/client.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/service.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/shared.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/constants.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/core_definitions.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/py.typed +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/schema.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service_callback_definitions.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service_definitions.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/shared_callback_definitions.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/conftest.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/e2e/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/e2e/test_examples.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/example_schema.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/return_type_mismatch.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/integration/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/integration/test_return_type_mismatch.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/integration/test_service.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/__init__.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_annotations.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_config.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_invalid_schema_runtime.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_lifecycle_message.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_userspace_message.py +0 -0
- {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_version_resolver.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: intersect-sdk
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.3
|
|
4
4
|
Summary: Python SDK to interact with INTERSECT
|
|
5
5
|
Keywords: intersect
|
|
6
6
|
Author-Email: Lance Drane <dranelt@ornl.gov>, Marshall McDonnell <mcdonnellmt@ornl.gov>, Seth Hitefield <hitefieldsd@ornl.gov>, Andrew Ayres <ayresaf@ornl.gov>, Gregory Cage <cagege@ornl.gov>, Jesse McGaha <mcgahajr@ornl.gov>, Robert Smith <smithrw@ornl.gov>, Gavin Wiggins <wigginsg@ornl.gov>, Michael Brim <brimmj@ornl.gov>, Rick Archibald <archibaldrk@ornl.gov>, Addi Malviya Thakur <malviyaa@ornl.gov>
|
|
@@ -29,6 +29,10 @@ Requires-Dist: furo>=2023.3.27; extra == "docs"
|
|
|
29
29
|
Description-Content-Type: text/markdown
|
|
30
30
|
|
|
31
31
|
[](https://doi.org/10.11578/dc.20240927.1)
|
|
32
|
+
[](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/ci.yml)
|
|
33
|
+
[](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/publish.yml)
|
|
34
|
+
[](https://badge.fury.io/py/intersect-sdk)
|
|
35
|
+
[](https://intersect-python-sdk.readthedocs.io/en/latest/)
|
|
32
36
|
|
|
33
37
|
# INTERSECT-SDK
|
|
34
38
|
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
[](https://doi.org/10.11578/dc.20240927.1)
|
|
2
|
+
[](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/ci.yml)
|
|
3
|
+
[](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/publish.yml)
|
|
4
|
+
[](https://badge.fury.io/py/intersect-sdk)
|
|
5
|
+
[](https://intersect-python-sdk.readthedocs.io/en/latest/)
|
|
2
6
|
|
|
3
7
|
# INTERSECT-SDK
|
|
4
8
|
|
|
@@ -33,11 +33,33 @@ if TYPE_CHECKING:
|
|
|
33
33
|
|
|
34
34
|
_AMQP_MAX_RETRIES = 10
|
|
35
35
|
|
|
36
|
+
_PREFETCH_COUNT = 1
|
|
37
|
+
"""This determines the maximum amount of messages a single consumer will handle. The consumer must acknowledge the message before it gets another message."""
|
|
38
|
+
|
|
36
39
|
# Note that we deliberately do NOT want this configurable at runtime. Any two INTERSECT services/clients could potentially exchange messages between one another.
|
|
37
40
|
_INTERSECT_MESSAGE_EXCHANGE = 'intersect-messages'
|
|
38
41
|
"""All INTERSECT messages get published to one exchange on the broker."""
|
|
39
42
|
|
|
40
43
|
|
|
44
|
+
class ConsumerTagInfo:
|
|
45
|
+
def __init__(self) -> None:
|
|
46
|
+
self.consumer_tag = ''
|
|
47
|
+
self.consumer_tag_event = threading.Event()
|
|
48
|
+
|
|
49
|
+
def obtain_consumer_tag(self, consumer_tag: str) -> None:
|
|
50
|
+
self.consumer_tag = consumer_tag
|
|
51
|
+
self.consumer_tag_event.set()
|
|
52
|
+
|
|
53
|
+
def consumer_tag_obtained(self) -> bool:
|
|
54
|
+
return self.consumer_tag_event.is_set()
|
|
55
|
+
|
|
56
|
+
def wait(self, time: float) -> None:
|
|
57
|
+
self.consumer_tag_event.wait(time)
|
|
58
|
+
|
|
59
|
+
def __repr__(self) -> str:
|
|
60
|
+
return f'{self.consumer_tag} -- OBTAINED: {self.consumer_tag_obtained()}'
|
|
61
|
+
|
|
62
|
+
|
|
41
63
|
def _get_queue_name(routing_key: str) -> str:
|
|
42
64
|
"""Generate a valid queue name from the routing key.
|
|
43
65
|
|
|
@@ -101,6 +123,10 @@ class AMQPClient(BrokerClient):
|
|
|
101
123
|
virtual_host='/',
|
|
102
124
|
credentials=pika.PlainCredentials(username, password),
|
|
103
125
|
connection_attempts=3,
|
|
126
|
+
# if not specified, this value is obtained by the broker. RabbitMQ sets it to 60s by default
|
|
127
|
+
heartbeat=10,
|
|
128
|
+
blocked_connection_timeout=5.0,
|
|
129
|
+
retry_delay=0.5,
|
|
104
130
|
)
|
|
105
131
|
|
|
106
132
|
# The pika connection to the broker
|
|
@@ -113,7 +139,8 @@ class AMQPClient(BrokerClient):
|
|
|
113
139
|
# Callback to the topics_to_handler list inside of
|
|
114
140
|
self._topics_to_handlers = topics_to_handlers
|
|
115
141
|
# mapping of topics to callables which can unsubscribe from the topic
|
|
116
|
-
self._topics_to_consumer_tags: dict[str,
|
|
142
|
+
self._topics_to_consumer_tags: dict[str, ConsumerTagInfo] = {}
|
|
143
|
+
self._consumer_tags_to_threads: dict[str, threading.Thread] = {}
|
|
117
144
|
|
|
118
145
|
self._should_disconnect = False
|
|
119
146
|
self._connection_retries = 0
|
|
@@ -129,7 +156,7 @@ class AMQPClient(BrokerClient):
|
|
|
129
156
|
self._should_disconnect = False
|
|
130
157
|
self._channel_flags.unset_all()
|
|
131
158
|
|
|
132
|
-
if not self._thread:
|
|
159
|
+
if not self._thread or not self._thread.is_alive():
|
|
133
160
|
self._thread = threading.Thread(target=self._init_connection, daemon=True)
|
|
134
161
|
self._thread.start()
|
|
135
162
|
|
|
@@ -139,8 +166,8 @@ class AMQPClient(BrokerClient):
|
|
|
139
166
|
def disconnect(self) -> None:
|
|
140
167
|
"""Close all connections."""
|
|
141
168
|
self._should_disconnect = True
|
|
142
|
-
for topic,
|
|
143
|
-
self._cancel_consumer_tag(topic,
|
|
169
|
+
for topic, tag_info in self._topics_to_consumer_tags.items():
|
|
170
|
+
self._cancel_consumer_tag(topic, tag_info.consumer_tag)
|
|
144
171
|
|
|
145
172
|
# since _should_disconnect was set, _connection.ioloop.stop() will now execute after explicit connection close
|
|
146
173
|
self._connection.close()
|
|
@@ -175,18 +202,23 @@ class AMQPClient(BrokerClient):
|
|
|
175
202
|
persist: True if message should persist until consumers available, False if message should be removed immediately.
|
|
176
203
|
"""
|
|
177
204
|
topic = _hierarchy_2_amqp(topic)
|
|
178
|
-
self.
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
205
|
+
while not self._channel_flags.is_set():
|
|
206
|
+
self._channel_flags.wait(1.0)
|
|
207
|
+
if self._connection and self._connection.is_open:
|
|
208
|
+
self._channel_out.basic_publish(
|
|
209
|
+
exchange=_INTERSECT_MESSAGE_EXCHANGE,
|
|
210
|
+
routing_key=topic,
|
|
211
|
+
body=payload,
|
|
212
|
+
properties=pika.BasicProperties(
|
|
213
|
+
content_type='text/plain',
|
|
214
|
+
delivery_mode=pika.delivery_mode.DeliveryMode.Persistent
|
|
215
|
+
if persist
|
|
216
|
+
else pika.delivery_mode.DeliveryMode.Transient,
|
|
217
|
+
# expiration=None if persist else '8640000',
|
|
218
|
+
),
|
|
219
|
+
)
|
|
220
|
+
else:
|
|
221
|
+
logger.error('Unable to publish message %s on topic %s', payload, topic)
|
|
190
222
|
|
|
191
223
|
def subscribe(self, topic: str, persist: bool) -> None:
|
|
192
224
|
"""Subscribe to a topic.
|
|
@@ -210,25 +242,41 @@ class AMQPClient(BrokerClient):
|
|
|
210
242
|
Therefore, transient queues will be cleaned up.
|
|
211
243
|
"""
|
|
212
244
|
amqp_topic = _hierarchy_2_amqp(topic)
|
|
213
|
-
|
|
214
|
-
if
|
|
215
|
-
self._cancel_consumer_tag(amqp_topic, consumer_tag)
|
|
245
|
+
consumer_tag_info = self._topics_to_consumer_tags.get(amqp_topic, None)
|
|
246
|
+
if consumer_tag_info:
|
|
247
|
+
self._cancel_consumer_tag(amqp_topic, consumer_tag_info.consumer_tag)
|
|
216
248
|
|
|
217
249
|
def _cancel_consumer_tag(self, topic: str, consumer_tag: str) -> None:
|
|
218
250
|
if self._channel_in and self._channel_in.is_open:
|
|
219
|
-
cb = functools.partial(
|
|
251
|
+
cb = functools.partial(
|
|
252
|
+
self._cancel_consumer_tag_cb, topic=topic, consumer_tag=consumer_tag
|
|
253
|
+
)
|
|
220
254
|
self._channel_in.basic_cancel(
|
|
221
255
|
consumer_tag,
|
|
222
256
|
callback=cb,
|
|
223
257
|
)
|
|
224
258
|
|
|
225
|
-
def _cancel_consumer_tag_cb(
|
|
259
|
+
def _cancel_consumer_tag_cb(
|
|
260
|
+
self, _frame: pika.frame.Frame, topic: str, consumer_tag: str
|
|
261
|
+
) -> None:
|
|
226
262
|
try:
|
|
227
263
|
del self._topics_to_consumer_tags[topic]
|
|
228
264
|
except KeyError:
|
|
229
265
|
# shouldn't happen because ControlPlaneManager gatekeeps consecutive remove_subscription_channel() calls
|
|
230
|
-
|
|
266
|
+
logger.error(
|
|
267
|
+
'Unable to clean up consumer tag related to topic %s , please inform an INTERSECT-SDK developer if you somehow see this message.',
|
|
268
|
+
topic,
|
|
269
|
+
)
|
|
231
270
|
logger.info('Unsubscribed from %s', topic)
|
|
271
|
+
try:
|
|
272
|
+
thread = self._consumer_tags_to_threads[consumer_tag]
|
|
273
|
+
# kill thread immediately if not recoverable, wait to send last message if we are shutting down gracefully
|
|
274
|
+
thread.join(0 if self.considered_unrecoverable() else None)
|
|
275
|
+
del self._consumer_tags_to_threads[consumer_tag]
|
|
276
|
+
logger.debug('Consumer cancelled')
|
|
277
|
+
except KeyError:
|
|
278
|
+
# This will commonly be encountered, as several topics will often share a single consumer
|
|
279
|
+
pass
|
|
232
280
|
|
|
233
281
|
# BEGIN CALLBACKS + THREADSAFE FUNCTIONS #
|
|
234
282
|
|
|
@@ -238,13 +286,18 @@ class AMQPClient(BrokerClient):
|
|
|
238
286
|
NOTE: ANY functions which are not eventually called from this function
|
|
239
287
|
should be called via self._connection.ioloop.add_callback_threadsafe(cb)
|
|
240
288
|
"""
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
289
|
+
self._connection = None
|
|
290
|
+
self._channel_in = None
|
|
291
|
+
self._channel_out = None
|
|
292
|
+
|
|
293
|
+
while not self._should_disconnect and not self.considered_unrecoverable():
|
|
294
|
+
if not self._connection or self._connection.is_closed:
|
|
295
|
+
self._connection = pika.adapters.SelectConnection(
|
|
296
|
+
parameters=self._connection_params,
|
|
297
|
+
on_close_callback=self._on_connection_closed,
|
|
298
|
+
on_open_error_callback=self._on_connection_open_error,
|
|
299
|
+
on_open_callback=self._on_connection_open,
|
|
300
|
+
)
|
|
248
301
|
|
|
249
302
|
# Loops forever until ioloop.stop is called WHEN self._should_disconnect is True
|
|
250
303
|
self._connection.ioloop.start()
|
|
@@ -415,6 +468,32 @@ class AMQPClient(BrokerClient):
|
|
|
415
468
|
topic: str,
|
|
416
469
|
queue_name: str,
|
|
417
470
|
persist: bool,
|
|
471
|
+
) -> None:
|
|
472
|
+
"""Sets up basic QOS on the current channel.
|
|
473
|
+
|
|
474
|
+
Args:
|
|
475
|
+
_unused_frame: AMQP response from binding to the queue. Ignored.
|
|
476
|
+
channel: The Channel being instantiated.
|
|
477
|
+
topic: Name of the topic on the broker.
|
|
478
|
+
queue_name: The name of the queue on the AMQP broker.
|
|
479
|
+
persist: Whether or not our queue should persist on either broker or application shutdown.
|
|
480
|
+
"""
|
|
481
|
+
cb = functools.partial(
|
|
482
|
+
self._on_basic_qos_set,
|
|
483
|
+
channel=channel,
|
|
484
|
+
topic=topic,
|
|
485
|
+
queue_name=queue_name,
|
|
486
|
+
persist=persist,
|
|
487
|
+
)
|
|
488
|
+
channel.basic_qos(prefetch_count=_PREFETCH_COUNT, callback=cb)
|
|
489
|
+
|
|
490
|
+
def _on_basic_qos_set(
|
|
491
|
+
self,
|
|
492
|
+
_unused_frame: Frame,
|
|
493
|
+
channel: Channel,
|
|
494
|
+
topic: str,
|
|
495
|
+
queue_name: str,
|
|
496
|
+
persist: bool,
|
|
418
497
|
) -> None:
|
|
419
498
|
"""Consumes a message from the given channel.
|
|
420
499
|
|
|
@@ -429,13 +508,15 @@ class AMQPClient(BrokerClient):
|
|
|
429
508
|
"""
|
|
430
509
|
cb = functools.partial(self._on_consume_ok, topic=topic)
|
|
431
510
|
message_cb = functools.partial(self._consume_message, persist=persist)
|
|
511
|
+
info = ConsumerTagInfo()
|
|
512
|
+
self._topics_to_consumer_tags[topic] = info
|
|
432
513
|
consumer_tag = channel.basic_consume(
|
|
433
514
|
queue=queue_name,
|
|
434
515
|
auto_ack=not persist, # persistent messages should be manually acked and we have no reason to NACK a message for now
|
|
435
516
|
on_message_callback=message_cb,
|
|
436
517
|
callback=cb,
|
|
437
518
|
)
|
|
438
|
-
|
|
519
|
+
info.obtain_consumer_tag(consumer_tag)
|
|
439
520
|
|
|
440
521
|
def _on_consume_ok(self, _unused_frame: Frame, topic: str) -> None:
|
|
441
522
|
"""Sets the consume subscription ready event.
|
|
@@ -468,14 +549,50 @@ class AMQPClient(BrokerClient):
|
|
|
468
549
|
body: the AMQP message to be handled.
|
|
469
550
|
persist: Whether or not our queue should persist on either broker or application shutdown.
|
|
470
551
|
"""
|
|
552
|
+
# if we got a message when we shouldn't, quickly try to requeue it
|
|
553
|
+
if self.considered_unrecoverable() or self._should_disconnect:
|
|
554
|
+
logger.warning(
|
|
555
|
+
"WARNING: A message for topic %s has been received when it shouldn't, attempting requeue"
|
|
556
|
+
)
|
|
557
|
+
channel.basic_reject(basic_deliver.delivery_tag)
|
|
558
|
+
return
|
|
559
|
+
|
|
471
560
|
tth_key = _amqp_2_hierarchy(basic_deliver.routing_key)
|
|
472
561
|
topic_handler = self._topics_to_handlers().get(tth_key)
|
|
473
562
|
if topic_handler:
|
|
474
|
-
|
|
475
|
-
|
|
563
|
+
consumer_tag_info = self._topics_to_consumer_tags.get(basic_deliver.routing_key)
|
|
564
|
+
if not consumer_tag_info:
|
|
565
|
+
logger.error(
|
|
566
|
+
'Could not fetch consumer tag for topic %s, please inform an INTERSECT-SDK developer that you saw this message',
|
|
567
|
+
tth_key,
|
|
568
|
+
)
|
|
569
|
+
return
|
|
570
|
+
while not consumer_tag_info.consumer_tag_obtained():
|
|
571
|
+
consumer_tag_info.wait(1.0)
|
|
572
|
+
thrd = threading.Thread(
|
|
573
|
+
target=self._consume_message_subthread,
|
|
574
|
+
args=(channel, topic_handler.callbacks, body, basic_deliver.delivery_tag, persist),
|
|
575
|
+
)
|
|
576
|
+
self._consumer_tags_to_threads[consumer_tag_info.consumer_tag] = thrd
|
|
577
|
+
thrd.start()
|
|
578
|
+
elif persist:
|
|
579
|
+
# we somehow got a message that we don't want, discard it
|
|
580
|
+
channel.basic_ack(basic_deliver.delivery_tag)
|
|
581
|
+
|
|
582
|
+
def _consume_message_subthread(
|
|
583
|
+
self,
|
|
584
|
+
channel: Channel,
|
|
585
|
+
callbacks: set[Callable[[bytes], None]],
|
|
586
|
+
body: bytes,
|
|
587
|
+
delivery_tag: int,
|
|
588
|
+
persist: bool,
|
|
589
|
+
) -> None:
|
|
590
|
+
"""This is a subthread which executes the consumer code without blocking the IO loop. Without using a subthread, the AMQP heartbeat checker will be blocked."""
|
|
591
|
+
for cb in callbacks:
|
|
592
|
+
cb(body)
|
|
476
593
|
# With persistent messages, we only acknowledge the message AFTER we are done processing
|
|
477
594
|
# (this removes the message from the broker queue)
|
|
478
595
|
# this allows us to retry a message if the broker OR this application goes down
|
|
479
596
|
# We currently never NACK or reject a message because in INTERSECT, applications currently never "share" a queue.
|
|
480
597
|
if persist:
|
|
481
|
-
channel.basic_ack(
|
|
598
|
+
channel.basic_ack(delivery_tag)
|
|
@@ -81,6 +81,7 @@ class MQTTClient(BrokerClient):
|
|
|
81
81
|
"""Connect to the defined broker."""
|
|
82
82
|
# Create a client to connect to RabbitMQ
|
|
83
83
|
# TODO MQTT v5 implementations should set clean_start to NEVER here
|
|
84
|
+
self._should_disconnect = False
|
|
84
85
|
self._connected_flag.clear()
|
|
85
86
|
self._connection.connect(self.host, self.port, 60)
|
|
86
87
|
self._connection.loop_start()
|
|
@@ -90,8 +91,9 @@ class MQTTClient(BrokerClient):
|
|
|
90
91
|
def disconnect(self) -> None:
|
|
91
92
|
"""Disconnect from the broker."""
|
|
92
93
|
self._should_disconnect = True
|
|
93
|
-
self._connection
|
|
94
|
-
|
|
94
|
+
if self._connection:
|
|
95
|
+
self._connection.disconnect()
|
|
96
|
+
self._connection.loop_stop()
|
|
95
97
|
|
|
96
98
|
def is_connected(self) -> bool:
|
|
97
99
|
"""Check if there is an active connection to the broker.
|
|
@@ -143,9 +143,9 @@ def _get_functions(
|
|
|
143
143
|
logger.warning(
|
|
144
144
|
f"Class '{capability.__name__}' has no function annotated with the @intersect_status() decorator. No status information will be provided when sending status messages."
|
|
145
145
|
)
|
|
146
|
-
if not intersect_messages and not intersect_events:
|
|
146
|
+
if not intersect_messages and not intersect_events and not intersect_status:
|
|
147
147
|
die(
|
|
148
|
-
f"No intersect annotations detected on class '{capability.__name__}'. Please annotate at least one entrypoint with '@intersect_message()', or one event-emitting function with '@intersect_event()' ."
|
|
148
|
+
f"No intersect annotations detected on class '{capability.__name__}'. Please annotate at least one entrypoint with '@intersect_message()', or one event-emitting function with '@intersect_event()', or create an '@intersect_status' function."
|
|
149
149
|
)
|
|
150
150
|
return intersect_status, intersect_messages, intersect_events
|
|
151
151
|
|
|
@@ -68,6 +68,8 @@ class IntersectBaseCapabilityImplementation:
|
|
|
68
68
|
is not IntersectBaseCapabilityImplementation.intersect_sdk_emit_event
|
|
69
69
|
or cls.intersect_sdk_call_service
|
|
70
70
|
is not IntersectBaseCapabilityImplementation.intersect_sdk_call_service
|
|
71
|
+
or cls.intersect_sdk_listen_for_service_event
|
|
72
|
+
is not IntersectBaseCapabilityImplementation.intersect_sdk_listen_for_service_event
|
|
71
73
|
):
|
|
72
74
|
msg = f"{cls.__name__}: Attempted to override a reserved INTERSECT-SDK function (don't start your function names with '_intersect_sdk_' or 'intersect_sdk_')"
|
|
73
75
|
raise RuntimeError(msg)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"asyncapi": "2.6.0",
|
|
3
|
-
"x-intersect-version": "0.8.
|
|
3
|
+
"x-intersect-version": "0.8.3",
|
|
4
4
|
"info": {
|
|
5
5
|
"title": "test.test.test.test.test",
|
|
6
6
|
"description": "INTERSECT schema",
|
|
@@ -982,11 +982,7 @@
|
|
|
982
982
|
"type": "string"
|
|
983
983
|
},
|
|
984
984
|
"data_handler": {
|
|
985
|
-
"
|
|
986
|
-
{
|
|
987
|
-
"$ref": "#/components/messageTraits/commonHeaders/userspaceHeaders/$defs/IntersectDataHandler"
|
|
988
|
-
}
|
|
989
|
-
],
|
|
985
|
+
"$ref": "#/components/messageTraits/commonHeaders/userspaceHeaders/$defs/IntersectDataHandler",
|
|
990
986
|
"default": 0,
|
|
991
987
|
"description": "Code signifying where data is stored."
|
|
992
988
|
},
|
|
@@ -1039,11 +1035,7 @@
|
|
|
1039
1035
|
"type": "string"
|
|
1040
1036
|
},
|
|
1041
1037
|
"data_handler": {
|
|
1042
|
-
"
|
|
1043
|
-
{
|
|
1044
|
-
"$ref": "#/components/messageTraits/commonHeaders/eventHeaders/$defs/IntersectDataHandler"
|
|
1045
|
-
}
|
|
1046
|
-
],
|
|
1038
|
+
"$ref": "#/components/messageTraits/commonHeaders/eventHeaders/$defs/IntersectDataHandler",
|
|
1047
1039
|
"default": 0,
|
|
1048
1040
|
"description": "Code signifying where data is stored."
|
|
1049
1041
|
},
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_base_capability_implementation.py
RENAMED
|
@@ -73,6 +73,19 @@ def test_no_override():
|
|
|
73
73
|
|
|
74
74
|
assert 'BadClass3: Attempted to override a reserved INTERSECT-SDK function' in str(ex)
|
|
75
75
|
|
|
76
|
+
with pytest.raises(RuntimeError) as ex:
|
|
77
|
+
|
|
78
|
+
class BadClass4(IntersectBaseCapabilityImplementation):
|
|
79
|
+
def intersect_sdk_listen_for_service_event(
|
|
80
|
+
self,
|
|
81
|
+
service,
|
|
82
|
+
event_name,
|
|
83
|
+
response_handler,
|
|
84
|
+
) -> None:
|
|
85
|
+
pass
|
|
86
|
+
|
|
87
|
+
assert 'BadClass4: Attempted to override a reserved INTERSECT-SDK function' in str(ex)
|
|
88
|
+
|
|
76
89
|
|
|
77
90
|
# Note that the ONLY thing the capability itself checks for are annotated functions.
|
|
78
91
|
# The event definitions and overall schema validation are a service-specific feature
|
|
@@ -197,7 +197,7 @@ def test_disallow_dynamic_list_subtyping(caplog: pytest.LogCaptureFixture):
|
|
|
197
197
|
with pytest.raises(SystemExit):
|
|
198
198
|
get_schema_helper([MockAnyList])
|
|
199
199
|
assert "parameter 'param' type annotation" in caplog.text
|
|
200
|
-
assert '
|
|
200
|
+
assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
|
|
201
201
|
|
|
202
202
|
|
|
203
203
|
def test_disallow_dynamic_list_subtyping_complex(caplog: pytest.LogCaptureFixture):
|
|
@@ -227,7 +227,7 @@ def test_disallow_dynamic_set_subtyping(caplog: pytest.LogCaptureFixture):
|
|
|
227
227
|
with pytest.raises(SystemExit):
|
|
228
228
|
get_schema_helper([MockAnySet])
|
|
229
229
|
assert "parameter 'param' type annotation" in caplog.text
|
|
230
|
-
assert '
|
|
230
|
+
assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
|
|
231
231
|
|
|
232
232
|
|
|
233
233
|
def test_disallow_dynamic_frozenset_subtyping(caplog: pytest.LogCaptureFixture):
|
|
@@ -241,7 +241,7 @@ def test_disallow_dynamic_frozenset_subtyping(caplog: pytest.LogCaptureFixture):
|
|
|
241
241
|
with pytest.raises(SystemExit):
|
|
242
242
|
get_schema_helper([MockAnyFrozenSet])
|
|
243
243
|
assert "parameter 'param' type annotation" in caplog.text
|
|
244
|
-
assert '
|
|
244
|
+
assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
|
|
245
245
|
|
|
246
246
|
|
|
247
247
|
def test_disallow_dynamic_generator_subtyping(caplog: pytest.LogCaptureFixture):
|
|
@@ -288,7 +288,7 @@ def test_disallow_dynamic_dict_value_type(caplog: pytest.LogCaptureFixture):
|
|
|
288
288
|
with pytest.raises(SystemExit):
|
|
289
289
|
get_schema_helper([MockAnyDictValue])
|
|
290
290
|
assert "parameter 'param' type annotation" in caplog.text
|
|
291
|
-
assert '
|
|
291
|
+
assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
|
|
292
292
|
|
|
293
293
|
|
|
294
294
|
def test_disallow_dynamic_tuple_subtyping(caplog: pytest.LogCaptureFixture):
|
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
import json
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
|
-
from intersect_sdk import
|
|
4
|
+
from intersect_sdk import (
|
|
5
|
+
IntersectBaseCapabilityImplementation,
|
|
6
|
+
IntersectDataHandler,
|
|
7
|
+
IntersectEventDefinition,
|
|
8
|
+
IntersectMimeType,
|
|
9
|
+
intersect_event,
|
|
10
|
+
intersect_message,
|
|
11
|
+
intersect_status,
|
|
12
|
+
)
|
|
5
13
|
from intersect_sdk._internal.constants import (
|
|
6
14
|
REQUEST_CONTENT,
|
|
7
15
|
RESPONSE_CONTENT,
|
|
@@ -22,7 +30,43 @@ def get_fixture_path(fixture: str):
|
|
|
22
30
|
return Path(__file__).absolute().parents[1] / 'fixtures' / fixture
|
|
23
31
|
|
|
24
32
|
|
|
25
|
-
# TESTS
|
|
33
|
+
# MINIMAL ANNOTATION TESTS ######################
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_minimal_intersect_annotations():
|
|
37
|
+
class CapWithMessage(IntersectBaseCapabilityImplementation):
|
|
38
|
+
intersect_sdk_capability_name = 'CapWithMessage'
|
|
39
|
+
|
|
40
|
+
@intersect_message
|
|
41
|
+
def message_function(self, theinput: int) -> int:
|
|
42
|
+
return theinput * 4
|
|
43
|
+
|
|
44
|
+
class CapWithEvent(IntersectBaseCapabilityImplementation):
|
|
45
|
+
intersect_sdk_capability_name = 'CapWithEvent'
|
|
46
|
+
|
|
47
|
+
@intersect_event(events={'event': IntersectEventDefinition(event_type=str)})
|
|
48
|
+
def event_function(self):
|
|
49
|
+
self.intersect_sdk_emit_event('event', 'emitted_value')
|
|
50
|
+
|
|
51
|
+
class CapWithStatus(IntersectBaseCapabilityImplementation):
|
|
52
|
+
intersect_sdk_capability_name = 'CapWithStatus'
|
|
53
|
+
|
|
54
|
+
@intersect_status
|
|
55
|
+
def status_function(self) -> str:
|
|
56
|
+
return 'Up'
|
|
57
|
+
|
|
58
|
+
schemas = get_schema_from_capability_implementations(
|
|
59
|
+
[
|
|
60
|
+
CapWithEvent,
|
|
61
|
+
CapWithMessage,
|
|
62
|
+
CapWithStatus,
|
|
63
|
+
],
|
|
64
|
+
FAKE_HIERARCHY_CONFIG,
|
|
65
|
+
)
|
|
66
|
+
assert len(schemas['capabilities']) == 3
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# FIXTURE TESTS ##################
|
|
26
70
|
|
|
27
71
|
|
|
28
72
|
def test_schema_comparison():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/minio_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/function_metadata.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/lifecycle.py
RENAMED
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/userspace.py
RENAMED
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/multi_flag_thread_event.py
RENAMED
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/pydantic_schema_generator.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/client_callback_definitions.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service_callback_definitions.py
RENAMED
|
File without changes
|
|
File without changes
|
{intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/shared_callback_definitions.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|