intersect-sdk 0.8.1__tar.gz → 0.8.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/PKG-INFO +5 -1
  2. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/README.md +4 -0
  3. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/pyproject.toml +1 -1
  4. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/amqp_client.py +150 -33
  5. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/mqtt_client.py +4 -2
  6. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/schema.py +2 -2
  7. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/capability/base.py +2 -0
  8. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/version.py +1 -1
  9. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/example_schema.json +3 -11
  10. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_base_capability_implementation.py +13 -0
  11. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_schema_invalids.py +4 -4
  12. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_schema_valid.py +46 -2
  13. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/LICENSE +0 -0
  14. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/__init__.py +0 -0
  15. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/__init__.py +0 -0
  16. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/constants.py +0 -0
  17. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/__init__.py +0 -0
  18. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/__init__.py +0 -0
  19. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/brokers/broker_client.py +0 -0
  20. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/control_plane_manager.py +0 -0
  21. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/discovery_service.py +0 -0
  22. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/control_plane/topic_handler.py +0 -0
  23. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/__init__.py +0 -0
  24. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/data_plane_manager.py +0 -0
  25. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/data_plane/minio_utils.py +0 -0
  26. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/event_metadata.py +0 -0
  27. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/exceptions.py +0 -0
  28. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/function_metadata.py +0 -0
  29. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/interfaces.py +0 -0
  30. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/logger.py +0 -0
  31. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/__init__.py +0 -0
  32. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/event.py +0 -0
  33. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/lifecycle.py +0 -0
  34. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/messages/userspace.py +0 -0
  35. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/multi_flag_thread_event.py +0 -0
  36. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/pydantic_schema_generator.py +0 -0
  37. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/stoppable_thread.py +0 -0
  38. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/utils.py +0 -0
  39. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/version.py +0 -0
  40. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/_internal/version_resolver.py +0 -0
  41. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/app_lifecycle.py +0 -0
  42. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/capability/__init__.py +0 -0
  43. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/client.py +0 -0
  44. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/client_callback_definitions.py +0 -0
  45. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/__init__.py +0 -0
  46. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/client.py +0 -0
  47. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/service.py +0 -0
  48. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/config/shared.py +0 -0
  49. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/constants.py +0 -0
  50. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/core_definitions.py +0 -0
  51. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/py.typed +0 -0
  52. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/schema.py +0 -0
  53. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service.py +0 -0
  54. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service_callback_definitions.py +0 -0
  55. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/service_definitions.py +0 -0
  56. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/src/intersect_sdk/shared_callback_definitions.py +0 -0
  57. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/__init__.py +0 -0
  58. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/conftest.py +0 -0
  59. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/e2e/__init__.py +0 -0
  60. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/e2e/test_examples.py +0 -0
  61. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/__init__.py +0 -0
  62. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/example_schema.py +0 -0
  63. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/fixtures/return_type_mismatch.py +0 -0
  64. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/integration/__init__.py +0 -0
  65. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/integration/test_return_type_mismatch.py +0 -0
  66. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/integration/test_service.py +0 -0
  67. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/__init__.py +0 -0
  68. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_annotations.py +0 -0
  69. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_config.py +0 -0
  70. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_invalid_schema_runtime.py +0 -0
  71. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_lifecycle_message.py +0 -0
  72. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_userspace_message.py +0 -0
  73. {intersect_sdk-0.8.1 → intersect_sdk-0.8.3}/tests/unit/test_version_resolver.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: intersect-sdk
3
- Version: 0.8.1
3
+ Version: 0.8.3
4
4
  Summary: Python SDK to interact with INTERSECT
5
5
  Keywords: intersect
6
6
  Author-Email: Lance Drane <dranelt@ornl.gov>, Marshall McDonnell <mcdonnellmt@ornl.gov>, Seth Hitefield <hitefieldsd@ornl.gov>, Andrew Ayres <ayresaf@ornl.gov>, Gregory Cage <cagege@ornl.gov>, Jesse McGaha <mcgahajr@ornl.gov>, Robert Smith <smithrw@ornl.gov>, Gavin Wiggins <wigginsg@ornl.gov>, Michael Brim <brimmj@ornl.gov>, Rick Archibald <archibaldrk@ornl.gov>, Addi Malviya Thakur <malviyaa@ornl.gov>
@@ -29,6 +29,10 @@ Requires-Dist: furo>=2023.3.27; extra == "docs"
29
29
  Description-Content-Type: text/markdown
30
30
 
31
31
  [![Static Badge](https://img.shields.io/badge/DOI-10.11578%2Fdc.20240927.1-blue)](https://doi.org/10.11578/dc.20240927.1)
32
+ [![CI](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/ci.yml/badge.svg)](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/ci.yml)
33
+ [![Release to PyPi](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/publish.yml/badge.svg)](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/publish.yml)
34
+ [![PyPI version](https://badge.fury.io/py/intersect-sdk.svg)](https://badge.fury.io/py/intersect-sdk)
35
+ [![ReadTheDocs](https://readthedocs.org/projects/intersect-python-sdk/badge/?version=latest)](https://intersect-python-sdk.readthedocs.io/en/latest/)
32
36
 
33
37
  # INTERSECT-SDK
34
38
 
@@ -1,4 +1,8 @@
1
1
  [![Static Badge](https://img.shields.io/badge/DOI-10.11578%2Fdc.20240927.1-blue)](https://doi.org/10.11578/dc.20240927.1)
2
+ [![CI](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/ci.yml/badge.svg)](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/ci.yml)
3
+ [![Release to PyPi](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/publish.yml/badge.svg)](https://github.com/INTERSECT-SDK/python-sdk/actions/workflows/publish.yml)
4
+ [![PyPI version](https://badge.fury.io/py/intersect-sdk.svg)](https://badge.fury.io/py/intersect-sdk)
5
+ [![ReadTheDocs](https://readthedocs.org/projects/intersect-python-sdk/badge/?version=latest)](https://intersect-python-sdk.readthedocs.io/en/latest/)
2
6
 
3
7
  # INTERSECT-SDK
4
8
 
@@ -35,7 +35,7 @@ dependencies = [
35
35
  "jsonschema[format-nongpl]>=4.21.1",
36
36
  "eval-type-backport>=0.1.3;python_version<'3.10'",
37
37
  ]
38
- version = "0.8.1"
38
+ version = "0.8.3"
39
39
 
40
40
  [project.license]
41
41
  text = "BSD-3-Clause"
@@ -33,11 +33,33 @@ if TYPE_CHECKING:
33
33
 
34
34
  _AMQP_MAX_RETRIES = 10
35
35
 
36
+ _PREFETCH_COUNT = 1
37
+ """This determines the maximum amount of messages a single consumer will handle. The consumer must acknowledge the message before it gets another message."""
38
+
36
39
  # Note that we deliberately do NOT want this configurable at runtime. Any two INTERSECT services/clients could potentially exchange messages between one another.
37
40
  _INTERSECT_MESSAGE_EXCHANGE = 'intersect-messages'
38
41
  """All INTERSECT messages get published to one exchange on the broker."""
39
42
 
40
43
 
44
+ class ConsumerTagInfo:
45
+ def __init__(self) -> None:
46
+ self.consumer_tag = ''
47
+ self.consumer_tag_event = threading.Event()
48
+
49
+ def obtain_consumer_tag(self, consumer_tag: str) -> None:
50
+ self.consumer_tag = consumer_tag
51
+ self.consumer_tag_event.set()
52
+
53
+ def consumer_tag_obtained(self) -> bool:
54
+ return self.consumer_tag_event.is_set()
55
+
56
+ def wait(self, time: float) -> None:
57
+ self.consumer_tag_event.wait(time)
58
+
59
+ def __repr__(self) -> str:
60
+ return f'{self.consumer_tag} -- OBTAINED: {self.consumer_tag_obtained()}'
61
+
62
+
41
63
  def _get_queue_name(routing_key: str) -> str:
42
64
  """Generate a valid queue name from the routing key.
43
65
 
@@ -101,6 +123,10 @@ class AMQPClient(BrokerClient):
101
123
  virtual_host='/',
102
124
  credentials=pika.PlainCredentials(username, password),
103
125
  connection_attempts=3,
126
+ # if not specified, this value is obtained by the broker. RabbitMQ sets it to 60s by default
127
+ heartbeat=10,
128
+ blocked_connection_timeout=5.0,
129
+ retry_delay=0.5,
104
130
  )
105
131
 
106
132
  # The pika connection to the broker
@@ -113,7 +139,8 @@ class AMQPClient(BrokerClient):
113
139
  # Callback to the topics_to_handler list inside of
114
140
  self._topics_to_handlers = topics_to_handlers
115
141
  # mapping of topics to callables which can unsubscribe from the topic
116
- self._topics_to_consumer_tags: dict[str, str] = {}
142
+ self._topics_to_consumer_tags: dict[str, ConsumerTagInfo] = {}
143
+ self._consumer_tags_to_threads: dict[str, threading.Thread] = {}
117
144
 
118
145
  self._should_disconnect = False
119
146
  self._connection_retries = 0
@@ -129,7 +156,7 @@ class AMQPClient(BrokerClient):
129
156
  self._should_disconnect = False
130
157
  self._channel_flags.unset_all()
131
158
 
132
- if not self._thread:
159
+ if not self._thread or not self._thread.is_alive():
133
160
  self._thread = threading.Thread(target=self._init_connection, daemon=True)
134
161
  self._thread.start()
135
162
 
@@ -139,8 +166,8 @@ class AMQPClient(BrokerClient):
139
166
  def disconnect(self) -> None:
140
167
  """Close all connections."""
141
168
  self._should_disconnect = True
142
- for topic, tag in self._topics_to_consumer_tags.items():
143
- self._cancel_consumer_tag(topic, tag)
169
+ for topic, tag_info in self._topics_to_consumer_tags.items():
170
+ self._cancel_consumer_tag(topic, tag_info.consumer_tag)
144
171
 
145
172
  # since _should_disconnect was set, _connection.ioloop.stop() will now execute after explicit connection close
146
173
  self._connection.close()
@@ -175,18 +202,23 @@ class AMQPClient(BrokerClient):
175
202
  persist: True if message should persist until consumers available, False if message should be removed immediately.
176
203
  """
177
204
  topic = _hierarchy_2_amqp(topic)
178
- self._channel_out.basic_publish(
179
- exchange=_INTERSECT_MESSAGE_EXCHANGE,
180
- routing_key=topic,
181
- body=payload,
182
- properties=pika.BasicProperties(
183
- content_type='text/plain',
184
- delivery_mode=pika.delivery_mode.DeliveryMode.Persistent
185
- if persist
186
- else pika.delivery_mode.DeliveryMode.Transient,
187
- # expiration=None if persist else '8640000',
188
- ),
189
- )
205
+ while not self._channel_flags.is_set():
206
+ self._channel_flags.wait(1.0)
207
+ if self._connection and self._connection.is_open:
208
+ self._channel_out.basic_publish(
209
+ exchange=_INTERSECT_MESSAGE_EXCHANGE,
210
+ routing_key=topic,
211
+ body=payload,
212
+ properties=pika.BasicProperties(
213
+ content_type='text/plain',
214
+ delivery_mode=pika.delivery_mode.DeliveryMode.Persistent
215
+ if persist
216
+ else pika.delivery_mode.DeliveryMode.Transient,
217
+ # expiration=None if persist else '8640000',
218
+ ),
219
+ )
220
+ else:
221
+ logger.error('Unable to publish message %s on topic %s', payload, topic)
190
222
 
191
223
  def subscribe(self, topic: str, persist: bool) -> None:
192
224
  """Subscribe to a topic.
@@ -210,25 +242,41 @@ class AMQPClient(BrokerClient):
210
242
  Therefore, transient queues will be cleaned up.
211
243
  """
212
244
  amqp_topic = _hierarchy_2_amqp(topic)
213
- consumer_tag = self._topics_to_consumer_tags.get(amqp_topic, None)
214
- if consumer_tag:
215
- self._cancel_consumer_tag(amqp_topic, consumer_tag)
245
+ consumer_tag_info = self._topics_to_consumer_tags.get(amqp_topic, None)
246
+ if consumer_tag_info:
247
+ self._cancel_consumer_tag(amqp_topic, consumer_tag_info.consumer_tag)
216
248
 
217
249
  def _cancel_consumer_tag(self, topic: str, consumer_tag: str) -> None:
218
250
  if self._channel_in and self._channel_in.is_open:
219
- cb = functools.partial(self._cancel_consumer_tag_cb, topic=topic)
251
+ cb = functools.partial(
252
+ self._cancel_consumer_tag_cb, topic=topic, consumer_tag=consumer_tag
253
+ )
220
254
  self._channel_in.basic_cancel(
221
255
  consumer_tag,
222
256
  callback=cb,
223
257
  )
224
258
 
225
- def _cancel_consumer_tag_cb(self, _frame: pika.frame.Frame, topic: str) -> None:
259
+ def _cancel_consumer_tag_cb(
260
+ self, _frame: pika.frame.Frame, topic: str, consumer_tag: str
261
+ ) -> None:
226
262
  try:
227
263
  del self._topics_to_consumer_tags[topic]
228
264
  except KeyError:
229
265
  # shouldn't happen because ControlPlaneManager gatekeeps consecutive remove_subscription_channel() calls
230
- pass
266
+ logger.error(
267
+ 'Unable to clean up consumer tag related to topic %s , please inform an INTERSECT-SDK developer if you somehow see this message.',
268
+ topic,
269
+ )
231
270
  logger.info('Unsubscribed from %s', topic)
271
+ try:
272
+ thread = self._consumer_tags_to_threads[consumer_tag]
273
+ # kill thread immediately if not recoverable, wait to send last message if we are shutting down gracefully
274
+ thread.join(0 if self.considered_unrecoverable() else None)
275
+ del self._consumer_tags_to_threads[consumer_tag]
276
+ logger.debug('Consumer cancelled')
277
+ except KeyError:
278
+ # This will commonly be encountered, as several topics will often share a single consumer
279
+ pass
232
280
 
233
281
  # BEGIN CALLBACKS + THREADSAFE FUNCTIONS #
234
282
 
@@ -238,13 +286,18 @@ class AMQPClient(BrokerClient):
238
286
  NOTE: ANY functions which are not eventually called from this function
239
287
  should be called via self._connection.ioloop.add_callback_threadsafe(cb)
240
288
  """
241
- while not self._should_disconnect:
242
- self._connection = pika.adapters.SelectConnection(
243
- parameters=self._connection_params,
244
- on_close_callback=self._on_connection_closed,
245
- on_open_error_callback=self._on_connection_open_error,
246
- on_open_callback=self._on_connection_open,
247
- )
289
+ self._connection = None
290
+ self._channel_in = None
291
+ self._channel_out = None
292
+
293
+ while not self._should_disconnect and not self.considered_unrecoverable():
294
+ if not self._connection or self._connection.is_closed:
295
+ self._connection = pika.adapters.SelectConnection(
296
+ parameters=self._connection_params,
297
+ on_close_callback=self._on_connection_closed,
298
+ on_open_error_callback=self._on_connection_open_error,
299
+ on_open_callback=self._on_connection_open,
300
+ )
248
301
 
249
302
  # Loops forever until ioloop.stop is called WHEN self._should_disconnect is True
250
303
  self._connection.ioloop.start()
@@ -415,6 +468,32 @@ class AMQPClient(BrokerClient):
415
468
  topic: str,
416
469
  queue_name: str,
417
470
  persist: bool,
471
+ ) -> None:
472
+ """Sets up basic QOS on the current channel.
473
+
474
+ Args:
475
+ _unused_frame: AMQP response from binding to the queue. Ignored.
476
+ channel: The Channel being instantiated.
477
+ topic: Name of the topic on the broker.
478
+ queue_name: The name of the queue on the AMQP broker.
479
+ persist: Whether or not our queue should persist on either broker or application shutdown.
480
+ """
481
+ cb = functools.partial(
482
+ self._on_basic_qos_set,
483
+ channel=channel,
484
+ topic=topic,
485
+ queue_name=queue_name,
486
+ persist=persist,
487
+ )
488
+ channel.basic_qos(prefetch_count=_PREFETCH_COUNT, callback=cb)
489
+
490
+ def _on_basic_qos_set(
491
+ self,
492
+ _unused_frame: Frame,
493
+ channel: Channel,
494
+ topic: str,
495
+ queue_name: str,
496
+ persist: bool,
418
497
  ) -> None:
419
498
  """Consumes a message from the given channel.
420
499
 
@@ -429,13 +508,15 @@ class AMQPClient(BrokerClient):
429
508
  """
430
509
  cb = functools.partial(self._on_consume_ok, topic=topic)
431
510
  message_cb = functools.partial(self._consume_message, persist=persist)
511
+ info = ConsumerTagInfo()
512
+ self._topics_to_consumer_tags[topic] = info
432
513
  consumer_tag = channel.basic_consume(
433
514
  queue=queue_name,
434
515
  auto_ack=not persist, # persistent messages should be manually acked and we have no reason to NACK a message for now
435
516
  on_message_callback=message_cb,
436
517
  callback=cb,
437
518
  )
438
- self._topics_to_consumer_tags[topic] = consumer_tag
519
+ info.obtain_consumer_tag(consumer_tag)
439
520
 
440
521
  def _on_consume_ok(self, _unused_frame: Frame, topic: str) -> None:
441
522
  """Sets the consume subscription ready event.
@@ -468,14 +549,50 @@ class AMQPClient(BrokerClient):
468
549
  body: the AMQP message to be handled.
469
550
  persist: Whether or not our queue should persist on either broker or application shutdown.
470
551
  """
552
+ # if we got a message when we shouldn't, quickly try to requeue it
553
+ if self.considered_unrecoverable() or self._should_disconnect:
554
+ logger.warning(
555
+ "WARNING: A message for topic %s has been received when it shouldn't, attempting requeue"
556
+ )
557
+ channel.basic_reject(basic_deliver.delivery_tag)
558
+ return
559
+
471
560
  tth_key = _amqp_2_hierarchy(basic_deliver.routing_key)
472
561
  topic_handler = self._topics_to_handlers().get(tth_key)
473
562
  if topic_handler:
474
- for cb in topic_handler.callbacks:
475
- cb(body)
563
+ consumer_tag_info = self._topics_to_consumer_tags.get(basic_deliver.routing_key)
564
+ if not consumer_tag_info:
565
+ logger.error(
566
+ 'Could not fetch consumer tag for topic %s, please inform an INTERSECT-SDK developer that you saw this message',
567
+ tth_key,
568
+ )
569
+ return
570
+ while not consumer_tag_info.consumer_tag_obtained():
571
+ consumer_tag_info.wait(1.0)
572
+ thrd = threading.Thread(
573
+ target=self._consume_message_subthread,
574
+ args=(channel, topic_handler.callbacks, body, basic_deliver.delivery_tag, persist),
575
+ )
576
+ self._consumer_tags_to_threads[consumer_tag_info.consumer_tag] = thrd
577
+ thrd.start()
578
+ elif persist:
579
+ # we somehow got a message that we don't want, discard it
580
+ channel.basic_ack(basic_deliver.delivery_tag)
581
+
582
+ def _consume_message_subthread(
583
+ self,
584
+ channel: Channel,
585
+ callbacks: set[Callable[[bytes], None]],
586
+ body: bytes,
587
+ delivery_tag: int,
588
+ persist: bool,
589
+ ) -> None:
590
+ """This is a subthread which executes the consumer code without blocking the IO loop. Without using a subthread, the AMQP heartbeat checker will be blocked."""
591
+ for cb in callbacks:
592
+ cb(body)
476
593
  # With persistent messages, we only acknowledge the message AFTER we are done processing
477
594
  # (this removes the message from the broker queue)
478
595
  # this allows us to retry a message if the broker OR this application goes down
479
596
  # We currently never NACK or reject a message because in INTERSECT, applications currently never "share" a queue.
480
597
  if persist:
481
- channel.basic_ack(basic_deliver.delivery_tag)
598
+ channel.basic_ack(delivery_tag)
@@ -81,6 +81,7 @@ class MQTTClient(BrokerClient):
81
81
  """Connect to the defined broker."""
82
82
  # Create a client to connect to RabbitMQ
83
83
  # TODO MQTT v5 implementations should set clean_start to NEVER here
84
+ self._should_disconnect = False
84
85
  self._connected_flag.clear()
85
86
  self._connection.connect(self.host, self.port, 60)
86
87
  self._connection.loop_start()
@@ -90,8 +91,9 @@ class MQTTClient(BrokerClient):
90
91
  def disconnect(self) -> None:
91
92
  """Disconnect from the broker."""
92
93
  self._should_disconnect = True
93
- self._connection.disconnect()
94
- self._connection.loop_stop()
94
+ if self._connection:
95
+ self._connection.disconnect()
96
+ self._connection.loop_stop()
95
97
 
96
98
  def is_connected(self) -> bool:
97
99
  """Check if there is an active connection to the broker.
@@ -143,9 +143,9 @@ def _get_functions(
143
143
  logger.warning(
144
144
  f"Class '{capability.__name__}' has no function annotated with the @intersect_status() decorator. No status information will be provided when sending status messages."
145
145
  )
146
- if not intersect_messages and not intersect_events:
146
+ if not intersect_messages and not intersect_events and not intersect_status:
147
147
  die(
148
- f"No intersect annotations detected on class '{capability.__name__}'. Please annotate at least one entrypoint with '@intersect_message()', or one event-emitting function with '@intersect_event()' ."
148
+ f"No intersect annotations detected on class '{capability.__name__}'. Please annotate at least one entrypoint with '@intersect_message()', or one event-emitting function with '@intersect_event()', or create an '@intersect_status' function."
149
149
  )
150
150
  return intersect_status, intersect_messages, intersect_events
151
151
 
@@ -68,6 +68,8 @@ class IntersectBaseCapabilityImplementation:
68
68
  is not IntersectBaseCapabilityImplementation.intersect_sdk_emit_event
69
69
  or cls.intersect_sdk_call_service
70
70
  is not IntersectBaseCapabilityImplementation.intersect_sdk_call_service
71
+ or cls.intersect_sdk_listen_for_service_event
72
+ is not IntersectBaseCapabilityImplementation.intersect_sdk_listen_for_service_event
71
73
  ):
72
74
  msg = f"{cls.__name__}: Attempted to override a reserved INTERSECT-SDK function (don't start your function names with '_intersect_sdk_' or 'intersect_sdk_')"
73
75
  raise RuntimeError(msg)
@@ -8,7 +8,7 @@ from __future__ import annotations
8
8
  from ._internal.version import strip_version_metadata
9
9
 
10
10
  # may include build metadata
11
- __version__ = '0.8.1'
11
+ __version__ = '0.8.3'
12
12
 
13
13
  version_string = strip_version_metadata(__version__)
14
14
  """
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "asyncapi": "2.6.0",
3
- "x-intersect-version": "0.8.0",
3
+ "x-intersect-version": "0.8.3",
4
4
  "info": {
5
5
  "title": "test.test.test.test.test",
6
6
  "description": "INTERSECT schema",
@@ -982,11 +982,7 @@
982
982
  "type": "string"
983
983
  },
984
984
  "data_handler": {
985
- "allOf": [
986
- {
987
- "$ref": "#/components/messageTraits/commonHeaders/userspaceHeaders/$defs/IntersectDataHandler"
988
- }
989
- ],
985
+ "$ref": "#/components/messageTraits/commonHeaders/userspaceHeaders/$defs/IntersectDataHandler",
990
986
  "default": 0,
991
987
  "description": "Code signifying where data is stored."
992
988
  },
@@ -1039,11 +1035,7 @@
1039
1035
  "type": "string"
1040
1036
  },
1041
1037
  "data_handler": {
1042
- "allOf": [
1043
- {
1044
- "$ref": "#/components/messageTraits/commonHeaders/eventHeaders/$defs/IntersectDataHandler"
1045
- }
1046
- ],
1038
+ "$ref": "#/components/messageTraits/commonHeaders/eventHeaders/$defs/IntersectDataHandler",
1047
1039
  "default": 0,
1048
1040
  "description": "Code signifying where data is stored."
1049
1041
  },
@@ -73,6 +73,19 @@ def test_no_override():
73
73
 
74
74
  assert 'BadClass3: Attempted to override a reserved INTERSECT-SDK function' in str(ex)
75
75
 
76
+ with pytest.raises(RuntimeError) as ex:
77
+
78
+ class BadClass4(IntersectBaseCapabilityImplementation):
79
+ def intersect_sdk_listen_for_service_event(
80
+ self,
81
+ service,
82
+ event_name,
83
+ response_handler,
84
+ ) -> None:
85
+ pass
86
+
87
+ assert 'BadClass4: Attempted to override a reserved INTERSECT-SDK function' in str(ex)
88
+
76
89
 
77
90
  # Note that the ONLY thing the capability itself checks for are annotated functions.
78
91
  # The event definitions and overall schema validation are a service-specific feature
@@ -197,7 +197,7 @@ def test_disallow_dynamic_list_subtyping(caplog: pytest.LogCaptureFixture):
197
197
  with pytest.raises(SystemExit):
198
198
  get_schema_helper([MockAnyList])
199
199
  assert "parameter 'param' type annotation" in caplog.text
200
- assert 'list subtyping may not be dynamic in INTERSECT' in caplog.text
200
+ assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
201
201
 
202
202
 
203
203
  def test_disallow_dynamic_list_subtyping_complex(caplog: pytest.LogCaptureFixture):
@@ -227,7 +227,7 @@ def test_disallow_dynamic_set_subtyping(caplog: pytest.LogCaptureFixture):
227
227
  with pytest.raises(SystemExit):
228
228
  get_schema_helper([MockAnySet])
229
229
  assert "parameter 'param' type annotation" in caplog.text
230
- assert 'set subtyping may not be dynamic in INTERSECT' in caplog.text
230
+ assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
231
231
 
232
232
 
233
233
  def test_disallow_dynamic_frozenset_subtyping(caplog: pytest.LogCaptureFixture):
@@ -241,7 +241,7 @@ def test_disallow_dynamic_frozenset_subtyping(caplog: pytest.LogCaptureFixture):
241
241
  with pytest.raises(SystemExit):
242
242
  get_schema_helper([MockAnyFrozenSet])
243
243
  assert "parameter 'param' type annotation" in caplog.text
244
- assert 'frozenset subtyping may not be dynamic in INTERSECT' in caplog.text
244
+ assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
245
245
 
246
246
 
247
247
  def test_disallow_dynamic_generator_subtyping(caplog: pytest.LogCaptureFixture):
@@ -288,7 +288,7 @@ def test_disallow_dynamic_dict_value_type(caplog: pytest.LogCaptureFixture):
288
288
  with pytest.raises(SystemExit):
289
289
  get_schema_helper([MockAnyDictValue])
290
290
  assert "parameter 'param' type annotation" in caplog.text
291
- assert 'dict or mapping: value type cannot be Any/object for INTERSECT' in caplog.text
291
+ assert 'dynamic typing is not allowed for INTERSECT schemas' in caplog.text
292
292
 
293
293
 
294
294
  def test_disallow_dynamic_tuple_subtyping(caplog: pytest.LogCaptureFixture):
@@ -1,7 +1,15 @@
1
1
  import json
2
2
  from pathlib import Path
3
3
 
4
- from intersect_sdk import IntersectDataHandler, IntersectMimeType
4
+ from intersect_sdk import (
5
+ IntersectBaseCapabilityImplementation,
6
+ IntersectDataHandler,
7
+ IntersectEventDefinition,
8
+ IntersectMimeType,
9
+ intersect_event,
10
+ intersect_message,
11
+ intersect_status,
12
+ )
5
13
  from intersect_sdk._internal.constants import (
6
14
  REQUEST_CONTENT,
7
15
  RESPONSE_CONTENT,
@@ -22,7 +30,43 @@ def get_fixture_path(fixture: str):
22
30
  return Path(__file__).absolute().parents[1] / 'fixtures' / fixture
23
31
 
24
32
 
25
- # TESTS ##################
33
+ # MINIMAL ANNOTATION TESTS ######################
34
+
35
+
36
+ def test_minimal_intersect_annotations():
37
+ class CapWithMessage(IntersectBaseCapabilityImplementation):
38
+ intersect_sdk_capability_name = 'CapWithMessage'
39
+
40
+ @intersect_message
41
+ def message_function(self, theinput: int) -> int:
42
+ return theinput * 4
43
+
44
+ class CapWithEvent(IntersectBaseCapabilityImplementation):
45
+ intersect_sdk_capability_name = 'CapWithEvent'
46
+
47
+ @intersect_event(events={'event': IntersectEventDefinition(event_type=str)})
48
+ def event_function(self):
49
+ self.intersect_sdk_emit_event('event', 'emitted_value')
50
+
51
+ class CapWithStatus(IntersectBaseCapabilityImplementation):
52
+ intersect_sdk_capability_name = 'CapWithStatus'
53
+
54
+ @intersect_status
55
+ def status_function(self) -> str:
56
+ return 'Up'
57
+
58
+ schemas = get_schema_from_capability_implementations(
59
+ [
60
+ CapWithEvent,
61
+ CapWithMessage,
62
+ CapWithStatus,
63
+ ],
64
+ FAKE_HIERARCHY_CONFIG,
65
+ )
66
+ assert len(schemas['capabilities']) == 3
67
+
68
+
69
+ # FIXTURE TESTS ##################
26
70
 
27
71
 
28
72
  def test_schema_comparison():
File without changes