qena-shared-lib 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. qena_shared_lib/__init__.py +3 -2
  2. qena_shared_lib/application.py +4 -4
  3. qena_shared_lib/background.py +9 -7
  4. qena_shared_lib/exception_handling.py +409 -0
  5. qena_shared_lib/exceptions.py +170 -57
  6. qena_shared_lib/http/__init__.py +90 -0
  7. qena_shared_lib/{http.py → http/_base.py} +36 -36
  8. qena_shared_lib/http/_exception_handlers.py +202 -0
  9. qena_shared_lib/kafka/__init__.py +21 -0
  10. qena_shared_lib/kafka/_base.py +233 -0
  11. qena_shared_lib/kafka/_consumer.py +597 -0
  12. qena_shared_lib/kafka/_exception_handlers.py +124 -0
  13. qena_shared_lib/kafka/_producer.py +133 -0
  14. qena_shared_lib/logging.py +17 -13
  15. qena_shared_lib/rabbitmq/__init__.py +4 -6
  16. qena_shared_lib/rabbitmq/_base.py +68 -132
  17. qena_shared_lib/rabbitmq/_channel.py +2 -4
  18. qena_shared_lib/rabbitmq/_exception_handlers.py +69 -142
  19. qena_shared_lib/rabbitmq/_listener.py +246 -157
  20. qena_shared_lib/rabbitmq/_publisher.py +5 -5
  21. qena_shared_lib/rabbitmq/_rpc_client.py +21 -22
  22. qena_shared_lib/remotelogging/_base.py +20 -20
  23. qena_shared_lib/remotelogging/logstash/_base.py +2 -2
  24. qena_shared_lib/remotelogging/logstash/_http_sender.py +2 -4
  25. qena_shared_lib/remotelogging/logstash/_tcp_sender.py +2 -2
  26. qena_shared_lib/scheduler.py +24 -15
  27. qena_shared_lib/security.py +39 -32
  28. qena_shared_lib/utils.py +13 -11
  29. {qena_shared_lib-0.1.16.dist-info → qena_shared_lib-0.1.18.dist-info}/METADATA +9 -1
  30. qena_shared_lib-0.1.18.dist-info/RECORD +38 -0
  31. qena_shared_lib/exception_handlers.py +0 -235
  32. qena_shared_lib-0.1.16.dist-info/RECORD +0 -31
  33. {qena_shared_lib-0.1.16.dist-info → qena_shared_lib-0.1.18.dist-info}/WHEEL +0 -0
@@ -0,0 +1,597 @@
1
+ from asyncio import Task, gather, iscoroutinefunction
2
+ from dataclasses import dataclass
3
+ from functools import partial
4
+ from inspect import Parameter, signature
5
+ from types import MappingProxyType
6
+ from typing import Any, Callable, Generic, TypeVar, cast
7
+
8
+ try:
9
+ from aiokafka.consumer import AIOKafkaConsumer
10
+ from aiokafka.structs import ConsumerRecord
11
+ except ImportError:
12
+ pass
13
+ from punq import Container
14
+ from pydantic_core import from_json
15
+
16
+ from ..dependencies.miscellaneous import validate_annotation
17
+ from ..exception_handling import ServiceContext
18
+ from ..logging import LoggerFactory
19
+ from ..remotelogging import BaseRemoteLogSender
20
+ from ..utils import AsyncEventLoopMixin, TypeAdapterCache
21
+
22
+ C = TypeVar("C")
23
+ K = TypeVar("K")
24
+ V = TypeVar("V")
25
+
26
+ DEFAULT_TARGET = "__default__"
27
+ CONSUMER_ATTRIBUTE = "__kafka_consumer__"
28
+ CONSUMER_METHOD_ATTRIBUTE = "__kafka_consumer_method__"
29
+ KEY_PARAMETER_POSITION = 1
30
+ KEY_PARAMETER_NAME = "key"
31
+ VALUE_PARAMETER_POSITION = 2
32
+ VALUE_PARAMETER_NAME = "value"
33
+
34
+
35
+ @dataclass
36
+ class ConsumerContext(ServiceContext):
37
+ topics: list[str]
38
+ group_id: str | None
39
+ target: str
40
+ key: bytes | None
41
+ value: bytes | None
42
+
43
+
44
+ @dataclass
45
+ class ConsumerConfigs:
46
+ bootstrap_servers: str
47
+ security_protocol: str
48
+ sasl_mechanism: str
49
+ sasl_plain_username: str | None
50
+ sasl_plain_password: str | None
51
+ extra_configs: dict[str, Any]
52
+
53
+
54
+ @dataclass
55
+ class ConsumerMethodContainer(Generic[K, V], AsyncEventLoopMixin):
56
+ consumer_method: Callable[..., Any]
57
+ parameters: MappingProxyType[str, Parameter]
58
+ key_type: type[K] | None
59
+ value_type: type[V] | None
60
+ dependency_types: dict[str, type[Any]]
61
+
62
+ def __post_init__(self) -> None:
63
+ self._is_async_consumer = iscoroutinefunction(self.consumer_method)
64
+
65
+ async def __call__(
66
+ self,
67
+ key: bytes | None,
68
+ value: bytes | None,
69
+ container: Container,
70
+ topics: list[str],
71
+ target: str,
72
+ group_id: str | None,
73
+ ) -> Any:
74
+ consumer_method_args, consumer_method_kwargs = self._parse_args(
75
+ key=key,
76
+ value=value,
77
+ container=container,
78
+ topics=topics,
79
+ target=target,
80
+ group_id=group_id,
81
+ )
82
+
83
+ if self._is_async_consumer:
84
+ await self.consumer_method(
85
+ *consumer_method_args, **consumer_method_kwargs
86
+ )
87
+ else:
88
+ await self.loop.run_in_executor(
89
+ executor=None,
90
+ func=partial(
91
+ self.consumer_method,
92
+ *consumer_method_args,
93
+ **consumer_method_kwargs,
94
+ ),
95
+ )
96
+
97
+ def _parse_args(
98
+ self,
99
+ key: bytes | None,
100
+ value: bytes | None,
101
+ container: Container,
102
+ topics: list[str],
103
+ target: str,
104
+ group_id: str | None,
105
+ ) -> tuple[list[Any], dict[str, Any]]:
106
+ if key is not None:
107
+ key = from_json(key)
108
+
109
+ if value is not None:
110
+ value = from_json(value)
111
+
112
+ consumer_method_args = []
113
+ consumer_method_kwargs = {}
114
+ assigned_args = []
115
+
116
+ for parameter_position, (parameter_name, parameter) in enumerate(
117
+ iterable=self.parameters.items(), start=1
118
+ ):
119
+ if parameter_name in assigned_args:
120
+ raise RuntimeError(
121
+ f"parameter {parameter_name} has already been assigned an argument"
122
+ )
123
+
124
+ arg: Any = None
125
+
126
+ if parameter_position == KEY_PARAMETER_POSITION:
127
+ if self.key_type is None:
128
+ arg = key
129
+ else:
130
+ arg = self._validate_key_value(
131
+ object=key, object_type=self.key_type
132
+ )
133
+ elif parameter_position == VALUE_PARAMETER_POSITION:
134
+ if self.value_type is None:
135
+ arg = value
136
+ else:
137
+ arg = self._validate_key_value(
138
+ object=value, object_type=self.value_type
139
+ )
140
+ else:
141
+ if parameter.annotation is Parameter.empty:
142
+ raise TypeError(
143
+ f"parameter {parameter_name} has no annotation"
144
+ )
145
+ elif parameter.annotation is ConsumerContext:
146
+ arg = ConsumerContext(
147
+ topics=topics,
148
+ group_id=group_id,
149
+ target=target,
150
+ key=key,
151
+ value=value,
152
+ )
153
+ else:
154
+ dependency_type = self.dependency_types.get(parameter_name)
155
+
156
+ if dependency_type is not None:
157
+ arg = container.resolve(dependency_type)
158
+ elif (
159
+ parameter_name not in assigned_args
160
+ and parameter.default is Parameter.empty
161
+ ):
162
+ raise ValueError(
163
+ f"parameter {parameter_name} has no default value"
164
+ )
165
+
166
+ assigned_args.append(parameter_name)
167
+
168
+ match parameter.kind:
169
+ case (
170
+ Parameter.POSITIONAL_ONLY
171
+ | Parameter.VAR_POSITIONAL
172
+ | Parameter.POSITIONAL_OR_KEYWORD
173
+ ):
174
+ consumer_method_args.append(arg)
175
+ case Parameter.KEYWORD_ONLY | Parameter.VAR_KEYWORD:
176
+ consumer_method_kwargs[parameter_name] = arg
177
+
178
+ return consumer_method_args, consumer_method_kwargs
179
+
180
+ def _validate_key_value(self, object: Any, object_type: type) -> Any:
181
+ return TypeAdapterCache.get_type_adapter(object_type).validate_python(
182
+ object
183
+ )
184
+
185
+
186
+ def topics_repr(topics: list[str]) -> str:
187
+ return ", ".join(topics)
188
+
189
+
190
+ def group_id_repr(group_id: str | None) -> str:
191
+ return group_id or "no_group_id"
192
+
193
+
194
+ class Consumer(AsyncEventLoopMixin):
195
+ def __init__(self, topics: list[str], group_id: str | None = None) -> None:
196
+ self._topics = topics
197
+ self._group_id = group_id
198
+ self._consumers: dict[str, ConsumerMethodContainer[object, object]] = {}
199
+ self._consumers_tasks: list[Task[Any]] = []
200
+ self._cancelled = False
201
+ self._logger = LoggerFactory.get_logger("kafka.consumer")
202
+
203
+ def consume(
204
+ self, target: str | None = None
205
+ ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
206
+ def wrapper(consumer_method: Callable[..., Any]) -> Callable[..., Any]:
207
+ if not callable(consumer_method):
208
+ raise TypeError(
209
+ f"consumer method argument not a callable, got {type(consumer_method)}"
210
+ )
211
+
212
+ self._register_consumer_method(
213
+ target=target,
214
+ consumer_method=consumer_method,
215
+ parameters=signature(consumer_method).parameters,
216
+ )
217
+
218
+ return consumer_method
219
+
220
+ return wrapper
221
+
222
+ def add_consumer_method(
223
+ self,
224
+ target: str | None,
225
+ consumer_method: Callable[..., Any],
226
+ ) -> None:
227
+ self._register_consumer_method(
228
+ target=target,
229
+ consumer_method=consumer_method,
230
+ parameters=signature(consumer_method).parameters,
231
+ )
232
+
233
+ def _register_consumer_method(
234
+ self,
235
+ target: str | None,
236
+ consumer_method: Callable[..., Any],
237
+ parameters: MappingProxyType[str, Parameter],
238
+ ) -> None:
239
+ target = target or DEFAULT_TARGET
240
+
241
+ if target in self._consumers:
242
+ self._logger.warning(
243
+ "consumer with a target `%s` already exists", target
244
+ )
245
+
246
+ key_parameter_type = None
247
+ value_parameter_type = None
248
+ dependency_types = {}
249
+
250
+ for parameter_position, (parameter_name, parameter) in enumerate(
251
+ iterable=parameters.items(), start=1
252
+ ):
253
+ if parameter.annotation is Parameter.empty:
254
+ continue
255
+
256
+ if parameter_position == KEY_PARAMETER_POSITION:
257
+ if parameter_name != KEY_PARAMETER_NAME:
258
+ raise ValueError(
259
+ f"key parameter name is not `{KEY_PARAMETER_NAME}` got `{parameter_name}`"
260
+ )
261
+
262
+ if (
263
+ parameter.kind is Parameter.VAR_POSITIONAL
264
+ or parameter.kind is Parameter.VAR_KEYWORD
265
+ ):
266
+ raise TypeError(
267
+ "`key` cannot be variable positional or keyword parameter"
268
+ )
269
+
270
+ key_parameter_type = parameter.annotation
271
+
272
+ TypeAdapterCache.cache_annotation(key_parameter_type)
273
+ elif parameter_position == VALUE_PARAMETER_POSITION:
274
+ if parameter_name != VALUE_PARAMETER_NAME:
275
+ raise ValueError(
276
+ f"value parameter name is not `{VALUE_PARAMETER_NAME}` got `{parameter_name}`"
277
+ )
278
+
279
+ if (
280
+ parameter.kind is Parameter.VAR_POSITIONAL
281
+ or parameter.kind is Parameter.VAR_KEYWORD
282
+ ):
283
+ raise TypeError(
284
+ "`value` cannot be variable positional or keyword parameter"
285
+ )
286
+
287
+ value_parameter_type = parameter.annotation
288
+
289
+ TypeAdapterCache.cache_annotation(value_parameter_type)
290
+ else:
291
+ if parameter.annotation is ConsumerContext:
292
+ continue
293
+
294
+ dependency = validate_annotation(parameter=parameter)
295
+
296
+ if dependency is None:
297
+ raise TypeError(
298
+ f"`{parameter_name}` has unsupported parameter type annotation"
299
+ )
300
+
301
+ dependency_types[parameter_name] = dependency
302
+
303
+ self._consumers[target] = ConsumerMethodContainer(
304
+ consumer_method=consumer_method,
305
+ parameters=parameters,
306
+ key_type=key_parameter_type,
307
+ value_type=value_parameter_type,
308
+ dependency_types=dependency_types,
309
+ )
310
+
311
+ async def configure(
312
+ self,
313
+ configs: ConsumerConfigs,
314
+ container: Container,
315
+ remote_logger: BaseRemoteLogSender,
316
+ on_exception_callback: Callable[[ConsumerContext, BaseException], bool],
317
+ ) -> None:
318
+ self._configs = configs
319
+ self._container = container
320
+ self._remote_logger = remote_logger
321
+ self._on_exception_callback = on_exception_callback
322
+ self._kafka_consumer = AIOKafkaConsumer(
323
+ *self._topics,
324
+ group_id=self._group_id,
325
+ bootstrap_servers=configs.bootstrap_servers,
326
+ security_protocol=configs.security_protocol,
327
+ sasl_mechanism=configs.sasl_mechanism,
328
+ sasl_plain_username=configs.sasl_plain_username,
329
+ sasl_plain_password=configs.sasl_plain_password,
330
+ **configs.extra_configs,
331
+ )
332
+
333
+ await self._start()
334
+
335
+ async def _start(self) -> None:
336
+ await self._kafka_consumer.start()
337
+ self.loop.create_task(self._start_consuming())
338
+
339
+ async def cancel(self) -> None:
340
+ self._cancelled = True
341
+
342
+ await self._kafka_consumer.stop()
343
+
344
+ _ = await gather(*self._consumers_tasks, return_exceptions=True)
345
+
346
+ async def _start_consuming(self) -> None:
347
+ async for consumer_record in self._kafka_consumer:
348
+ target = self._get_target(consumer_record)
349
+ consumer = self._get_consumer(target)
350
+
351
+ if consumer is None:
352
+ topics = topics_repr(self._topics)
353
+ group_id = group_id_repr(self._group_id)
354
+
355
+ self._remote_logger.error(
356
+ message=f"no consumer registered for target `{target}` on topics `{topics}` and group id `{group_id}`",
357
+ tags=[
358
+ "kafka",
359
+ "consumer_doesnt_exist",
360
+ *self._topics,
361
+ group_id,
362
+ target,
363
+ ],
364
+ extra={
365
+ "serviceType": "kafka",
366
+ "topics": topics,
367
+ "groupId": group_id,
368
+ "target": target,
369
+ },
370
+ )
371
+
372
+ continue
373
+
374
+ self._execute(
375
+ target=target,
376
+ consumer=consumer,
377
+ key=consumer_record.key,
378
+ value=consumer_record.value,
379
+ )
380
+
381
+ def _get_target(self, consumer_record: ConsumerRecord) -> str:
382
+ target = dict(consumer_record.headers).get("target")
383
+
384
+ if target is not None:
385
+ return cast(str, from_json(target))
386
+
387
+ return DEFAULT_TARGET
388
+
389
+ def _get_consumer(
390
+ self, target: str
391
+ ) -> ConsumerMethodContainer[object, object] | None:
392
+ return self._consumers.get(target)
393
+
394
+ def _execute(
395
+ self,
396
+ target: str,
397
+ consumer: ConsumerMethodContainer[K, V],
398
+ key: bytes | None,
399
+ value: bytes | None,
400
+ ) -> None:
401
+ consumer_task = self.loop.create_task(
402
+ consumer(
403
+ key=key,
404
+ value=value,
405
+ container=self._container,
406
+ topics=self._topics,
407
+ target=target,
408
+ group_id=self._group_id,
409
+ )
410
+ )
411
+
412
+ self._consumers_tasks.append(consumer_task)
413
+ consumer_task.add_done_callback(
414
+ partial(
415
+ self._consumer_done_callback,
416
+ target=target,
417
+ key=key,
418
+ value=value,
419
+ )
420
+ )
421
+
422
+ def _consumer_done_callback(
423
+ self,
424
+ task: Task[Any],
425
+ target: str,
426
+ key: bytes | None,
427
+ value: bytes | None,
428
+ ) -> None:
429
+ if not self._cancelled and task in self._consumers_tasks:
430
+ self._consumers_tasks.remove(task)
431
+
432
+ if task.cancelled():
433
+ return
434
+
435
+ exception = task.exception()
436
+ topics = topics_repr(self._topics)
437
+ group_id = group_id_repr(self._group_id)
438
+
439
+ if exception is not None:
440
+ tags = ["kafka", "consumer_error", *self._topics, group_id, target]
441
+ extra = {
442
+ "serviceType": "kafka",
443
+ "topics": topics,
444
+ "groupId": group_id,
445
+ "target": target,
446
+ }
447
+
448
+ try:
449
+ exception_callback_succeeded = self._on_exception_callback(
450
+ ConsumerContext(
451
+ topics=self._topics,
452
+ group_id=self._group_id,
453
+ target=target,
454
+ key=key,
455
+ value=value,
456
+ ).set_labels(
457
+ {
458
+ "topics": topics,
459
+ "group_id": group_id,
460
+ "target": target,
461
+ "exception": exception.__class__.__name__,
462
+ }
463
+ ),
464
+ exception,
465
+ )
466
+ except:
467
+ self._remote_logger.exception(
468
+ message=f"error occured while invoking kafka exception handler callback for target `{target}` , group id `{group_id}` and topics `{topics}`",
469
+ tags=tags,
470
+ extra=extra,
471
+ )
472
+ else:
473
+ if not exception_callback_succeeded:
474
+ self._remote_logger.error(
475
+ message=f"error occured while consuming event in consumer `{target}` and for topic `{topics}` and group id `{group_id}`",
476
+ tags=tags,
477
+ extra=extra,
478
+ exception=exception,
479
+ )
480
+
481
+ return
482
+
483
+ self._logger.debug(
484
+ "event from topics `%s` and group id `%s` consumed by consumer `%s`",
485
+ topics,
486
+ group_id,
487
+ target,
488
+ )
489
+
490
+ def __call__(self, consumer: type[C]) -> type[C]:
491
+ setattr(consumer, CONSUMER_ATTRIBUTE, self)
492
+
493
+ return consumer
494
+
495
+
496
+ @dataclass
497
+ class ConsumerMethodMetadata:
498
+ target: str | None = None
499
+
500
+
501
+ def consumer(
502
+ topics: list[str],
503
+ group_id: str | None = None,
504
+ ) -> Consumer:
505
+ return Consumer(
506
+ topics=topics,
507
+ group_id=group_id,
508
+ )
509
+
510
+
511
+ def consume(
512
+ target: str | None = None,
513
+ ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
514
+ def wrapper(consumer_method: Callable[..., Any]) -> Callable[..., Any]:
515
+ if not callable(consumer_method):
516
+ raise TypeError(
517
+ f"consumer method argument not a callable, got {type(consumer_method)}"
518
+ )
519
+
520
+ setattr(
521
+ consumer_method,
522
+ CONSUMER_METHOD_ATTRIBUTE,
523
+ ConsumerMethodMetadata(target=target),
524
+ )
525
+
526
+ return consumer_method
527
+
528
+ return wrapper
529
+
530
+
531
+ class ConsumerBase:
532
+ def get_inner_consumer(self) -> Consumer:
533
+ consumer = getattr(self, CONSUMER_ATTRIBUTE, None)
534
+
535
+ if not isinstance(consumer, Consumer):
536
+ raise TypeError(
537
+ f"{self.__class__.__name__} not a consumer, possibly no annotated with either `consumer`"
538
+ )
539
+
540
+ return consumer
541
+
542
+ def register_consumer_methods(self) -> Consumer:
543
+ consumer = self.get_inner_consumer()
544
+
545
+ for attribute_name in dir(self):
546
+ attribute = getattr(self, attribute_name, None)
547
+
548
+ if attribute is None:
549
+ continue
550
+
551
+ consumer_method_metadata = self._validate_consumer_method_attribute(
552
+ attribute=attribute,
553
+ consumer=consumer,
554
+ )
555
+
556
+ if consumer_method_metadata is None:
557
+ continue
558
+
559
+ consumer.add_consumer_method(
560
+ target=consumer_method_metadata.target,
561
+ consumer_method=attribute,
562
+ )
563
+
564
+ return consumer
565
+
566
+ def _validate_consumer_method_attribute(
567
+ self,
568
+ attribute: Any,
569
+ consumer: Consumer,
570
+ ) -> ConsumerMethodMetadata | None:
571
+ consumer_method_metadata = getattr(
572
+ attribute, CONSUMER_METHOD_ATTRIBUTE, None
573
+ )
574
+
575
+ if consumer_method_metadata is None:
576
+ return None
577
+
578
+ if not isinstance(consumer_method_metadata, ConsumerMethodMetadata):
579
+ raise TypeError(
580
+ f"expected `{CONSUMER_METHOD_ATTRIBUTE}` to be of type `ConsumerMethodMetadata`, got {type(consumer_method_metadata)}"
581
+ )
582
+
583
+ if not callable(attribute):
584
+ raise TypeError(
585
+ f"object annotated with `{CONSUMER_METHOD_ATTRIBUTE}` is not callable"
586
+ )
587
+
588
+ if Consumer not in consumer.__class__.mro():
589
+ consumer_mro = ", ".join(
590
+ map(lambda c: c.__name__, consumer.__class__.mro())
591
+ )
592
+
593
+ raise TypeError(
594
+ f"consumer method is not correct member of `Consumer` got `[{consumer_mro}]` super classes"
595
+ )
596
+
597
+ return consumer_method_metadata