explicit-python-kafka 1.1.1__py3-none-any.whl

explicit/kafka/adapters/messaging.py

@@ -0,0 +1,202 @@
+"""Реализация адаптера обмена сообщениями."""
+from typing import Dict
+from typing import Generator
+from typing import Optional
+import logging
+
+from confluent_kafka import Consumer as Subscriber
+from confluent_kafka import KafkaError
+from confluent_kafka import KafkaException
+from confluent_kafka import Message as KafkaMessage
+from confluent_kafka import Producer as Publisher
+from confluent_kafka.admin import AdminClient
+from confluent_kafka.cimpl import NewTopic
+
+from explicit.adapters.messaging import AbstractAdapter
+from explicit.kafka.domain.model import PublishConfig
+from explicit.kafka.domain.model import SubscribeConfig
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def publish_callback(error, message: KafkaMessage):
+    if error is not None:
+        logger.error('Ошибка при публикации сообщения: %s', error)
+    else:
+        logger.info('Сообщение доставлено: %s [%s]', message.topic(), message.partition())
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug('Тело сообщения: %s', message.value())
+
+
+class Adapter(AbstractAdapter):
+
+    """Адаптер обмена сообщениями через Kafka."""
+
+    def __init__(
+        self, *,
+        subscribe_config: Optional[SubscribeConfig] = None,
+        publish_config: Optional[PublishConfig] = None,
+        must_ensure_topics: Optional[bool] = True
+    ) -> None:
+        """Инициализация адаптера.
+
+        :param subscribe_config: параметры роли подписчика.
+        :param publish_config: параметры роли издателя.
+        :param must_ensure_topics: убедиться в существовании топика
+            перед выполнением действий.
+        """
+        self._subscribe_config = subscribe_config
+        self._publish_config = publish_config
+        self._publisher: Publisher = None
+        self._subscribers: Dict[str, Subscriber] = {}
+        self._must_ensure_topics = must_ensure_topics
+
+        self._available_config = self._publish_config or self._subscribe_config
+
+    def _ensure_topics(self, *topics: str):
+        existing_topics = self.list_topics()
+
+        new_topics = [topic for topic in topics if topic not in existing_topics]
+
+        if not new_topics:  # Все используемые топики созданы
+            return
+
+        self.create_topics(*new_topics)
+
+    def _ensure_publisher(self):
+        if self._publish_config is None:
+            raise RuntimeError('Publisher is not configured')
+
+        if self._publisher is None:
+            self._publisher = Publisher(self._publish_config.dict(by_alias=True))
+
+        return self._publisher
+
+    def _make_subscriber_key(self, *topics: str) -> str:
+        return ','.join(sorted(topics))
+
+    def _ensure_subscriber(self, *topics: str) -> Subscriber:
+        if self._subscribe_config is None:
+            raise RuntimeError('Subscriber is not configured')
+
+        key = self._make_subscriber_key(*topics)
+
+        subscriber = self._subscribers.get(key)
+
+        if subscriber is None:
+            self._subscribers[key] = Subscriber(self._subscribe_config.dict(by_alias=True))
+        return self._subscribers[key]
+
+    def _should_continue_polling(
+        self, message: KafkaMessage, break_on_eof: bool, break_on_error: bool
+    ) -> Optional[bool]:
+        """Продолжать ли опрос, если возникла ошибка или был достигнут конец партиции.
+
+        :param message: Сообщение из Kafka.
+        :param break_on_eof: Останавливать опрос подписчика при исчерпании сообщений.
+        :param break_on_error: Останавливать опрос подписчика при ошибках.
+        :return: True если нужно продолжить опрос, False нужно прекратить опрос, None действия не требуются.
+        """
+        eof = message is None or message.error() == KafkaError._PARTITION_EOF  # pylint: disable=protected-access
+        if eof:
+            return not break_on_eof
+        elif error := message.error():
+            logger.error('Ошибка при получении сообщения: %s', error)
+            return not break_on_error
+        return None
+
+    def _poll_subscriber(
+        self,
+        subscriber: Subscriber,
+        break_on_eof: bool = False,
+        break_on_error: bool = False,
+    ) -> Generator[KafkaMessage, None, None]:
+        """Опрашивает подписчики и генерирует сообщения.
+
+        :param subscriber: Набор подписчиков для опроса.
+        :param break_on_eof: Останавливать опрос подписчика при исчерпании сообщений.
+        :param break_on_error: Останавливать опрос подписчика при ошибках.
+        :return: Генератор сообщений из Kafka.
+        """
+
+        while True:
+            message: KafkaMessage = subscriber.poll(1.0)
+
+            continue_polling = self._should_continue_polling(message, break_on_eof, break_on_error)
+            if continue_polling is True:
+                continue
+            elif continue_polling is False:
+                break
+
+            logger.info('Получено сообщение из %s', message.topic())
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug('Тело сообщения: %s', message.value())
+
+            yield message
+
+    def publish(self, topic: str, message: str, *args, **kwargs) -> None:  # pylint: disable=unused-argument
+        publisher = self._ensure_publisher()
+
+        if self._must_ensure_topics:
+            self._ensure_topics(topic)
+
+        publisher.poll(0)
+        publisher.produce(topic, message, callback=publish_callback)
+        publisher.flush()
+
+    def subscribe(self, *topics: str) -> Generator[KafkaMessage, None, None]:
+        subscriber: Subscriber = self._ensure_subscriber(*topics)
+
+        if self._must_ensure_topics:
+            self._ensure_topics(*topics)
+
+        subscriber.subscribe(list(topics))
+
+        yield from self._poll_subscriber(subscriber)
+
+    def list_topics(self, timeout=30.0):
+        admin = self._get_admin_client()
+        return list(admin.list_topics(timeout=timeout).topics.keys())
+
+    def create_topics(self, *topics: str, timeout=30.0):
+        admin = self._get_admin_client()
+        futures = admin.create_topics(
+            [
+                NewTopic(topic, num_partitions=1) for topic in topics
+            ],
+            request_timeout=timeout
+        )
+
+        for topic, future in futures.items():
+            try:
+                future.result()
+            except KafkaException as e:
+                error = e.args[0]
+                if error.code() in ('TOPIC_ALREADY_EXISTS', ):
+                    logger.warning('Топик уже существует')
+
+            except Exception:  # pylint: disable=broad-exception-caught
+                logger.exception('Невозможно создать topic')
+
+            else:
+                logger.info('Topic %s создан', topic)
+
+    def delete_topic(self, topic: str):
+        """Удалить единичный топик."""
+        return self.delete_topics(topic)[0]
+
+    def delete_topics(self, *topics: str):
+        """Удалить набор топиков."""
+        admin = self._get_admin_client()
+        return [
+            (topic, future.result())
+            for topic, future in admin.delete_topics(list(topics)).items()
+        ]
+
+    def _get_admin_client(self):
+        if self._available_config is None:
+            raise RuntimeError('Не указаны параметры подключения')
+
+        return AdminClient(self._available_config.dict(by_alias=True))

explicit/kafka/domain/__init__.py

@@ -0,0 +1,7 @@
+"""Модель предметной области адаптера к Apache Kafka."""
+from .model import AbstractAuth
+from .model import AuthDisabled
+from .model import BaseConfig
+from .model import PublishConfig
+from .model import SASLPlainAuth
+from .model import SubscribeConfig

explicit/kafka/domain/model/__init__.py

@@ -0,0 +1,7 @@
+"""Модель предметной области адаптера к Apache Kafka."""
+from .auth import AbstractAuth
+from .auth import AuthDisabled
+from .auth import SASLPlainAuth
+from .config import BaseConfig
+from .config import PublishConfig
+from .config import SubscribeConfig

explicit/kafka/domain/model/auth.py

@@ -0,0 +1,39 @@
+"""Параметры аутентификации."""
+from typing import Final
+from typing import Literal
+
+from pydantic.main import BaseModel
+
+
+SASL_PLAINTEXT: Final = 'SASL_PLAINTEXT'
+SASL_SSL: Final = 'SASL_SSL'
+
+SASL_PROTOCOLS = [SASL_PLAINTEXT, SASL_SSL]
+
+
+class AbstractAuth(BaseModel):
+    """Параметры аутентификации.
+
+    Может нести параметры по-умолчанию и доп. валидацию.
+    """
+
+
+class AuthDisabled(AbstractAuth):
+    """Аутентификация отключена."""
+
+
+class SASLPlainAuth(AbstractAuth):
+
+    sasl_username: str
+    sasl_password: str
+    sasl_mechanism: str = 'PLAIN'
+    security_protocol: Literal['SASL_PLAINTEXT'] = SASL_PLAINTEXT
+
+    class Config:
+        allow_population_by_field_name = True
+        fields = {
+            'sasl_username': 'sasl.username',
+            'sasl_password': 'sasl.password',
+            'sasl_mechanism': 'sasl.mechanism',
+            'security_protocol': 'security.protocol'
+        }

explicit/kafka/domain/model/config.py

@@ -0,0 +1,63 @@
+"""Набор конфигураций."""
+from typing import TYPE_CHECKING
+from typing import Dict
+from typing import Union
+
+from pydantic.fields import Field
+from pydantic.main import BaseModel
+from pydantic.utils import import_string
+
+from .auth import AbstractAuth
+from .auth import AuthDisabled
+
+
+if TYPE_CHECKING:
+    from pydantic.typing import AbstractSetIntStr
+
+
+class BaseConfig(BaseModel):
+    """Общие параметры издателя и подписчика."""
+
+    bootstrap__servers: str
+
+    auth: AbstractAuth = AuthDisabled()
+
+    class Config:
+        fields = {  # Aliases
+            'bootstrap__servers': 'bootstrap.servers',
+        }
+
+    def set_auth(self, auth: Dict) -> AbstractAuth:
+        auth_cls = import_string(auth['BACKEND'])
+        assert issubclass(auth_cls, AbstractAuth)
+        self.auth = auth_cls(**auth['OPTIONS'])
+        return self.auth
+
+    def dict(
+        self, *args, exclude: Union['AbstractSetIntStr', None] = None, **kwargs
+    ) -> dict:
+        """Возвращает пареметры, пригодные для передачи в низкоуровневый пакет взаимодействия."""
+        self_dict = super().dict(*args, exclude=(exclude or set()) | {'auth'}, **kwargs)
+        auth_dict = self.auth.dict(by_alias=True)
+        return self_dict | auth_dict
+
+
+class PublishConfig(BaseConfig):
+
+    class Config(BaseConfig.Config):
+        fields = {
+            **BaseConfig.Config.fields
+        }
+
+
+class SubscribeConfig(BaseConfig):
+
+    group__id: str
+    auto__offset__reset: str = Field('earliest', const=True)
+
+    class Config(BaseConfig.Config):
+        fields = {
+            'auto__offset__reset': 'auto.offset.reset',
+            'group__id': 'group.id',
+            **BaseConfig.Config.fields
+        }

explicit_python_kafka-1.1.1.dist-info/METADATA

@@ -0,0 +1,95 @@
+Metadata-Version: 2.1
+Name: explicit-python-kafka
+Version: 1.1.1
+Summary: Набор компонентов для интеграции explicit с kafka
+Author: BARS Group
+Author-email: education_dev@bars-open.ru
+License: MIT
+Classifier: Intended Audience :: Developers
+Classifier: Environment :: Web Environment
+Classifier: Natural Language :: Russian
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Development Status :: 5 - Production/Stable
+Description-Content-Type: text/markdown
+Requires-Dist: explicit <3,>=2
+Requires-Dist: confluent-kafka
+
+# Explicit-Kafka
+## Набор компонентов для интеграции explicit с kafka.
+
+Содержит реализацию адаптера обмена сообщениями через Kafka.
+
+### Пример использования
+Настройка адаптера
+```python
+# persons/core/apps.py
+from django.apps.config import AppConfig as AppConfigBase
+
+
+class AppConfig(AppConfigBase):
+
+    name = __package__
+    
+    def _setup_adapter(self):
+        from explicit.kafka.adapters.messaging import Adapter
+        from explicit.kafka.adapters.messaging import PublishConfig
+        from explicit.kafka.adapters.messaging import SubscribeConfig
+
+        from persons import core
+        
+        # конфигурация адаптера
+        adapter_base_config = {'bootstrap.servers': 'kafka:9092'}
+        publish_config = PublishConfig(adapter_base_config)
+        subscribe_config = SubscribeConfig(adapter_base_config | {'group.id': f'edu.persons'})
+    
+        adapter = Adapter(subscribe_config=subscribe_config, publish_config=publish_config)
+        core.adapter = adapter
+
+    def ready(self):
+        self._setup_adapter()
+```
+Отправка сообщений
+```python
+# persons/core/persons/services/handlers/events.py
+
+def on_person_created(
+    event: 'PersonCreated',
+    messaging_adapter: 'AbstractMessagingAdapter'
+):
+    messaging_adapter.publish('edu.persons.person', event.dump())
+```
+
+Подписка на сообщения
+```python
+# education/entrypoints/eventconsumer.py
+
+def bootstrap():
+    import json
+    
+    from education.core import adapter
+    from education.core import bus
+    from education.core.persons.domain.events import PersonCreated 
+    
+    TOPIC_EVENTS = {
+      'edu.persons.person': PersonCreated,
+    }
+    
+    for message in adapter.subscribe(*TOPIC_EVENTS):
+       for message in adapter.subscribe(*TOPIC_EVENTS):
+          event = TOPIC_EVENTS[message.topic()](
+              **json.loads(message.value())
+          )
+          bus.handle(event)
+
+bootstrap()
+```
+
+### Запуск тестов
+```sh
+$ tox
+```

explicit_python_kafka-1.1.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+explicit/kafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+explicit/kafka/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+explicit/kafka/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+explicit/kafka/adapters/messaging.py,sha256=V1DfOcaj_Wq9JfwzqmJIC24T-_XyaqOvnR0sLa2Jqg0,8063
+explicit/kafka/domain/__init__.py,sha256=mkYZMtbCNw90NUxGEusosWe5EQsA4iucdMLQmS6q7SI,284
+explicit/kafka/domain/model/__init__.py,sha256=BKA_dqvNOhOrUI7pUn3rYQO1X5YUijCmjyEHpuUkfp0,284
+explicit/kafka/domain/model/auth.py,sha256=h2-P-aDP3_6JLYM3e51wSJqhDBEwAKI2Rfo1uoSMHg0,1025
+explicit/kafka/domain/model/config.py,sha256=9HMY4ei180j6BEfR3w-D1FQsIMEh40S6I7Xw9Btrgrs,1808
+explicit_python_kafka-1.1.1.dist-info/METADATA,sha256=Ju_DUsZAO9iJ9cR1WmAGoyfW4VggvzdrW_1NBtW-UBo,2914
+explicit_python_kafka-1.1.1.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+explicit_python_kafka-1.1.1.dist-info/dependency_links.txt,sha256=z2Fi-sEElLMsunjPT4vpjRUYQ4oGu_tX_9NeLaEKlgo,44
+explicit_python_kafka-1.1.1.dist-info/top_level.txt,sha256=tt6T8l4Yji4ww87qZQcD4CbcwTIHy7NAPmU7QAfMcpY,9
+explicit_python_kafka-1.1.1.dist-info/RECORD,,

explicit_python_kafka-1.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (74.1.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

explicit_python_kafka-1.1.1.dist-info/dependency_links.txt

@@ -0,0 +1 @@
+https://pypi.bars-open.ru/simple/m3-builder

explicit_python_kafka-1.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+explicit