zrb 0.28.1__py3-none-any.whl → 1.0.0a2__py3-none-any.whl

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/kafka/publisher.py

@@ -1,166 +0,0 @@
-import asyncio
-import logging
-from typing import Any, Optional
-
-from aiokafka import AIOKafkaProducer
-from aiokafka.producer.producer import DefaultPartitioner, _missing
-from component.messagebus.kafka.admin import KafkaAdmin, must_get_kafka_admin
-from component.messagebus.messagebus import (
-    MessageSerializer,
-    Publisher,
-    must_get_message_serializer,
-)
-from pydantic import BaseModel
-
-
-class KafkaPublisher(Publisher):
-    def __init__(
-        self,
-        logger: logging.Logger,
-        bootstrap_servers: str = "localhost",
-        client_id: Optional[Any] = None,
-        metadata_max_age_ms=300000,
-        request_timeout_ms=40000,
-        api_version="auto",
-        acks=_missing,
-        key_serializer=None,
-        value_serializer=None,
-        compression_type=None,
-        max_batch_size=16384,
-        partitioner=DefaultPartitioner(),
-        max_request_size=1048576,
-        linger_ms=0,
-        send_backoff_ms=100,
-        retry_backoff_ms=100,
-        security_protocol="PLAINTEXT",
-        ssl_context=None,
-        connections_max_idle_ms=540000,
-        enable_idempotence=False,
-        transactional_id=None,
-        transaction_timeout_ms=60000,
-        sasl_mechanism="PLAIN",
-        sasl_plain_password=None,
-        sasl_plain_username=None,
-        sasl_kerberos_service_name="kafka",
-        sasl_kerberos_domain_name=None,
-        sasl_oauth_token_provider=None,
-        serializer: Optional[MessageSerializer] = None,
-        kafka_admin: Optional[KafkaAdmin] = None,
-        retry: int = 3,
-        retry_interval: int = 10,
-        identifier="kafka-publisher",
-    ):
-        self.logger = logger
-        self.serializer = must_get_message_serializer(serializer)
-        self.kafka_admin = must_get_kafka_admin(
-            logger=logger,
-            kafka_admin=kafka_admin,
-            bootstrap_servers=bootstrap_servers,
-            security_protocol=security_protocol,
-            sasl_mechanism=sasl_mechanism,
-            sasl_plain_password=sasl_plain_password,
-            sasl_plain_username=sasl_plain_username,
-            sasl_kerberos_service_name=sasl_kerberos_service_name,
-            sasl_kerberos_domain_name=sasl_kerberos_domain_name,
-            sasl_oauth_token_provider=sasl_oauth_token_provider,
-        )
-        self.producer: Optional[AIOKafkaProducer] = None
-        self.bootstrap_servers = bootstrap_servers
-        self.client_id = client_id
-        self.metadata_max_age_ms = metadata_max_age_ms
-        self.request_timeout_ms = request_timeout_ms
-        self.api_version = api_version
-        self.acks = acks
-        self.key_serializer = key_serializer
-        self.value_serializer = value_serializer
-        self.compression_type = compression_type
-        self.max_batch_size = max_batch_size
-        self.partitioner = partitioner
-        self.max_request_size = max_request_size
-        self.linger_ms = linger_ms
-        self.send_backoff_ms = send_backoff_ms
-        self.retry_backoff_ms = retry_backoff_ms
-        self.security_protocol = security_protocol
-        self.ssl_context = ssl_context
-        self.connections_max_idle_ms = connections_max_idle_ms
-        self.enable_idempotence = enable_idempotence
-        self.transactional_id = transactional_id
-        self.transaction_timeout_ms = transaction_timeout_ms
-        self.sasl_mechanism = sasl_mechanism
-        self.sasl_plain_password = sasl_plain_password
-        self.sasl_plain_username = sasl_plain_username
-        self.sasl_kerberos_service_name = sasl_kerberos_service_name
-        self.sasl_kerberos_domain_name = sasl_kerberos_domain_name
-        self.sasl_oauth_token_provider = sasl_oauth_token_provider
-        self.retry = retry
-        self.retry_interval = retry_interval
-        self.identifier = identifier
-
-    async def publish(self, event_name: str, message: Any):
-        await self.kafka_admin.create_events([event_name])
-        topic_name = self.kafka_admin.get_topic_name(event_name)
-        if isinstance(message, BaseModel):
-            message = message.model_dump()
-        for attempt in range(self.retry):
-            try:
-                await self._connect()
-                encoded_value = self.serializer.encode(event_name, message)
-                self.logger.info(
-                    f'🐼 [{self.identifier}] Publish to "{topic_name}": ' + f"{message}"
-                )
-                return await self.producer.send_and_wait(topic_name, encoded_value)
-            except (asyncio.CancelledError, GeneratorExit, Exception) as e:
-                self.logger.error(
-                    f"🐼 [{self.identifier}] Failed to publish message: {e}"
-                )
-                await self._disconnect()
-                await asyncio.sleep(self.retry_interval)
-                continue
-        self.logger.error(
-            f"🐼 [{self.identifier}] Failed to publish message after "
-            + f"{self.retry} attempts"
-        )
-        raise RuntimeError("Failed to publish message after retrying")
-
-    async def _connect(self):
-        self.logger.info(f"🐼 [{self.identifier}] Create kafka producer")
-        self.producer = AIOKafkaProducer(
-            bootstrap_servers=self.bootstrap_servers,
-            client_id=self.client_id,
-            metadata_max_age_ms=self.metadata_max_age_ms,
-            request_timeout_ms=self.request_timeout_ms,
-            api_version=self.api_version,
-            acks=self.acks,
-            key_serializer=self.key_serializer,
-            value_serializer=self.value_serializer,
-            compression_type=self.compression_type,
-            max_batch_size=self.max_batch_size,
-            partitioner=self.partitioner,
-            max_request_size=self.max_request_size,
-            linger_ms=self.linger_ms,
-            send_backoff_ms=self.send_backoff_ms,
-            retry_backoff_ms=self.retry_backoff_ms,
-            security_protocol=self.security_protocol,
-            ssl_context=self.ssl_context,
-            connections_max_idle_ms=self.connections_max_idle_ms,
-            enable_idempotence=self.enable_idempotence,
-            transactional_id=self.transactional_id,
-            transaction_timeout_ms=self.transaction_timeout_ms,
-            sasl_mechanism=self.sasl_mechanism,
-            sasl_plain_password=self.sasl_plain_password,
-            sasl_plain_username=self.sasl_plain_username,
-            sasl_kerberos_service_name=self.sasl_kerberos_service_name,
-            sasl_kerberos_domain_name=self.sasl_kerberos_domain_name,
-            sasl_oauth_token_provider=self.sasl_oauth_token_provider,
-        )
-        self.logger.info(f"🐼 [{self.identifier}] Start kafka producer")
-        await self.producer.start()
-        self.logger.info(f"🐼 [{self.identifier}] Kafka producer started")
-        return self
-
-    async def _disconnect(self):
-        self.logger.info(f"🐼 [{self.identifier}] Stop kafka producer")
-        if self.producer is not None:
-            await self.producer.stop()
-        self.logger.info(f"🐼 [{self.identifier}] Kafka producer stopped")
-        self.producer = None

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/messagebus.py

@@ -1,63 +0,0 @@
-from abc import ABC, abstractmethod
-from collections.abc import Callable, Mapping
-from typing import Any, Optional
-
-from component.serializer.serializer import JsonSerializer, Serializer
-
-TEventHandler = Callable[[Any], Any]
-
-
-class Admin(ABC):
-    @abstractmethod
-    async def create_events(self, event_names: list[str]):
-        pass
-
-    @abstractmethod
-    async def delete_events(self, event_names: list[str]):
-        pass
-
-
-class Publisher(ABC):
-    @abstractmethod
-    async def publish(self, event_name: str, message: Any):
-        pass
-
-
-class Consumer(ABC):
-    @abstractmethod
-    def register(self, event_name: str) -> Callable[[TEventHandler], Any]:
-        pass
-
-    @abstractmethod
-    async def start(self):
-        pass
-
-    @abstractmethod
-    async def stop(self):
-        pass
-
-
-class MessageSerializer:
-    def __init__(self, serializers: Optional[Mapping[str, Serializer]] = None):
-        serializers = serializers if serializers is not None else {}
-        self.serializers: Mapping[str, Serializer] = serializers
-        self.default_serializer = JsonSerializer()
-
-    def encode(self, event_name: str, message: Any) -> Any:
-        serializer = self._get_serializer(event_name)
-        return serializer.encode(message)
-
-    def decode(self, event_name: str, encoded_message: Any) -> Any:
-        serializer = self._get_serializer(event_name)
-        return serializer.decode(encoded_message)
-
-    def _get_serializer(self, event_name: str) -> Serializer:
-        return self.serializers.get(event_name, self.default_serializer)
-
-
-def must_get_message_serializer(
-    serializer: Optional[MessageSerializer] = None,
-) -> MessageSerializer:
-    if serializer is None:
-        return MessageSerializer()
-    return serializer

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/mock.py

@@ -1,67 +0,0 @@
-import asyncio
-import inspect
-import logging
-from collections.abc import Callable, Mapping
-from typing import Any
-
-from component.messagebus.messagebus import (
-    Admin,
-    Consumer,
-    MessageSerializer,
-    Publisher,
-    TEventHandler,
-)
-
-
-class MockConsumer(Consumer):
-    def __init__(self, logger: logging.Logger, serializer: MessageSerializer):
-        self.logger = logger
-        self.serializer = serializer
-        self._handlers: Mapping[str, TEventHandler] = {}
-
-    def register(self, event_name: str) -> Callable[[TEventHandler], Any]:
-        def wrapper(handler: TEventHandler):
-            self.logger.warning(f'🪵 Register handler for "{event_name}"')
-            self._handlers[event_name] = handler
-            return handler
-
-        return wrapper
-
-    async def handle(self, event_name: str, encoded_value: Any):
-        message_handler = self._handlers[event_name]
-        decoded_value = self.serializer.decode(event_name, encoded_value)
-        self.logger.info(f'🪵 Consume "{event_name}": {decoded_value}')
-        if inspect.iscoroutinefunction(message_handler):
-            return asyncio.create_task(message_handler(decoded_value))
-        return message_handler(decoded_value)
-
-    async def start(self):
-        return
-
-    async def stop(self):
-        return
-
-
-class MockPublisher(Publisher):
-    def __init__(
-        self,
-        logger: logging.Logger,
-        consumer: MockConsumer,
-        serializer: MessageSerializer,
-    ):
-        self.logger = logger
-        self.consumer = consumer
-        self.serializer = serializer
-
-    async def publish(self, event_name: str, message: Any):
-        encoded_value = self.serializer.encode(event_name, message)
-        self.logger.info(f'🪵 Publish "{event_name}": {message}')
-        await self.consumer.handle(event_name, encoded_value)
-
-
-class MockAdmin(Admin):
-    async def create_events(self, event_names: list[str]):
-        return
-
-    async def delete_events(self, event_names: list[str]):
-        return

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/rabbitmq/admin.py

@@ -1,146 +0,0 @@
-import asyncio
-import logging
-from collections.abc import Mapping
-from typing import Any, Optional
-
-import aiormq
-from component.messagebus.messagebus import Admin
-
-
-class RMQEventConfig:
-    def __init__(
-        self,
-        queue_name: str,
-        exchange_name: str = "",
-    ):
-        self.queue_name = queue_name
-        self.exchange_name = exchange_name
-
-
-class RMQAdmin(Admin):
-    def __init__(
-        self,
-        logger: logging.Logger,
-        configs: Mapping[str, RMQEventConfig],
-        connection_string: str,
-    ):
-        self.logger = logger
-        self.connection_string = connection_string
-        self.configs = configs
-        self._existing_events: Mapping[str, bool] = {}
-
-    async def create_events(self, event_names: list[str]):
-        # Only handle non-existing events
-        event_names = [
-            event_name
-            for event_name in event_names
-            if event_name not in self._existing_events
-        ]
-        if len(event_names) == 0:
-            return
-        try:
-            connection = await aiormq.connect(self.connection_string)
-            channel = await connection.channel()
-            for event_name in event_names:
-                config = self.get_config(event_name)
-                if config.exchange_name != "":
-                    await self._declare_fanned_out_exchange(channel, config)
-                    self._existing_events[event_name] = True
-                    continue
-                await self._declare_queue(channel, config)
-                self._existing_events[event_name] = True
-            await self._clean_up(connection, channel)
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(
-                " ".join(
-                    [
-                        "🐰 [rabbitmq-admin] Something wrong when ",
-                        f"creating events: {event_names}",
-                    ]
-                ),
-                exc_info=True,
-            )
-
-    async def delete_events(self, event_names: list[str]):
-        # Only handle existing events
-        event_names = [
-            event_name
-            for event_name in event_names
-            if event_name in self._existing_events
-        ]
-        if len(event_names) == 0:
-            return
-        try:
-            connection = await aiormq.connect(self.connection_string)
-            channel = await connection.channel()
-            for event_name in event_names:
-                config = self.get_config(event_name)
-                # delete the queue
-                await channel.queue_delete(queue=config.queue_name)
-                if config.exchange_name != "":
-                    # delete the exchange
-                    await channel.exchange_delete(exchange_name=config.exchange_name)
-                del self._existing_events[event_name]
-            await self._clean_up(connection, channel)
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(
-                " ".join(
-                    [
-                        "🐰 [rabbitmq-admin] Something wrong when ",
-                        f"deleting events: {event_names}",
-                    ]
-                ),
-                exc_info=True,
-            )
-
-    async def _clean_up(self, connection: aiormq.Connection, channel: aiormq.Channel):
-        try:
-            await channel.close()
-            await connection.close()
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error("🐰 [rabbitmq-admin]", exc_info=True)
-
-    async def _declare_fanned_out_exchange(
-        self, channel: aiormq.Channel, config: RMQEventConfig
-    ):
-        await channel.exchange_declare(
-            exchange=config.exchange_name,
-            exchange_type="fanout",
-            durable=False,
-            auto_delete=False,
-        )
-        # declare a queue
-        result = await self._declare_queue(channel, config)
-        queue_name = result["queue"]
-        # bind the queue to the exchange
-        await channel.queue_bind(
-            queue=queue_name, exchange=config.exchange_name, routing_key=""
-        )
-
-    async def _declare_queue(
-        self, channel: aiormq.Channel, config: RMQEventConfig
-    ) -> Any:
-        return await channel.queue_declare(
-            queue=config.queue_name, durable=True, auto_delete=False
-        )
-
-    def get_config(self, event_name: str) -> RMQEventConfig:
-        if event_name in self.configs:
-            return self.configs[event_name]
-        return RMQEventConfig(queue_name=event_name)
-
-    def get_exchange_name(self, event_name: str) -> str:
-        event_config = self.get_config(event_name)
-        return event_config.exchange_name
-
-    def get_queue_name(self, event_name: str) -> str:
-        event_config = self.get_config(event_name)
-        return event_config.queue_name
-
-
-def must_get_rmq_admin(
-    logger: logging.Logger, rmq_admin: Optional[RMQAdmin], connection_string: str
-) -> RMQAdmin:
-    if rmq_admin is None:
-        return RMQAdmin(logger=logger, configs={}, connection_string=connection_string)
-    return rmq_admin

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/rabbitmq/consumer.py

@@ -1,160 +0,0 @@
-import asyncio
-import inspect
-import logging
-from collections.abc import Callable, Mapping
-from typing import Any, Optional
-
-import aiormq
-from component.messagebus.messagebus import (
-    Consumer,
-    MessageSerializer,
-    TEventHandler,
-    must_get_message_serializer,
-)
-from component.messagebus.rabbitmq.admin import RMQAdmin, must_get_rmq_admin
-
-
-class RMQConsumer(Consumer):
-    def __init__(
-        self,
-        logger: logging.Logger,
-        connection_string: str,
-        serializer: Optional[MessageSerializer] = None,
-        rmq_admin: Optional[RMQAdmin] = None,
-        retry: int = 5,
-        retry_interval: int = 10,
-        prefetch_count: int = 20,
-        identifier="rmq-consumer",
-    ):
-        self.logger = logger
-        self.rmq_admin = must_get_rmq_admin(
-            logger=logger, rmq_admin=rmq_admin, connection_string=connection_string
-        )
-        self.connection_string = connection_string
-        self.connection: Optional[aiormq.Connection] = None
-        self.channel: Optional[aiormq.Channel] = None
-        self.serializer = must_get_message_serializer(serializer)
-        self.retry = retry
-        self.retry_interval = retry_interval
-        self.prefetch_count = prefetch_count
-        self._handlers: Mapping[str, TEventHandler] = {}
-        self._is_start_triggered = False
-        self._is_stop_triggered = False
-        self.identifier = identifier
-
-    def register(self, event_name: str) -> Callable[[TEventHandler], Any]:
-        def wrapper(handler: TEventHandler):
-            self.logger.warning(
-                f'🐰 [{self.identifier}] Register handler for "{event_name}"'
-            )
-            self._handlers[event_name] = handler
-            return handler
-
-        return wrapper
-
-    async def start(self):
-        if self._is_start_triggered:
-            return
-        self._is_start_triggered = True
-        return await self._start(self.retry)
-
-    async def stop(self):
-        if self._is_stop_triggered:
-            return
-        self._is_stop_triggered = True
-        await self._disconnect()
-
-    async def _start(self, retry: int):
-        try:
-            await self._connect()
-            event_names = list(self._handlers.keys())
-            await self.rmq_admin.create_events(event_names)
-            f'🐰 [{self.identifier}] listening from "{event_names}"'
-            for event_name in event_names:
-                queue_name = self.rmq_admin.get_queue_name(event_name)
-                on_message = self._create_consumer_callback(self.channel, event_name)
-                await self.channel.basic_consume(
-                    queue=queue_name, consumer_callback=on_message
-                )
-            retry = self.retry
-            while not self._is_stop_triggered:
-                await asyncio.sleep(0.01)
-                if not self._is_stop_triggered and (
-                    self.connection is None or self.connection.is_closed
-                ):
-                    raise Exception("Rabbitmq connection is closed")
-        except (asyncio.CancelledError, GeneratorExit, Exception) as e:
-            if retry > 0:
-                self.logger.error(f"🐰 [{self.identifier}]", exc_info=True)
-            if retry == 0:
-                self.logger.error(
-                    f"🐰 [{self.identifier}] Failed to consume message after "
-                    + f"{self.retry} attempts"
-                )
-                raise e
-            await self._disconnect()
-            await asyncio.sleep(self.retry_interval)
-            await self._start(retry - 1)
-        finally:
-            await self._disconnect()
-
-    async def _connect(self):
-        try:
-            connection_created = False
-            if self.connection is None or self.connection.is_closed:
-                self.logger.info(f"🐰 [{self.identifier}] Create consumer connection")
-                self.connection = await aiormq.connect(self.connection_string)
-                self.logger.info(f"🐰 [{self.identifier}] Consumer connection created")
-                connection_created = True
-            if connection_created or self.channel is None or self.channel.is_closed:
-                self.logger.info(f"🐰 [{self.identifier}] Get consumer channel")
-                self.channel = await self.connection.channel()
-                await self.channel.basic_qos(prefetch_count=self.prefetch_count)
-                self.logger.info(f"🐰 [{self.identifier}] Consumer channel created")
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(f"🐰 [{self.identifier}]", exc_info=True)
-            raise Exception("Cannot connect")
-
-    async def _disconnect(self):
-        try:
-            if self.channel is not None and not self.channel.is_closed:
-                self.logger.info(f"🐰 [{self.identifier}] Close consumer channel")
-                await self.channel.close()
-                self.logger.info(f"🐰 [{self.identifier}] Consumer channel closed")
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(f"🐰 [{self.identifier}]", exc_info=True)
-        try:
-            if self.connection is not None and not self.connection.is_closed:
-                self.logger.info(f"🐰 [{self.identifier}] Close consumer connection")
-                await self.connection.close()
-                self.logger.info(f"🐰 [{self.identifier}] Consumer connection closed")
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(f"🐰 [{self.identifier}]", exc_info=True)
-        self.connection = None
-        self.channel = None
-
-    def _create_consumer_callback(
-        self,
-        channel: aiormq.Channel,
-        event_name: str,
-    ) -> Callable[[Any], Any]:
-        async def on_message(message):
-            try:
-                decoded_value = self.serializer.decode(event_name, message.body)
-                handler = self._handlers.get(event_name)
-                queue_name = self.rmq_admin.get_queue_name(event_name)
-                self.logger.info(
-                    f'🐰 [{self.identifier}] Consume from "{queue_name}": '
-                    + f"{decoded_value}"
-                )
-                await self._run_handler(handler, decoded_value)
-                await channel.basic_ack(message.delivery_tag)
-            except (asyncio.CancelledError, GeneratorExit, Exception):
-                self.logger.error(f"🐰 [{self.identifier}]", exc_info=True)
-
-        return on_message
-
-    async def _run_handler(self, message_handler: TEventHandler, decoded_value: Any):
-        if inspect.iscoroutinefunction(message_handler):
-            return asyncio.create_task(message_handler(decoded_value))
-        return message_handler(decoded_value)