zrb 0.0.38__py3-none-any.whl → 0.0.40__py3-none-any.whl

zrb/builtin/generator/fastapp/template/src/kebab-app-name/deployment/__main__.py

@@ -0,0 +1,79 @@
+'''A Kubernetes Python Pulumi program to deploy kebab-app-name'''
+
+from dotenv import dotenv_values
+import pulumi
+import pulumi_kubernetes as k8s
+import os
+
+CURRENT_DIR = os.path.dirname(__file__)
+APP_DIR = os.path.abspath(os.path.join(CURRENT_DIR, '..', 'src'))
+TEMPLATE_ENV_FILE_NAME = os.path.join(APP_DIR, 'template.env')
+
+image = os.getenv('IMAGE', 'kebab-app-name:latest')
+replica = int(os.getenv('REPLICA', '1'))
+app_labels = {'app': 'kebab-app-name'}
+env_map = dotenv_values(TEMPLATE_ENV_FILE_NAME)
+app_port = int(os.getenv('APP_PORT', env_map.get('APP_PORT', '8080')))
+
+# Pulumi deployment docs:
+# https://www.pulumi.com/registry/packages/kubernetes/api-docs/apps/v1/deployment/
+deployment = k8s.apps.v1.Deployment(
+    resource_name='kebab-app-name',
+    spec=k8s.apps.v1.DeploymentSpecArgs(
+        selector=k8s.meta.v1.LabelSelectorArgs(match_labels=app_labels),
+        replicas=replica,
+        template=k8s.core.v1.PodTemplateSpecArgs(
+            metadata=k8s.meta.v1.ObjectMetaArgs(labels=app_labels),
+            spec=k8s.core.v1.PodSpecArgs(
+                containers=[
+                    k8s.core.v1.ContainerArgs(
+                        name='kebab-app-name',
+                        image=image,
+                        env=[
+                            k8s.core.v1.EnvVarArgs(
+                                name=env_name,
+                                value=os.getenv(env_name, default_value)
+                            )
+                            for env_name, default_value in env_map.items()
+                        ],
+                        ports=[
+                            k8s.core.v1.ContainerPortArgs(
+                                container_port=app_port
+                            )
+                        ],
+                        liveness_probe=k8s.core.v1.ProbeArgs(
+                            http_get=k8s.core.v1.HTTPGetActionArgs(
+                                port=app_port
+                            )
+                        ),
+                        readiness_probe=k8s.core.v1.ProbeArgs(
+                            http_get=k8s.core.v1.HTTPGetActionArgs(
+                                port=app_port
+                            )
+                        ),
+                    )
+                ]
+            )
+        )
+    )
+)
+
+# Pulumi services docs:
+# https://www.pulumi.com/registry/packages/kubernetes/api-docs/core/v1/service/
+service = k8s.core.v1.Service(
+    resource_name='kebab-app-name',
+    spec=k8s.core.v1.ServiceSpecArgs(
+        selector=app_labels,
+        ports=[
+            k8s.core.v1.ServicePortArgs(
+                port=80,
+                protocol="TCP",
+                target_port=app_port,
+            )
+        ],
+        type='LoadBalancer',
+    )
+)
+
+pulumi.export('deployment-name', deployment.metadata['name'])
+pulumi.export('service-name', service.metadata['name'])

zrb/builtin/generator/fastapp/template/src/kebab-app-name/deployment/requirements.txt

@@ -0,0 +1,3 @@
+pulumi>=3.0.0,<4.0.0
+pulumi-kubernetes>=3.0.0,<4.0.0
+python-dotenv==0.21.1

zrb/builtin/generator/fastapp/template/src/kebab-app-name/docker-compose.yml

@@ -0,0 +1,82 @@
+version: '3'
+services:
+
+
+  redpanda:
+    image: docker.redpanda.com/redpandadata/redpanda:v22.3.11
+    hostname: redpanda
+    command:
+      - redpanda start
+      - --smp 1
+      - --reserve-memory 0M
+      - --overprovisioned
+      - --node-id 1
+      - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+      - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
+      - --pandaproxy-addr PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082
+      - --advertise-pandaproxy-addr PLAINTEXT://redpanda:28082,OUTSIDE://localhost:8082
+      - --rpc-addr 0.0.0.0:33145
+      - --advertise-rpc-addr redpanda:33145
+    ports:
+      - 8082:8082
+      - ${KAFKA_OUTSIDE_HOST_PORT:-9092}:9092
+      - 28082:28082
+      - ${KAFKA_PLAINTEXT_HOST_PORT:-29092}:29092
+    restart: on-failure
+    profiles:
+      - kafka
+    healthcheck:
+      test: ["CMD", "redpanda", "admin", "check"]
+      interval: 5s
+      timeout: 1s
+      retries: 30
+
+
+  redpanda-console:
+    image: docker.redpanda.com/redpandadata/console:v2.2.2
+    hostname: redpanda-console
+    ports:
+      - ${REDPANDA_CONSOLE_HOST_PORT:-9000}:8080
+    environment:
+      - KAFKA_BROKERS=redpanda:29092
+    profiles:
+      - kafka
+    depends_on:
+      - redpanda
+    restart: on-failure
+
+
+  rabbitmq:
+    image: rabbitmq:3-management
+    hostname: rabbitmq
+    ports:
+      - "${RABBITMQ_HOST_PORT:-5672}:5672"
+      - "${RABBITMQ_MANAGEMENT_HOST_PORT:-15672}:15672"
+    # volumes:
+    #   - rabbitmq:/var/lib/rabbitmq/mnesia
+    environment:
+      - RABBITMQ_DEFAULT_USER=guest
+      - RABBITMQ_DEFAULT_PASS=guest
+    restart: on-failure
+    profiles:
+      - rabbitmq
+    healthcheck:
+      test: ["CMD", "rabbitmqctl", "status"]
+      interval: 5s
+      timeout: 1s
+      retries: 30
+
+
+  snake_app_name:
+    build: src
+    image: ${IMAGE:-kebab-app-name}
+    container_name: snake_app_name
+    hostname: snake_app_name
+    env_file: 'src/template.env'
+    environment:
+      APP_PORT: ${APP_PORT:-8080}
+    ports:
+      - "${APP_HOST_PORT:-8080}:${APP_PORT:-8080}"
+    restart: on-failure
+    profiles:
+      - monolith

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/.dockerignore

@@ -0,0 +1,3 @@
+.env
+__pycache__
+venv

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/.gitignore

@@ -0,0 +1,3 @@
+.env
+__pycache__
+venv

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.10-slim
+WORKDIR /home
+COPY . .
+RUN pip install -r requirements.txt
+CMD python main.py

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/app.py

@@ -0,0 +1,51 @@
+from fastapi import status
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI
+from config import (
+    cors_allow_credentials, cors_allow_headers, cors_allow_methods,
+    cors_allow_origin_regex, cors_allow_origins, cors_expose_headers,
+    cors_max_age
+)
+from component.app_state import app_state
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=cors_allow_origins,
+    allow_origin_regex=cors_allow_origin_regex,
+    allow_methods=cors_allow_methods,
+    allow_headers=cors_allow_headers,
+    allow_credentials=cors_allow_credentials,
+    expose_headers=cors_expose_headers,
+    max_age=cors_max_age,
+)
+
+
+@app.head('/liveness')
+@app.get('/liveness')
+def handle_liveness():
+    if app_state.get_liveness():
+        return JSONResponse(
+            content={'message': 'Service is alive'},
+            status_code=status.HTTP_200_OK
+        )
+    return JSONResponse(
+        content={'message': 'Service is not alive'},
+        status_code=status.HTTP_503_SERVICE_UNAVAILABLE
+    )
+
+
+@app.head('/readiness')
+@app.get('/readiness')
+def handle_readiness():
+    if app_state.get_readiness():
+        return JSONResponse(
+            content={'message': 'Service is ready'},
+            status_code=status.HTTP_200_OK
+        )
+    return JSONResponse(
+        content={'message': 'Service is not ready'},
+        status_code=status.HTTP_503_SERVICE_UNAVAILABLE
+    )

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/app_state.py

@@ -0,0 +1,26 @@
+class AppState():
+    __instance = None
+
+    def __new__(cls):
+        if cls.__instance is None:
+            cls.__instance = super().__new__(cls)
+        return cls.__instance
+
+    def __init__(self):
+        self.readiness: bool = False
+        self.liveness: bool = False
+
+    def set_liveness(self, value: bool):
+        self.liveness = value
+
+    def set_readiness(self, value: bool):
+        self.readiness = value
+
+    def get_liveness(self) -> bool:
+        return self.liveness
+
+    def get_readiness(self) -> bool:
+        return self.readiness
+
+
+app_state = AppState()

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/log.py

@@ -0,0 +1,20 @@
+from config import app_logging_level
+import logging
+
+# create logger
+logger = logging.getLogger('src')
+logger.setLevel(app_logging_level)
+
+ch = logging.StreamHandler()
+ch.setLevel(app_logging_level)
+
+# create formatter
+formatter = logging.Formatter(
+    '%(levelname)s:\t%(message)s'
+)
+
+# add formatter to ch
+ch.setFormatter(formatter)
+
+# add ch to logger
+logger.addHandler(ch)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/message_consumer.py

@@ -0,0 +1,42 @@
+from config import (
+    app_name, app_broker_type, app_rmq_connection, app_kafka_bootstrap_servers
+)
+from core.messagebus.messagebus import Consumer, MessageSerializer
+from core.messagebus.rabbitmq.consumer import RMQConsumeConnection, RMQConsumer
+from core.messagebus.kafka.consumer import (
+    KafkaConsumeConnection, KafkaConsumer
+)
+from component.message_mocker import mock_consumer
+from component.message_serializer import message_serializer
+from component.log import logger
+
+
+def init_consumer(
+    broker_type: str, serializer: MessageSerializer
+) -> Consumer:
+    if broker_type == 'rabbitmq':
+        consume_connection = RMQConsumeConnection(
+            logger=logger, connection_string=app_rmq_connection
+        )
+        return RMQConsumer(
+            logger=logger,
+            consume_connection=consume_connection,
+            serializer=serializer
+        )
+    if broker_type == 'kafka':
+        consume_connection = KafkaConsumeConnection(
+            logger=logger,
+            bootstrap_servers=app_kafka_bootstrap_servers,
+            group_id=app_name
+        )
+        return KafkaConsumer(
+            logger=logger,
+            consume_connection=consume_connection,
+            serializer=serializer
+        )
+    if broker_type == 'mock':
+        return mock_consumer
+    raise Exception(f'Invalid broker type: {broker_type}')
+
+
+consumer = init_consumer(app_broker_type, message_serializer)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/message_mocker.py

@@ -0,0 +1,6 @@
+from core.messagebus.mock import MockConsumer, MockPublisher
+from component.message_serializer import message_serializer
+from component.log import logger
+
+mock_consumer = MockConsumer(logger, message_serializer)
+mock_publisher = MockPublisher(logger, mock_consumer, message_serializer)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/message_publisher.py

@@ -0,0 +1,42 @@
+from config import (
+    app_broker_type, app_rmq_connection, app_kafka_bootstrap_servers
+)
+from core.messagebus.messagebus import Publisher, MessageSerializer
+from core.messagebus.kafka.publisher import (
+    KafkaPublishConnection, KafkaPublisher
+)
+from core.messagebus.rabbitmq.publisher import (
+    RMQPublishConnection, RMQPublisher
+)
+from component.message_mocker import mock_publisher
+from component.message_serializer import message_serializer
+from component.log import logger
+
+
+def init_publisher(
+    broker_type: str, serializer: MessageSerializer
+) -> Publisher:
+    if broker_type == 'rabbitmq':
+        publish_connection = RMQPublishConnection(
+            logger=logger, connection_string=app_rmq_connection
+        )
+        return RMQPublisher(
+            logger=logger,
+            publish_connection=publish_connection,
+            serializer=serializer
+        )
+    if broker_type == 'kafka':
+        publish_connection = KafkaPublishConnection(
+            logger=logger, bootstrap_servers=app_kafka_bootstrap_servers
+        )
+        return KafkaPublisher(
+            logger=logger,
+            publish_connection=publish_connection,
+            serializer=serializer
+        )
+    if broker_type == 'mock':
+        return mock_publisher
+    raise Exception(f'Invalid broker type: {broker_type}')
+
+
+publisher = init_publisher(app_broker_type, message_serializer)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/component/message_serializer.py

@@ -0,0 +1,9 @@
+from core.messagebus.messagebus import MessageSerializer
+
+
+def init_message_serializer() -> MessageSerializer:
+    # Add custom logic if necessary
+    return MessageSerializer()
+
+
+message_serializer = init_message_serializer()

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/config.py

@@ -0,0 +1,43 @@
+from typing import List
+from helper.conversion import str_to_boolean, str_to_logging_level
+import os
+import json
+
+app_name = os.environ.get('APP_NAME', 'app')
+app_logging_level = str_to_logging_level(
+    os.environ.get('APP_LOGGING_LEVEL', 'INFO')
+)
+app_broker_type = os.environ.get('APP_BROKER_TYPE', 'mock')
+app_host = os.environ.get('APP_HOST', '0.0.0.0')
+app_port = int(os.environ.get('APP_PORT', '8080'))
+app_reload = str_to_boolean(os.environ.get('APP_RELOAD', 'true'))
+
+app_rmq_connection = os.environ.get(
+    'APP_RMQ_CONNECTION', 'amqp://guest:guest@localhost/'
+)
+
+app_kafka_bootstrap_servers = os.environ.get(
+    'APP_KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'
+)
+
+cors_allow_origins: List[str] = json.loads(os.getenv(
+    'APP_CORS_ALLOW_ORIGINS', '["*"]'
+))
+cors_allow_origin_regex: str = os.getenv(
+    'APP_CORS_ALLOW_ORIGIN_REGEX', ''
+)
+cors_allow_methods: List[str] = json.loads(os.getenv(
+    'APP_CORS_ALLOW_METHODS', '["*"]'
+))
+cors_allow_headers: List[str] = json.loads(os.getenv(
+    'APP_CORS_ALLOW_HEADERS', '["*"]'
+))
+cors_allow_credentials: bool = str_to_boolean(os.getenv(
+    'APP_CORS_ALLOW_CREDENTIALS', 'false'
+))
+cors_expose_headers: bool = str_to_boolean(os.getenv(
+    'APP_CORS_EXPOSE_HEADERS', 'false'
+))
+cors_max_age: int = int(os.getenv(
+    'APP_CORS_MAX_AGE', '600'
+))

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/messagebus/kafka/consumer.py

@@ -0,0 +1,198 @@
+from typing import Any, Callable, Mapping, Optional
+from core.messagebus.messagebus import (
+    Consumer, THandler, MessageSerializer,
+    get_message_serializer
+)
+from aiokafka import AIOKafkaConsumer, __version__
+from aiokafka.consumer.consumer import RoundRobinPartitionAssignor
+
+import asyncio
+import inspect
+import logging
+
+
+class KafkaConsumeConnection():
+    def __init__(
+        self,
+        logger: logging.Logger,
+        bootstrap_servers: str,
+        client_id='aiokafka-' + __version__,
+        group_id: Optional[str] = None,
+        key_deserializer=None,
+        value_deserializer=None,
+        fetch_max_wait_ms=500,
+        fetch_max_bytes=52428800,
+        fetch_min_bytes=1,
+        max_partition_fetch_bytes=1 * 1024 * 1024,
+        request_timeout_ms=40 * 1000,
+        retry_backoff_ms=100,
+        auto_offset_reset='latest',
+        enable_auto_commit=True,
+        auto_commit_interval_ms=5000,
+        check_crcs=True,
+        metadata_max_age_ms=5 * 60 * 1000,
+        partition_assignment_strategy=(RoundRobinPartitionAssignor,),
+        max_poll_interval_ms=300000,
+        rebalance_timeout_ms=None,
+        session_timeout_ms=10000,
+        heartbeat_interval_ms=3000,
+        consumer_timeout_ms=200,
+        max_poll_records=None,
+        ssl_context=None,
+        security_protocol='PLAINTEXT',
+        api_version='auto',
+        exclude_internal_topics=True,
+        connections_max_idle_ms=540000,
+        isolation_level="read_uncommitted",
+        sasl_mechanism="PLAIN",
+        sasl_plain_password=None,
+        sasl_plain_username=None,
+        sasl_kerberos_service_name='kafka',
+        sasl_kerberos_domain_name=None,
+        sasl_oauth_token_provider=None
+    ):
+        self.logger = logger
+        self.consumer: Optional[AIOKafkaConsumer] = None
+        self.bootstrap_servers = bootstrap_servers
+        self.client_id = client_id
+        self.group_id = group_id
+        self.key_deserializer = key_deserializer
+        self.value_deserializer = value_deserializer
+        self.fetch_max_wait_ms = fetch_max_wait_ms
+        self.fetch_max_bytes = fetch_max_bytes
+        self.fetch_min_bytes = fetch_min_bytes
+        self.max_partition_fetch_bytes = max_partition_fetch_bytes
+        self.request_timeout_ms = request_timeout_ms
+        self.retry_backoff_ms = retry_backoff_ms
+        self.auto_offset_reset = auto_offset_reset
+        self.enable_auto_commit = enable_auto_commit
+        self.auto_commit_interval_ms = auto_commit_interval_ms
+        self.check_crcs = check_crcs
+        self.metadata_max_age_ms = metadata_max_age_ms
+        self.partition_assignment_strategy = partition_assignment_strategy
+        self.max_poll_interval_ms = max_poll_interval_ms
+        self.rebalance_timeout_ms = rebalance_timeout_ms
+        self.session_timeout_ms = session_timeout_ms
+        self.heartbeat_interval_ms = heartbeat_interval_ms
+        self.consumer_timeout_ms = consumer_timeout_ms
+        self.max_poll_records = max_poll_records
+        self.ssl_context = ssl_context
+        self.security_protocol = security_protocol
+        self.api_version = api_version
+        self.exclude_internal_topics = exclude_internal_topics
+        self.connections_max_idle_ms = connections_max_idle_ms
+        self.isolation_level = isolation_level
+        self.sasl_mechanism = sasl_mechanism
+        self.sasl_plain_password = sasl_plain_password
+        self.sasl_plain_username = sasl_plain_username
+        self.sasl_kerberos_service_name = sasl_kerberos_service_name
+        self.sasl_kerberos_domain_name = sasl_kerberos_domain_name
+        self.sasl_oauth_token_provider = sasl_oauth_token_provider
+
+    async def __aenter__(self):
+        self.logger.info('🐼 Create kafka consumer')
+        self.consumer = AIOKafkaConsumer(
+            bootstrap_servers=self.bootstrap_servers,
+            client_id=self.client_id,
+            group_id=self.group_id,
+            key_deserializer=self.key_deserializer,
+            value_deserializer=self.value_deserializer,
+            fetch_max_wait_ms=self.fetch_max_wait_ms,
+            fetch_max_bytes=self.fetch_max_bytes,
+            fetch_min_bytes=self.fetch_min_bytes,
+            max_partition_fetch_bytes=self.max_partition_fetch_bytes,
+            request_timeout_ms=self.request_timeout_ms,
+            retry_backoff_ms=self.retry_backoff_ms,
+            auto_offset_reset=self.auto_offset_reset,
+            enable_auto_commit=self.enable_auto_commit,
+            auto_commit_interval_ms=self.auto_commit_interval_ms,
+            check_crcs=self.check_crcs,
+            metadata_max_age_ms=self.metadata_max_age_ms,
+            partition_assignment_strategy=self.partition_assignment_strategy,
+            max_poll_interval_ms=self.max_poll_interval_ms,
+            rebalance_timeout_ms=self.rebalance_timeout_ms,
+            session_timeout_ms=self.session_timeout_ms,
+            heartbeat_interval_ms=self.heartbeat_interval_ms,
+            consumer_timeout_ms=self.consumer_timeout_ms,
+            max_poll_records=self.max_poll_records,
+            ssl_context=self.ssl_context,
+            security_protocol=self.security_protocol,
+            api_version=self.api_version,
+            exclude_internal_topics=self.exclude_internal_topics,
+            connections_max_idle_ms=self.connections_max_idle_ms,
+            isolation_level=self.isolation_level,
+            sasl_mechanism=self.sasl_mechanism,
+            sasl_plain_password=self.sasl_plain_password,
+            sasl_plain_username=self.sasl_plain_username,
+            sasl_kerberos_service_name=self.sasl_kerberos_service_name,
+            sasl_kerberos_domain_name=self.sasl_kerberos_domain_name,
+            sasl_oauth_token_provider=self.sasl_oauth_token_provider,
+        )
+        self.logger.info('🐼 Start kafka consumer')
+        await self.consumer.start()
+        self.logger.info('🐼 Kafka consumer started')
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self.logger.info('🐼 Unsubscribe kafka consumer from all topics')
+        self.consumer.unsubscribe()
+        self.logger.info('🐼 Stop kafka consumer')
+        await self.consumer.stop()
+        self.logger.info('🐼 Kafka consumer stopped')
+
+
+class KafkaConsumer(Consumer):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        consume_connection: KafkaConsumeConnection,
+        serializer: Optional[MessageSerializer] = None,
+        retry: int = 5
+    ):
+        self.logger = logger
+        self.serializer = get_message_serializer(serializer)
+        self.conn = consume_connection
+        self._handlers: Mapping[str, THandler] = {}
+        self._retry = retry
+
+    def register(self, event_name: str) -> Callable[[THandler], Any]:
+        def wrapper(handler: THandler):
+            self.logger.warning(f'🐼 Register handler for "{event_name}"')
+            self._handlers[event_name] = handler
+            return handler
+        return wrapper
+
+    async def run(self):
+        return await self._run(self._retry)
+
+    async def _run(self, retry: int):
+        try:
+            async with self.conn as conn:
+                consumer: AIOKafkaConsumer = conn.consumer
+                topics = list(self._handlers.keys())
+                self.logger.warning(f'🐼 Subscribe to topics: {topics}')
+                consumer.subscribe(topics=topics)
+                async for message in consumer:
+                    event_name = message.topic
+                    message_handler = self._handlers.get(event_name)
+                    decoded_value = self.serializer.decode(
+                        event_name, message.value
+                    )
+                    self.logger.info(
+                        f'🐼 Consume "{event_name}": {decoded_value}'
+                    )
+                    await self._run_handler(message_handler, decoded_value)
+                retry = self._retry
+        except Exception:
+            if retry == 0:
+                self.logger.fatal('🐼 Cannot retry')
+                raise
+            self.logger.warning('🐼 Retry to consume')
+            await self._run(retry-1)
+
+    async def _run_handler(
+        self, message_handler: THandler, decoded_value: Any
+    ):
+        if inspect.iscoroutinefunction(message_handler):
+            return asyncio.create_task(message_handler(decoded_value))
+        return message_handler(decoded_value)