zrb 0.0.38__py3-none-any.whl → 0.0.40__py3-none-any.whl

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/messagebus/kafka/publisher.py

@@ -0,0 +1,144 @@
+from typing import Any, Optional
+from core.messagebus.messagebus import (
+    Publisher, MessageSerializer, get_message_serializer
+)
+from aiokafka import AIOKafkaProducer
+from aiokafka.producer.producer import _missing, DefaultPartitioner
+
+import logging
+
+
+class KafkaPublishConnection():
+    def __init__(
+        self,
+        logger: logging.Logger,
+        bootstrap_servers: str = 'localhost',
+        client_id: Optional[Any] = None,
+        metadata_max_age_ms=300000,
+        request_timeout_ms=40000,
+        api_version='auto',
+        acks=_missing,
+        key_serializer=None,
+        value_serializer=None,
+        compression_type=None,
+        max_batch_size=16384,
+        partitioner=DefaultPartitioner(),
+        max_request_size=1048576,
+        linger_ms=0,
+        send_backoff_ms=100,
+        retry_backoff_ms=100,
+        security_protocol="PLAINTEXT",
+        ssl_context=None,
+        connections_max_idle_ms=540000,
+        enable_idempotence=False,
+        transactional_id=None,
+        transaction_timeout_ms=60000,
+        sasl_mechanism="PLAIN",
+        sasl_plain_password=None,
+        sasl_plain_username=None,
+        sasl_kerberos_service_name='kafka',
+        sasl_kerberos_domain_name=None,
+        sasl_oauth_token_provider=None
+    ):
+        self.logger = logger
+        self.producer: Optional[AIOKafkaProducer] = None
+        self.bootstrap_servers = bootstrap_servers
+        self.client_id = client_id
+        self.metadata_max_age_ms = metadata_max_age_ms
+        self.request_timeout_ms = request_timeout_ms
+        self.api_version = api_version
+        self.acks = acks
+        self.key_serializer = key_serializer
+        self.value_serializer = value_serializer
+        self.compression_type = compression_type
+        self.max_batch_size = max_batch_size
+        self.partitioner = partitioner
+        self.max_request_size = max_request_size
+        self.linger_ms = linger_ms
+        self.send_backoff_ms = send_backoff_ms
+        self.retry_backoff_ms = retry_backoff_ms
+        self.security_protocol = security_protocol
+        self.ssl_context = ssl_context
+        self.connections_max_idle_ms = connections_max_idle_ms
+        self.enable_idempotence = enable_idempotence
+        self.transactional_id = transactional_id
+        self.transaction_timeout_ms = transaction_timeout_ms
+        self.sasl_mechanism = sasl_mechanism
+        self.sasl_plain_password = sasl_plain_password
+        self.sasl_plain_username = sasl_plain_username
+        self.sasl_kerberos_service_name = sasl_kerberos_service_name
+        self.sasl_kerberos_domain_name = sasl_kerberos_domain_name
+        self.sasl_oauth_token_provider = sasl_oauth_token_provider
+
+    async def __aenter__(self):
+        self.logger.info('🐼 Create kafka producer')
+        self.producer = AIOKafkaProducer(
+            bootstrap_servers=self.bootstrap_servers,
+            client_id=self.client_id,
+            metadata_max_age_ms=self.metadata_max_age_ms,
+            request_timeout_ms=self.request_timeout_ms,
+            api_version=self.api_version,
+            acks=self.acks,
+            key_serializer=self.key_serializer,
+            value_serializer=self.value_serializer,
+            compression_type=self.compression_type,
+            max_batch_size=self.max_batch_size,
+            partitioner=self.partitioner,
+            max_request_size=self.max_request_size,
+            linger_ms=self.linger_ms,
+            send_backoff_ms=self.send_backoff_ms,
+            retry_backoff_ms=self.retry_backoff_ms,
+            security_protocol=self.security_protocol,
+            ssl_context=self.ssl_context,
+            connections_max_idle_ms=self.connections_max_idle_ms,
+            enable_idempotence=self.enable_idempotence,
+            transactional_id=self.transactional_id,
+            transaction_timeout_ms=self.transaction_timeout_ms,
+            sasl_mechanism=self.sasl_mechanism,
+            sasl_plain_password=self.sasl_plain_password,
+            sasl_plain_username=self.sasl_plain_username,
+            sasl_kerberos_service_name=self.sasl_kerberos_service_name,
+            sasl_kerberos_domain_name=self.sasl_kerberos_domain_name,
+            sasl_oauth_token_provider=self.sasl_oauth_token_provider,
+        )
+        self.logger.info('🐼 Start kafka producer')
+        await self.producer.start()
+        self.logger.info('🐼 Kafka producer started')
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self.logger.info('🐼 Stop kafka producer')
+        await self.producer.stop()
+        self.logger.info('🐼 Kafka producer stopped')
+
+
+class KafkaPublisher(Publisher):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        publish_connection: KafkaPublishConnection,
+        serializer: Optional[MessageSerializer] = None,
+        retry: int = 3
+    ):
+        self.logger = logger
+        self.serializer = get_message_serializer(serializer)
+        self.conn = publish_connection
+        self._retry = retry
+
+    async def publish(self, event_name: str, message: Any):
+        return await self._publish(event_name, message, self._retry)
+
+    async def _publish(self, event_name: str, message: Any, retry: int):
+        try:
+            async with self.conn as conn:
+                producer: AIOKafkaProducer = conn.producer
+                encoded_value = self.serializer.encode(event_name, message)
+                self.logger.info(
+                    f'🐼 Publish "{event_name}": {message}'
+                )
+                await producer.send_and_wait(event_name, encoded_value)
+                retry = self._retry
+        except Exception:
+            if retry == 0:
+                raise
+            await self._publish(event_name, message, retry-1)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/messagebus/messagebus.py

@@ -0,0 +1,52 @@
+from typing import Any, Callable, Mapping, Optional
+from abc import ABC, abstractmethod
+from core.serializer.serializer import Serializer, JsonSerializer
+
+THandler = Callable[[Any], Any]
+
+
+class Publisher(ABC):
+    @abstractmethod
+    def publish(self, event_name: str, message: Any):
+        pass
+
+
+class Consumer(ABC):
+    @abstractmethod
+    def register(self, event_name: str) -> Callable[[THandler], Any]:
+        pass
+
+    @abstractmethod
+    def run(self):
+        pass
+
+
+class MessageSerializer():
+    def __init__(
+        self,
+        serializers: Optional[Mapping[str, Serializer]] = None
+    ):
+        serializers = serializers if serializers is not None else {}
+        self.serializers: Mapping[str, Serializer] = serializers
+        self.default_serializer = JsonSerializer()
+
+    def encode(self, event_name: str, message: Any) -> Any:
+        serializer = self._get_serializer(event_name)
+        return serializer.encode(message)
+
+    def decode(self, event_name: str, encoded_message: Any) -> Any:
+        serializer = self._get_serializer(event_name)
+        return serializer.decode(encoded_message)
+
+    def _get_serializer(self, event_name: str) -> Serializer:
+        return self.serializers.get(
+            event_name, self.default_serializer
+        )
+
+
+def get_message_serializer(
+    serializer: Optional[MessageSerializer] = None
+) -> MessageSerializer:
+    if serializer is None:
+        return MessageSerializer()
+    return serializer

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/messagebus/mock.py

@@ -0,0 +1,51 @@
+from typing import Any, Callable, Mapping
+from core.messagebus.messagebus import (
+    Publisher, Consumer, MessageSerializer, THandler
+)
+import asyncio
+import inspect
+import logging
+
+
+class MockConsumer(Consumer):
+    def __init__(
+        self, logger: logging.Logger, serializer: MessageSerializer
+    ):
+        self.logger = logger
+        self.serializer = serializer
+        self._handlers: Mapping[str, THandler] = {}
+
+    def register(self, event_name: str) -> Callable[[THandler], Any]:
+        def wrapper(handler: THandler):
+            self.logger.warning(f'🪵 Register handler for "{event_name}"')
+            self._handlers[event_name] = handler
+            return handler
+        return wrapper
+
+    async def handle(self, event_name: str, encoded_value: Any):
+        message_handler = self._handlers[event_name]
+        decoded_value = self.serializer.decode(event_name, encoded_value)
+        self.logger.info(f'🪵 Consume "{event_name}": {decoded_value}')
+        if inspect.iscoroutinefunction(message_handler):
+            return asyncio.create_task(message_handler(decoded_value))
+        return message_handler(decoded_value)
+
+    async def run(self):
+        return
+
+
+class MockPublisher(Publisher):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        consumer: MockConsumer,
+        serializer: MessageSerializer
+    ):
+        self.logger = logger
+        self.consumer = consumer
+        self.serializer = serializer
+
+    async def publish(self, event_name: str, message: Any):
+        encoded_value = self.serializer.encode(event_name, message)
+        self.logger.info(f'🪵 Publish "{event_name}": {message}')
+        await self.consumer.handle(event_name, encoded_value)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/messagebus/rabbitmq/consumer.py

@@ -0,0 +1,92 @@
+from typing import Any, Callable, Mapping, Optional
+from core.messagebus.messagebus import (
+    Consumer, THandler, MessageSerializer, get_message_serializer
+)
+import asyncio
+import aiormq
+import inspect
+import logging
+
+
+class RMQConsumeConnection():
+    def __init__(self, logger: logging.Logger, connection_string: str):
+        self.logger = logger
+        self.connection_string = connection_string
+        self.connection: Optional[aiormq.Connection] = None
+
+    async def __aenter__(self):
+        self.logger.info('🐰 Create consumer connection')
+        self.connection = await aiormq.connect(self.connection_string)
+        self.logger.info('🐰 Consumer connection created')
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self.logger.info('🐰 Close consumer connection')
+        await self.connection.close()
+        self.logger.info('🐰 Consumer connection closed')
+
+
+class RMQConsumer(Consumer):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        consume_connection: RMQConsumeConnection,
+        serializer: Optional[MessageSerializer] = None,
+        retry: int = 5
+    ):
+        self.logger = logger
+        self.conn = consume_connection
+        self._handlers: Mapping[str, THandler] = {}
+        self.serializer = get_message_serializer(serializer)
+        self._retry = retry
+
+    def register(self, event_name: str) -> Callable[[THandler], Any]:
+        def wrapper(handler: THandler):
+            self.logger.warning(f'🐰 Register handler for "{event_name}"')
+            self._handlers[event_name] = handler
+            return handler
+        return wrapper
+
+    async def run(self):
+        return await self._run(self._retry)
+
+    async def _run(self, retry: int):
+        try:
+            async with self.conn as conn:
+                connection: aiormq.Connection = conn.connection
+                self.logger.info('🐰 Get channel')
+                channel = await connection.channel()
+                for event_name, handler in self._handlers.items():
+                    self.logger.info(f'🐰 Declare queue: {event_name}')
+                    await channel.queue_declare(event_name)
+                    on_message = self._create_consumer_callback(
+                        channel, event_name
+                    )
+                    asyncio.create_task(channel.basic_consume(
+                        queue=event_name, consumer_callback=on_message
+                    ))
+            retry = self._retry
+        except Exception:
+            if retry == 0:
+                raise
+            await self._run(retry-1)
+
+    def _create_consumer_callback(
+        self,
+        channel: aiormq.Channel,
+        event_name: str,
+    ) -> Callable[[Any], Any]:
+        async def on_message(message):
+            decoded_value = self.serializer.decode(event_name, message.body)
+            handler = self._handlers.get(event_name)
+            self.logger.info(f'🐰 Consume "{event_name}": {decoded_value}')
+            await self._run_handler(handler, decoded_value)
+            await channel.basic_ack(message.delivery_tag)
+        return on_message
+
+    async def _run_handler(
+        self, message_handler: THandler, decoded_value: Any
+    ):
+        if inspect.iscoroutinefunction(message_handler):
+            return asyncio.create_task(message_handler(decoded_value))
+        return message_handler(decoded_value)

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/messagebus/rabbitmq/publisher.py

@@ -0,0 +1,61 @@
+from typing import Any, Optional
+from core.messagebus.messagebus import (
+    Publisher,  MessageSerializer, get_message_serializer
+)
+import aiormq
+import logging
+
+
+class RMQPublishConnection():
+    def __init__(self, logger: logging.Logger, connection_string: str):
+        self.logger = logger
+        self.connection_string = connection_string
+        self.connection: Optional[aiormq.Connection] = None
+
+    async def __aenter__(self):
+        self.logger.info('🐰 Create publisher connection')
+        self.connection = await aiormq.connect(self.connection_string)
+        self.logger.info('🐰 Publisher connection created')
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self.logger.info('🐰 Close publisher connection')
+        await self.connection.close()
+        self.logger.info('🐰 Publisher connection closed')
+
+
+class RMQPublisher(Publisher):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        publish_connection: RMQPublishConnection,
+        serializer: Optional[MessageSerializer] = None,
+        retry: int = 5
+    ):
+        self.logger = logger
+        self.serializer = get_message_serializer(serializer)
+        self.conn = publish_connection
+        self._retry = retry
+
+    async def publish(self, event_name: str, message: Any):
+        return await self._publish(event_name, message, self._retry)
+
+    async def _publish(self, event_name: str, message: Any, retry: int):
+        try:
+            async with self.conn as conn:
+                connection: aiormq.Connection = conn.connection
+                self.logger.info('🐰 Get channel')
+                channel = await connection.channel()
+                self.logger.info(f'🐰 Declare queue: {event_name}')
+                await channel.queue_declare(event_name)
+                self.logger.info(f'🐰 Publish "{event_name}": {message}')
+                await channel.basic_publish(
+                    body=self.serializer.encode(event_name, message),
+                    routing_key=event_name,
+                )
+            retry = self._retry
+        except Exception:
+            if retry == 0:
+                raise
+            await self._publish(event_name, message, retry-1)
+

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/core/serializer/serializer.py

@@ -0,0 +1,35 @@
+from typing import Any, Callable
+from abc import ABC, abstractmethod
+import jsons
+
+
+class Serializer(ABC):
+    @abstractmethod
+    def encode(self, message: Any) -> Any:
+        pass
+
+    @abstractmethod
+    def decode(self, message: Any) -> Any:
+        pass
+
+
+class CustomSerializer(Serializer):
+    def __init__(
+        self, encoder: Callable[[Any], Any], decoder: Callable[[Any], Any]
+    ):
+        self.encoder = encoder
+        self.decoder = decoder
+
+    def encode(self, message: Any) -> Any:
+        return self.encoder(message)
+
+    def decode(self, encoded_message: Any) -> Any:
+        return self.decoder(encoded_message)
+
+
+class JsonSerializer(Serializer):
+    def encode(self, message: Any) -> Any:
+        return jsons.dumps(message).encode()
+
+    def decode(self, encoded_message: Any) -> Any:
+        return jsons.loads(encoded_message.decode())

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/helper/async_task.py

@@ -0,0 +1,17 @@
+from typing import Awaitable, Callable
+import asyncio
+import inspect
+
+
+def create_task(
+    awaitable: Awaitable,
+    on_error: Callable
+) -> asyncio.Task:
+    async def critical_task(awaitable):
+        try:
+            return await awaitable
+        except Exception as exception:
+            if inspect.iscoroutinefunction(on_error):
+                return await on_error(exception)
+            return on_error(exception)
+    return asyncio.create_task(critical_task(awaitable))

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/helper/conversion.py

@@ -0,0 +1,27 @@
+import logging
+
+LOGGING_LEVEL_MAP = {
+    'critical': logging.CRITICAL,
+    'fatal': logging.FATAL,
+    'error': logging.ERROR,
+    'warning': logging.WARNING,
+    'warn': logging.WARN,
+    'info': logging.INFO,
+    'debug': logging.DEBUG,
+    'notset': logging.NOTSET,
+}
+
+
+def str_to_boolean(value: str):
+    if value.lower() in ('0', 'false', 'no', 'n'):
+        return False
+    if value.lower() in ('1', 'true', 'yes', 'y'):
+        return True
+    raise Exception(f'Cannot convert to boolean: "{value}"')
+
+
+def str_to_logging_level(logging_level_str: str) -> int:
+    lower_logging_level_str = logging_level_str.lower()
+    if lower_logging_level_str in LOGGING_LEVEL_MAP:
+        return LOGGING_LEVEL_MAP[lower_logging_level_str]
+    return logging.WARNING

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/main.py

@@ -0,0 +1,38 @@
+from component.app import app
+from component.app_state import app_state
+from component.log import logger
+from component.message_consumer import consumer
+from component.message_publisher import publisher
+from helper.async_task import create_task
+
+
+app_state.set_liveness(True)
+messages = []
+
+
+async def on_error(exception: Exception):
+    logger.critical(exception)
+    app_state.set_readiness(False)
+
+
+@app.on_event('startup')
+async def startup_event():
+    logger.info('Started')
+    create_task(consumer.run(), on_error=on_error)
+    app_state.set_readiness(True)
+
+
+@consumer.register('coba')
+async def handle_event(message):
+    messages.append(message)
+    print(messages)
+
+
+@app.get('/')
+def handle_get():
+    return ('hello world')
+
+
+@app.get('/send')
+async def handle_send():
+    return await publisher.publish('coba', 'sesuatu')

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/requirements.txt

@@ -0,0 +1,8 @@
+zrb==0.0.38
+aiormq==6.7.3
+aiokafka==0.8.0
+fastapi==0.93.0
+uvicorn==0.20.0
+fastapi==0.93.0
+jsons==1.6.3
+pydantic==1.10.6

zrb/builtin/generator/fastapp/template/src/kebab-app-name/src/template.env

@@ -0,0 +1,17 @@
+APP_NAME=fastapp
+APP_LOGGING_LEVEL=info
+APP_HOST=0.0.0.0
+APP_PORT=8080
+APP_RELOAD=true
+
+APP_BROKER_TYPE=rabbitmq
+APP_RMQ_CONNECTION=amqp://guest:guest@localhost/
+APP_KAFKA_BOOTSTRAP_SERVERS=localhost:9092
+
+APP_CORS_ALLOW_ORIGINS='["*"]'
+APP_CORS_ALLOW_ORIGIN_REGEX=''
+APP_CORS_ALLOW_METHODS='["*"]'
+APP_CORS_ALLOW_HEADERS='["*"]'
+APP_CORS_ALLOW_CREDENTIALS=0
+APP_CORS_EXPOSE_HEADERS=0
+APP_CORS_MAX_AGE=600

zrb/builtin/generator/simple_python_app/template/_automate/snake_app_name/cmd/start.sh

@@ -5,9 +5,11 @@ then
   echo "Init venv"
   python -m venv venv
 fi
+echo "Activate venv"
+source venv/bin/activate
 
 echo "Install packages"
 pip install -r requirements.txt
 
 echo "Start app"
-python main.py
+uvicorn main:app --host {{env.get("APP_HOST", "0.0.0.0")}} --port {{env.get("APP_PORT", "8080")}}

zrb/builtin/generator/simple_python_app/template/_automate/snake_app_name/container.py

@@ -51,6 +51,7 @@ start_snake_app_name_container = DockerComposeTask(
     envs=compose_envs,
     checkers=[
         HTTPChecker(
+            name='check-kebab-app-name',
             host='{{input.snake_app_name_host}}',
             port='{{env.HOST_PORT}}',
             is_https='{{input.snake_app_name_https}}'

zrb/builtin/generator/simple_python_app/template/_automate/snake_app_name/local.py

@@ -31,6 +31,7 @@ start_snake_app_name = CmdTask(
     cmd_path=os.path.join(CURRENT_DIR, 'cmd', 'start.sh'),
     checkers=[
         HTTPChecker(
+            name='check-kebab-app-name',
             host='{{input.snake_app_name_host}}',
             port='{{env.APP_PORT}}',
             is_https='{{input.snake_app_name_https}}'

zrb/builtin/generator/simple_python_app/template/src/kebab-app-name/src/template.env

@@ -1,3 +1,4 @@
 PYTHONUNBUFFERED=1
+APP_HOST=0.0.0.0
 APP_MESSAGE="Salve Mane"
 APP_PORT="httpPort"

zrb/helper/list/ensure_uniqueness.py

@@ -0,0 +1,22 @@
+from typing import Any, Callable, List, Optional
+
+TComparator = Callable[[Any, Any], bool]
+
+
+def ensure_uniqueness(
+    data: List[Any], comparator: Optional[TComparator] = None
+) -> List[Any]:
+    unique_data = []
+    for datum in data:
+        if datum in unique_data:
+            continue
+        if comparator is not None:
+            is_exist = False
+            for unique_datum in unique_data:
+                if comparator(unique_datum, datum):
+                    is_exist = True
+                    break
+            if is_exist:
+                continue
+        unique_data.append(datum)
+    return unique_data

zrb/helper/render_data.py

@@ -1,6 +1,6 @@
 from .util import (
     coalesce, coalesce_str, to_camel_case, to_pascal_case, to_kebab_case,
-    to_snake_case, to_human_readable
+    to_snake_case, to_human_readable, to_boolean
 )
 import datetime
 import os
@@ -19,6 +19,7 @@ DEFAULT_RENDER_DATA = {
         'to_pascal_case': to_pascal_case,
         'to_kebab_case': to_kebab_case,
         'to_snake_case': to_snake_case,
-        'to_human_readable': to_human_readable
+        'to_human_readable': to_human_readable,
+        'to_boolean': to_boolean,
     }
 }

zrb/helper/string/conversion.py

@@ -1,9 +1,9 @@
+import keyword
 import logging
 import re
 
 NON_WORD = re.compile(r'[\W]+')
-NON_ALPHA_NUM = re.compile(r'[^0-9a-zA-Z]+')
-LEADING_NUM = re.compile(r'^[0-9]+')
+LEADING_NUM = re.compile(r'^\d+')
 LOGGING_LEVEL_MAP = {
     'critical': logging.CRITICAL,
     'fatal': logging.FATAL,
@@ -21,14 +21,14 @@ def to_cmd_name(name: str) -> str:
 
 
 def to_variable_name(string: str) -> str:
-    # Remove any non-alphanumeric characters
-    string = NON_ALPHA_NUM.sub(' ', string).strip()
-    # Convert to lowercase
-    string = string.lower()
-    # Replace spaces with underscores
-    string = string.replace(' ', '_')
+    # Replace any non-word characters with underscore
+    string = NON_WORD.sub('_', string).strip()
     # Remove leading digits
     string = LEADING_NUM.sub('', string)
+    # Convert to lowercase
+    string = string.lower()
+    if keyword.iskeyword(string):
+        return string + '_'
     return string
 
 

zrb/helper/util.py

@@ -1,4 +1,5 @@
 from typing import Any, Optional
+from .string.conversion import to_boolean as conversion_to_boolean
 import re
 import jinja2
 
@@ -87,3 +88,7 @@ def to_human_readable(text: Optional[str]) -> str:
         if new_part != '':
             new_parts.append(new_part)
     return ' '.join(new_parts).strip(' ')
+
+
+def to_boolean(text: str) -> bool:
+    return conversion_to_boolean(text)