zrb 0.28.1__py3-none-any.whl → 1.0.0a2__py3-none-any.whl

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/monitoring/otel-collector-metrics/otel-collector-metrics-config.yaml

@@ -1,69 +0,0 @@
-receivers:
-  otlp:
-    protocols:
-      grpc:
-      http:
-  prometheus:
-    config:
-      scrape_configs:
-        # otel-collector-metrics internal metrics
-        - job_name: otel-collector-metrics
-          scrape_interval: 60s
-          static_configs:
-            - targets:
-                - localhost:8888
-              labels:
-                job_name: otel-collector-metrics
-        # SigNoz span metrics
-        - job_name: signozspanmetrics-collector
-          scrape_interval: 60s
-          static_configs:
-            - targets:
-              - otel-collector:8889
-
-processors:
-  batch:
-    send_batch_size: 10000
-    send_batch_max_size: 11000
-    timeout: 10s
-  # memory_limiter:
-  #   # 80% of maximum memory up to 2G
-  #   limit_mib: 1500
-  #   # 25% of limit up to 2G
-  #   spike_limit_mib: 512
-  #   check_interval: 5s
-  #
-  #   # 50% of the maximum memory
-  #   limit_percentage: 50
-  #   # 20% of max memory usage spike expected
-  #   spike_limit_percentage: 20
-  # queued_retry:
-  #   num_workers: 4
-  #   queue_size: 100
-  #   retry_on_failure: true
-
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
-  zpages:
-    endpoint: 0.0.0.0:55679
-  pprof:
-    endpoint: 0.0.0.0:1777
-
-exporters:
-  clickhousemetricswrite:
-    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
-
-service:
-  telemetry:
-    metrics:
-      address: 0.0.0.0:8888
-  extensions:
-    - health_check
-    - zpages
-    - pprof
-  pipelines:
-    metrics:
-      receivers: [prometheus]
-      processors: [batch]
-      exporters: [clickhousemetricswrite]

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/monitoring/query-service/data/.gitignore

@@ -1 +0,0 @@
-signoz.db

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/monitoring/query-service/prometheus.yml

@@ -1,25 +0,0 @@
-# my global config
-global:
-  scrape_interval:     5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
-  # scrape_timeout is set to the global default (10s).
-
-# Alertmanager configuration
-alerting:
-  alertmanagers:
-  - static_configs:
-    - targets:
-      - alertmanager:9093
-
-# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
-rule_files:
-  # - "first_rules.yml"
-  # - "second_rules.yml"
-  - 'alerts.yml'
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
-scrape_configs: []
-
-remote_read:
-  - url: tcp://clickhouse:9000/?database=signoz_metrics

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/.dockerignore

@@ -1,9 +0,0 @@
-module_enabled.env
-module_disabled.env
-.env
-.venv
-__pycache__
-frontend/node_modules
-frontend/build
-frontend/svelte-kit
-frontend/package

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/.gitignore

@@ -1,5 +0,0 @@
-.env
-.venv
-frontend/build
-frontend/build
-__pycache__

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/.python-version

@@ -1 +0,0 @@
-3.10.0

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/Dockerfile

@@ -1,20 +0,0 @@
-# First stage: build the frontend
-FROM node:18-alpine AS frontend-builder
-WORKDIR /frontend
-COPY frontend .
-RUN npm install --save-dev && npm run build
-
-# Second stage: build the API and copy the frontend build
-FROM python:3.10-slim
-ENV APP_HOST 0.0.0.0
-ENV APP_PORT 8080
-ENV POETRY_VIRTUAL_ENVS_CREATE=false
-WORKDIR /home
-RUN apt update && apt install curl psmisc -y
-RUN pip install -U pip
-RUN pip install "poetry==1.7.1"
-COPY ./pyproject.toml ./pyproject.toml
-RUN poetry install --only main --no-root
-COPY . .
-COPY --from=frontend-builder /frontend/build /home/frontend/build
-CMD ./start.sh

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/error.py

@@ -1,56 +0,0 @@
-from collections.abc import Mapping
-from typing import Any, Optional
-
-from fastapi import HTTPException
-
-
-class HTTPAPIException(HTTPException):
-    def __init__(
-        self,
-        status_code: Optional[int] = None,
-        error_message: Optional[str] = None,
-        headers: Optional[Mapping[str, Any]] = None,
-        error: Optional[Exception] = None,
-    ):
-        if error is HTTPException:
-            status_code = (error.status_code,)
-            detail = (error.detail,)
-            headers = self._get_headers(error.headers)
-            return super().__init__(
-                status_code=status_code, detail=detail, headers=headers
-            )
-        if error is not None:
-            error_message = f"{error}"
-        status_code = self._get_status_code(status_code, error_message)
-        headers = self._get_headers(headers)
-        return super().__init__(
-            status_code=status_code, detail=error_message, headers=headers
-        )
-
-    def _get_status_code(
-        self, status_code: Optional[int], error_message: Optional[str]
-    ) -> int:
-        if status_code is not None:
-            return status_code
-        if error_message is None:
-            return 500
-        if error_message.lower().startswith("not found"):
-            return 404
-        if error_message.lower().startswith("forbidden"):
-            return 403
-        if error_message.lower().startswith("unauthorized"):
-            return 403
-        if error_message.lower().startswith("unauthenticated"):
-            return 401
-        if error_message.lower().startswith("unprocessable"):
-            return 422
-        if error_message.lower().startswith("invalid"):
-            return 422
-        return 500
-
-    def _get_headers(
-        self, original_headers: Optional[Mapping[str, Any]] = None
-    ) -> Mapping[str, Any]:
-        headers = {} if original_headers is None else dict(original_headers)
-        headers["api-error"] = "yes"
-        return headers

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/__init__.py

@@ -1,27 +0,0 @@
-from component.messagebus.kafka.admin import KafkaAdmin
-from component.messagebus.kafka.consumer import KafkaConsumer
-from component.messagebus.kafka.publisher import KafkaPublisher
-from component.messagebus.messagebus import (
-    Admin,
-    Consumer,
-    MessageSerializer,
-    Publisher,
-)
-from component.messagebus.mock import MockAdmin, MockConsumer, MockPublisher
-from component.messagebus.rabbitmq.admin import RMQAdmin
-from component.messagebus.rabbitmq.consumer import RMQConsumer
-from component.messagebus.rabbitmq.publisher import RMQPublisher
-
-assert Admin
-assert Consumer
-assert Publisher
-assert MessageSerializer
-assert MockAdmin
-assert MockConsumer
-assert MockPublisher
-assert KafkaAdmin
-assert KafkaConsumer
-assert KafkaPublisher
-assert RMQAdmin
-assert RMQConsumer
-assert RMQPublisher

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/kafka/admin.py

@@ -1,167 +0,0 @@
-import asyncio
-import logging
-from collections.abc import Mapping
-from typing import Optional
-
-from aiokafka import __version__
-from aiokafka.admin import AIOKafkaAdminClient, NewTopic
-from component.messagebus.messagebus import Admin
-
-
-class KafkaEventConfig:
-    def __init__(
-        self, topic_name: str, num_partitions: int = 1, replication_factor: int = 1
-    ):
-        self.topic_name = topic_name
-        self.num_partitions = num_partitions
-        self.replication_factor = replication_factor
-
-
-class KafkaAdmin(Admin):
-    def __init__(
-        self,
-        logger: logging.Logger,
-        bootstrap_servers: str,
-        configs: Mapping[str, KafkaEventConfig],
-        client_id: str = "kafka-admin-" + __version__,
-        security_protocol="PLAINTEXT",
-        sasl_mechanism="PLAIN",
-        sasl_plain_password=None,
-        sasl_plain_username=None,
-        sasl_kerberos_service_name="kafka",
-        sasl_kerberos_domain_name=None,
-        sasl_oauth_token_provider=None,
-    ):
-        self.logger = logger
-        self.configs = configs
-        self.bootstrap_servers = bootstrap_servers
-        self.client_id = client_id
-        self.security_protocol = security_protocol
-        self.sasl_mechanism = sasl_mechanism
-        self.sasl_plain_password = sasl_plain_password
-        self.sasl_plain_username = sasl_plain_username
-        self.sasl_kerberos_service_name = sasl_kerberos_service_name
-        self.sasl_kerberos_domain_name = sasl_kerberos_domain_name
-        self.sasl_oauth_token_provider = sasl_oauth_token_provider
-        self._existing_events: Mapping[str, bool] = {}
-
-    async def create_events(self, event_names: list[str]):
-        # Only handle non-existing events
-        event_names = [
-            event_name
-            for event_name in event_names
-            if event_name not in self._existing_events
-        ]
-        if len(event_names) == 0:
-            return
-        # Create topics
-        topics = [self.get_new_topic(event_name) for event_name in event_names]
-        try:
-            admin_client = self._create_connection()
-            admin_client.create_topics(topics)
-            admin_client.close()
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(
-                " ".join(
-                    [
-                        "🐼 [kafka-admin] Something wrong when ",
-                        f"creating topics: {topics}",
-                    ]
-                ),
-                exc_info=True,
-            )
-        for event_name in event_names:
-            self._existing_events[event_name] = True
-
-    async def delete_events(self, event_names: list[str]):
-        # Only handle existing events
-        event_names = [
-            event_name
-            for event_name in event_names
-            if event_name in self._existing_events
-        ]
-        if len(event_names) == 0:
-            return
-        # Create topic names
-        topic_names = [
-            self.get_topic_name(event_name)
-            for event_name in event_names
-            if event_name in self._existing_events
-        ]
-        try:
-            admin_client = self._create_connection()
-            admin_client.delete_topics(topic_names)
-            admin_client.close()
-            for event_name in event_names:
-                del self._existing_events[event_name]
-        except (asyncio.CancelledError, GeneratorExit, Exception):
-            self.logger.error(
-                " ".join(
-                    [
-                        "🐼 [kafka-admin] Something wrong when ",
-                        f"deleting topics: {topic_names}",
-                    ]
-                ),
-                exc_info=True,
-            )
-
-    def get_config(self, event_name: str) -> KafkaEventConfig:
-        if event_name in self.configs:
-            return self.configs[event_name]
-        return KafkaEventConfig(topic_name=event_name)
-
-    def get_topic_name(self, event_name: str) -> str:
-        event_config = self.get_config(event_name)
-        return event_config.topic_name
-
-    def get_new_topic(self, event_name: str) -> NewTopic:
-        event_config = self.get_config(event_name)
-        topic = NewTopic(
-            name=event_config.topic_name,
-            num_partitions=event_config.num_partitions,
-            replication_factor=event_config.replication_factor,
-        )
-        return topic
-
-    def _create_connection(self) -> AIOKafkaAdminClient:
-        return AIOKafkaAdminClient(
-            bootstrap_servers=self.bootstrap_servers,
-            client_id=self.client_id,
-            security_protocol=self.security_protocol,
-            sasl_mechanism=self.sasl_mechanism,
-            sasl_plain_password=self.sasl_plain_password,
-            sasl_plain_username=self.sasl_plain_username,
-            sasl_kerberos_service_name=self.sasl_kerberos_service_name,
-            sasl_kerberos_domain_name=self.sasl_kerberos_domain_name,
-            sasl_oauth_token_provider=self.sasl_oauth_token_provider,
-        )
-
-
-def must_get_kafka_admin(
-    logger: logging.Logger,
-    kafka_admin: Optional[KafkaAdmin],
-    bootstrap_servers: str,
-    client_id="kafka-admin-" + __version__,
-    security_protocol="PLAINTEXT",
-    sasl_mechanism="PLAIN",
-    sasl_plain_password=None,
-    sasl_plain_username=None,
-    sasl_kerberos_service_name="kafka",
-    sasl_kerberos_domain_name=None,
-    sasl_oauth_token_provider=None,
-) -> KafkaAdmin:
-    if kafka_admin is None:
-        return KafkaAdmin(
-            logger=logger,
-            bootstrap_servers=bootstrap_servers,
-            configs={},
-            client_id=client_id,
-            security_protocol=security_protocol,
-            sasl_mechanism=sasl_mechanism,
-            sasl_plain_password=sasl_plain_password,
-            sasl_plain_username=sasl_plain_username,
-            sasl_kerberos_service_name=sasl_kerberos_service_name,
-            sasl_kerberos_domain_name=sasl_kerberos_domain_name,
-            sasl_oauth_token_provider=sasl_oauth_token_provider,
-        )
-    return kafka_admin

zrb/builtin/project/add/fastapp/app/template/src/kebab-zrb-app-name/src/component/messagebus/kafka/consumer.py

@@ -1,248 +0,0 @@
-import asyncio
-import inspect
-import logging
-from collections.abc import Callable, Mapping
-from typing import Any, Optional
-
-from aiokafka import AIOKafkaConsumer, __version__
-from aiokafka.consumer.consumer import RoundRobinPartitionAssignor
-from component.messagebus.kafka.admin import KafkaAdmin, must_get_kafka_admin
-from component.messagebus.messagebus import (
-    Consumer,
-    MessageSerializer,
-    TEventHandler,
-    must_get_message_serializer,
-)
-
-
-class KafkaConsumer(Consumer):
-    def __init__(
-        self,
-        logger: logging.Logger,
-        bootstrap_servers: str,
-        client_id="aiokafka-" + __version__,
-        group_id: Optional[str] = None,
-        key_deserializer=None,
-        value_deserializer=None,
-        fetch_max_wait_ms=500,
-        fetch_max_bytes=52428800,
-        fetch_min_bytes=1,
-        max_partition_fetch_bytes=1 * 1024 * 1024,
-        request_timeout_ms=40 * 1000,
-        retry_backoff_ms=100,
-        auto_offset_reset="latest",
-        enable_auto_commit=True,
-        auto_commit_interval_ms=5000,
-        check_crcs=True,
-        metadata_max_age_ms=5 * 60 * 1000,
-        partition_assignment_strategy=(RoundRobinPartitionAssignor,),
-        max_poll_interval_ms=300000,
-        rebalance_timeout_ms=None,
-        session_timeout_ms=10000,
-        heartbeat_interval_ms=3000,
-        consumer_timeout_ms=200,
-        max_poll_records=None,
-        ssl_context=None,
-        security_protocol="PLAINTEXT",
-        api_version="auto",
-        exclude_internal_topics=True,
-        connections_max_idle_ms=540000,
-        isolation_level="read_uncommitted",
-        sasl_mechanism="PLAIN",
-        sasl_plain_password=None,
-        sasl_plain_username=None,
-        sasl_kerberos_service_name="kafka",
-        sasl_kerberos_domain_name=None,
-        sasl_oauth_token_provider=None,
-        serializer: Optional[MessageSerializer] = None,
-        kafka_admin: Optional[KafkaAdmin] = None,
-        retry: int = 3,
-        retry_interval: int = 10,
-        identifier="kafka-consumer",
-    ):
-        self.logger = logger
-        self.serializer = must_get_message_serializer(serializer)
-        self.kafka_admin = must_get_kafka_admin(
-            logger=logger,
-            kafka_admin=kafka_admin,
-            bootstrap_servers=bootstrap_servers,
-            security_protocol=security_protocol,
-            sasl_mechanism=sasl_mechanism,
-            sasl_plain_password=sasl_plain_password,
-            sasl_plain_username=sasl_plain_username,
-            sasl_kerberos_service_name=sasl_kerberos_service_name,
-            sasl_kerberos_domain_name=sasl_kerberos_domain_name,
-            sasl_oauth_token_provider=sasl_oauth_token_provider,
-        )
-        self.bootstrap_servers = bootstrap_servers
-        self.client_id = client_id
-        self.group_id = group_id
-        self.key_deserializer = key_deserializer
-        self.value_deserializer = value_deserializer
-        self.fetch_max_wait_ms = fetch_max_wait_ms
-        self.fetch_max_bytes = fetch_max_bytes
-        self.fetch_min_bytes = fetch_min_bytes
-        self.max_partition_fetch_bytes = max_partition_fetch_bytes
-        self.request_timeout_ms = request_timeout_ms
-        self.retry_backoff_ms = retry_backoff_ms
-        self.auto_offset_reset = auto_offset_reset
-        self.enable_auto_commit = enable_auto_commit
-        self.auto_commit_interval_ms = auto_commit_interval_ms
-        self.check_crcs = check_crcs
-        self.metadata_max_age_ms = metadata_max_age_ms
-        self.partition_assignment_strategy = partition_assignment_strategy
-        self.max_poll_interval_ms = max_poll_interval_ms
-        self.rebalance_timeout_ms = rebalance_timeout_ms
-        self.session_timeout_ms = session_timeout_ms
-        self.heartbeat_interval_ms = heartbeat_interval_ms
-        self.consumer_timeout_ms = consumer_timeout_ms
-        self.max_poll_records = max_poll_records
-        self.ssl_context = ssl_context
-        self.security_protocol = security_protocol
-        self.api_version = api_version
-        self.exclude_internal_topics = exclude_internal_topics
-        self.connections_max_idle_ms = connections_max_idle_ms
-        self.isolation_level = isolation_level
-        self.sasl_mechanism = sasl_mechanism
-        self.sasl_plain_password = sasl_plain_password
-        self.sasl_plain_username = sasl_plain_username
-        self.sasl_kerberos_service_name = sasl_kerberos_service_name
-        self.sasl_kerberos_domain_name = sasl_kerberos_domain_name
-        self.sasl_oauth_token_provider = sasl_oauth_token_provider
-        self.consumer: Optional[AIOKafkaConsumer] = None
-        self.retry = retry
-        self.retry_interval = retry_interval
-        self._handlers: Mapping[str, TEventHandler] = {}
-        self._is_start_triggered = False
-        self._is_stop_triggered = False
-        self._topic_to_event_map: Mapping[str, str] = {}
-        self.identifier = identifier
-
-    def register(self, event_name: str) -> Callable[[TEventHandler], Any]:
-        def wrapper(handler: TEventHandler):
-            self.logger.warning(
-                f'🐼 [{self.identifier}] Register handler for "{event_name}"'
-            )
-            self._handlers[event_name] = handler
-            return handler
-
-        return wrapper
-
-    async def start(self):
-        if self._is_start_triggered:
-            return
-        self._is_start_triggered = True
-        return await self._start(self.retry)
-
-    async def stop(self):
-        if self._is_stop_triggered:
-            return
-        self._is_stop_triggered = True
-        await self._disconnect()
-
-    async def _start(self, retry: int):
-        try:
-            if self.consumer is None:
-                await self._connect()
-            await self._init_topics()
-            topics = list(self._topic_to_event_map.keys())
-            self.logger.warning(f"🐼 [{self.identifier}] Subscribe to topics: {topics}")
-            self.consumer.subscribe(topics=topics)
-            async for message in self.consumer:
-                topic_name = message.topic
-                event_name = self._topic_to_event_map[topic_name]
-                message_handler = self._handlers.get(event_name)
-                decoded_value = self.serializer.decode(event_name, message.value)
-                self.logger.info(
-                    f'🐼 [{self.identifier}] Consume from "{topic_name}": '
-                    + f"{decoded_value}"
-                )
-                await self._run_handler(message_handler, decoded_value)
-            retry = self.retry
-        except (asyncio.CancelledError, GeneratorExit, Exception) as e:
-            if retry > 0:
-                self.logger.error(f"🐼 [{self.identifier}]", exc_info=True)
-            if retry == 0:
-                self.logger.error(
-                    f"🐼 [{self.identifier}] Failed to consume message after "
-                    + f"{self.retry} attempts"
-                )
-                self.logger.fatal(f"🐼 [{self.identifier}] Cannot retry")
-                raise e
-            self.logger.warning(f"🐼 [{self.identifier}] Retry to consume")
-            await self._disconnect()
-            await asyncio.sleep(self.retry_interval)
-            await self._start(retry - 1)
-        finally:
-            await self._disconnect()
-
-    async def _init_topics(self):
-        event_names = self._handlers.keys()
-        await self.kafka_admin.create_events(event_names)
-        self._topic_to_event_map = {
-            event_name: self.kafka_admin.get_topic_name(event_name)
-            for event_name in event_names
-        }
-
-    async def _connect(self):
-        self.logger.info(f"🐼 [{self.identifier}] Create kafka consumer")
-        self.consumer = AIOKafkaConsumer(
-            bootstrap_servers=self.bootstrap_servers,
-            client_id=self.client_id,
-            group_id=self.group_id,
-            key_deserializer=self.key_deserializer,
-            value_deserializer=self.value_deserializer,
-            fetch_max_wait_ms=self.fetch_max_wait_ms,
-            fetch_max_bytes=self.fetch_max_bytes,
-            fetch_min_bytes=self.fetch_min_bytes,
-            max_partition_fetch_bytes=self.max_partition_fetch_bytes,
-            request_timeout_ms=self.request_timeout_ms,
-            retry_backoff_ms=self.retry_backoff_ms,
-            auto_offset_reset=self.auto_offset_reset,
-            enable_auto_commit=self.enable_auto_commit,
-            auto_commit_interval_ms=self.auto_commit_interval_ms,
-            check_crcs=self.check_crcs,
-            metadata_max_age_ms=self.metadata_max_age_ms,
-            partition_assignment_strategy=self.partition_assignment_strategy,
-            max_poll_interval_ms=self.max_poll_interval_ms,
-            rebalance_timeout_ms=self.rebalance_timeout_ms,
-            session_timeout_ms=self.session_timeout_ms,
-            heartbeat_interval_ms=self.heartbeat_interval_ms,
-            consumer_timeout_ms=self.consumer_timeout_ms,
-            max_poll_records=self.max_poll_records,
-            ssl_context=self.ssl_context,
-            security_protocol=self.security_protocol,
-            api_version=self.api_version,
-            exclude_internal_topics=self.exclude_internal_topics,
-            connections_max_idle_ms=self.connections_max_idle_ms,
-            isolation_level=self.isolation_level,
-            sasl_mechanism=self.sasl_mechanism,
-            sasl_plain_password=self.sasl_plain_password,
-            sasl_plain_username=self.sasl_plain_username,
-            sasl_kerberos_service_name=self.sasl_kerberos_service_name,
-            sasl_kerberos_domain_name=self.sasl_kerberos_domain_name,
-            sasl_oauth_token_provider=self.sasl_oauth_token_provider,
-        )
-        self.logger.info(f"🐼 [{self.identifier}] Start kafka consumer")
-        await self.consumer.start()
-        self.logger.info(f"🐼 [{self.identifier}] Kafka consumer started")
-
-    async def _disconnect(self):
-        if self.consumer is not None:
-            try:
-                self.logger.info(
-                    f"🐼 [{self.identifier}] Unsubscribe kafka consumer "
-                    + "from all topics"
-                )
-                self.consumer.unsubscribe()
-                self.logger.info(f"🐼 [{self.identifier}] Stop kafka consumer")
-                await self.consumer.stop()
-                self.logger.info(f"🐼 [{self.identifier}] Kafka consumer stopped")
-            except (asyncio.CancelledError, GeneratorExit, Exception):
-                self.logger.error(f"🐼 [{self.identifier}]", exc_info=True)
-        self.consumer = None
-
-    async def _run_handler(self, message_handler: TEventHandler, decoded_value: Any):
-        if inspect.iscoroutinefunction(message_handler):
-            return asyncio.create_task(message_handler(decoded_value))
-        return message_handler(decoded_value)