qena-shared-lib 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

qena_shared_lib/__init__.py

@@ -1,11 +1,22 @@
 try:
-    from . import rabbitmq, scheduler, security
+    from . import (
+        cache,
+        eventbus,
+        kafka,
+        mongodb,
+        rabbitmq,
+        redis,
+        scheduler,
+        security,
+        sync,
+    )
 except NameError:
     pass
 from . import (
     application,
     background,
     dependencies,
+    enums,
     exceptions,
     http,
     logging,
@@ -16,13 +27,20 @@ from . import (
 __all__ = [
     "application",
     "background",
+    "cache",
     "dependencies",
+    "enums",
+    "eventbus",
     "exceptions",
     "http",
+    "kafka",
     "logging",
-    "remotelogging",
+    "mongodb",
     "rabbitmq",
+    "redis",
+    "remotelogging",
     "scheduler",
     "security",
+    "sync",
     "utils",
 ]

qena_shared_lib/alias.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel, ConfigDict
+from pydantic.alias_generators import to_camel, to_snake
+
+__all__ = [
+    "CamelCaseAliasedBaseModel",
+    "SnakeCaseAliasedBaseModel",
+]
+
+
+class CamelCaseAliasedBaseModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        arbitrary_types_allowed=True,
+        extra="ignore",
+        strict=False,
+    )
+
+
+class SnakeCaseAliasedBaseModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_snake,
+        populate_by_name=True,
+        arbitrary_types_allowed=True,
+        extra="ignore",
+        strict=False,
+    )

qena_shared_lib/application.py

@@ -7,13 +7,13 @@ from punq import Container, Scope, empty
 from starlette.types import Lifespan
 from typing_extensions import Self
 
-from .exception_handlers import (
+from .http import ControllerBase
+from .http._exception_handlers import (
     AbstractHttpExceptionHandler,
-    GeneralHttpExceptionHandler,
+    HttpGeneralExceptionHandler,
     HTTPServiceExceptionHandler,
     RequestValidationErrorHandler,
 )
-from .http import ControllerBase
 
 __all__ = [
     "Builder",
@@ -127,7 +127,7 @@ class Builder:
 
     def with_default_exception_handlers(self) -> Self:
         self.with_exception_handlers(
-            GeneralHttpExceptionHandler,
+            HttpGeneralExceptionHandler,
             HTTPServiceExceptionHandler,
             RequestValidationErrorHandler,
         )

qena_shared_lib/background.py

@@ -9,7 +9,7 @@ from uuid import uuid4
 from prometheus_client import Enum as PrometheusEnum
 from starlette.background import BackgroundTask
 
-from .logging import LoggerProvider
+from .logging import LoggerFactory
 from .remotelogging import BaseRemoteLogSender
 from .utils import AsyncEventLoopMixin
 
@@ -20,7 +20,7 @@ __all__ = [
 
 
 class Background(AsyncEventLoopMixin):
-    BACKGROUND_RUNNER_STATE = PrometheusEnum(
+    _BACKGROUND_RUNNER_STATE = PrometheusEnum(
         name="background_runner_state",
         documentation="Background runner state",
         states=["running", "stopped"],
@@ -34,7 +34,7 @@ class Background(AsyncEventLoopMixin):
         self._started = False
         self._stopped = False
         self._remote_logger = remote_logger
-        self._logger = LoggerProvider.default().get_logger("backgroud")
+        self._logger = LoggerFactory.get_logger("background")
         self._tasks: dict[str, Task[Any]] = {}
 
     async def _task_manager(
@@ -54,8 +54,10 @@ class Background(AsyncEventLoopMixin):
 
             await self._tasks[task_id]
         except Exception:
-            self._remote_logger.error(
-                "exception occured when running background task {task.func.__name__} with id {task_id}"
+            self._remote_logger.exception(
+                message=f"exception occured while running background task {task.func.__name__} with id {task_id}",
+                tags=["background", "task_execution_failed", task_id],
+                extra={"serviceType": "background", "taskId": task_id},
             )
         finally:
             self._logger.info("finished running %s", task.func.__name__)
@@ -91,7 +93,7 @@ class Background(AsyncEventLoopMixin):
             raise RuntimeError("background runner already running")
 
         self.loop.create_task(self._run_tasks())
-        self.BACKGROUND_RUNNER_STATE.state("running")
+        self._BACKGROUND_RUNNER_STATE.state("running")
 
         self._started = True
 
@@ -101,7 +103,7 @@ class Background(AsyncEventLoopMixin):
 
         self._stopped = True
         self._queue.put_nowait((None, None))
-        self.BACKGROUND_RUNNER_STATE.state("stopped")
+        self._BACKGROUND_RUNNER_STATE.state("stopped")
 
     def is_alive(self, task_id: str) -> bool:
         if task_id in self._tasks and not self._tasks[task_id].done():

qena_shared_lib/cache.py

@@ -0,0 +1,61 @@
+from typing import Any, TypeVar, cast
+
+from redis.asyncio import Redis
+
+from .alias import CamelCaseAliasedBaseModel
+from .redis import RedisDependent
+
+__all__ = [
+    "CachedObject",
+    "CacheManager",
+]
+
+
+CO = TypeVar("CO", bound="CachedObject")
+
+
+class CachedObject(CamelCaseAliasedBaseModel):
+    @classmethod
+    def from_raw_value(
+        cls, obj: Any, *args: Any, **kwargs: Any
+    ) -> "CachedObject":
+        cache_object = cls.model_validate_json(json_data=obj, *args, **kwargs)
+
+        return cast(CachedObject, cache_object)
+
+    @classmethod
+    def redis_key(cls) -> str:
+        return cls.__name__
+
+
+class CacheManager(RedisDependent):
+    def attach(self, redis_client: Redis) -> None:
+        self._redis_client = redis_client
+
+    @property
+    def redis(self) -> Redis:
+        return self._redis_client
+
+    async def get(self, cached_object_type: type[CO]) -> CO | None:
+        cache_object = await self._redis_client.get(
+            cached_object_type.redis_key()
+        )
+
+        if cache_object is None:
+            return None
+
+        return cast(CO, cached_object_type.from_raw_value(obj=cache_object))
+
+    async def set(self, cache_object: CachedObject) -> None:
+        if not isinstance(cache_object, CachedObject):
+            raise TypeError(
+                f"object is not type of `CachedObject`, got `{cache_object.__class__.__name__}`"
+            )
+
+        await self._redis_client.set(
+            name=cache_object.redis_key(),
+            value=cache_object.model_dump_json(),
+        )
+
+    async def unset(self, cached_object_type: type[CO]) -> None:
+        await self._redis_client.delete(cached_object_type.redis_key())

qena_shared_lib/enums.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+__all__ = ["ServiceType"]
+
+
+class ServiceType(Enum):
+    HTTP = 0
+    RABBIT_MQ = 1

qena_shared_lib/eventbus.py

@@ -0,0 +1,373 @@
+from asyncio import Future, Queue, Task, wait_for
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Callable, cast
+from uuid import UUID, uuid4
+
+from pika.adapters.asyncio_connection import AsyncioConnection
+from pika.channel import Channel
+from pika.exchange_type import ExchangeType
+from pika.frame import Method
+from pika.spec import Basic, BasicProperties
+from pydantic_core import to_json
+
+from qena_shared_lib.rabbitmq import (
+    AbstractRabbitMQService,
+    BaseChannel,
+    ChannelPool,
+)
+
+from .alias import CamelCaseAliasedBaseModel
+from .remotelogging import BaseRemoteLogSender
+from .utils import AsyncEventLoopMixin
+
+__all__ = [
+    "Event",
+    "EventBus",
+    "EventBusChannelAdapter",
+    "EventQueue",
+    "EventReciever",
+    "GlobalEvent",
+]
+
+
+@dataclass
+class Event:
+    payload: Any | None = None
+
+    def to_json(self) -> str:
+        return cast(str, to_json(self.payload).decode())
+
+
+EventQueue = Queue[Event]
+
+
+class GlobalEvent(CamelCaseAliasedBaseModel):
+    event_key: str
+    payload: Any | None = None
+
+
+class EventBusChannelAdapter(BaseChannel):
+    def __init__(
+        self,
+        connection: AsyncioConnection,
+        on_event_bus_channel_opened: Callable[[Channel], None],
+        on_event_bus_consumer_cancel: Callable[[], None],
+    ):
+        super().__init__(connection=connection, failed_reopen_threshold=None)
+
+        self._on_event_bus_channel_opened = on_event_bus_channel_opened
+        self._on_event_bus_consumer_cancel = on_event_bus_consumer_cancel
+
+    def _hook_on_channel_opened(self) -> None:
+        if not isinstance(self._channel, Channel):
+            raise RuntimeError("channel not initialized")
+
+        self._on_event_bus_channel_opened(self._channel)
+
+    def _hook_on_cancelled(self) -> None:
+        self._on_event_bus_consumer_cancel()
+
+
+class EventReciever:
+    def __init__(self, event_key: str) -> None:
+        self._event_key = event_key
+        self._event_queue = EventQueue()
+        self._reciever_id = uuid4()
+
+    @property
+    def event_key(self) -> str:
+        return self._event_key
+
+    @property
+    def event_queue(self) -> EventQueue:
+        return self._event_queue
+
+    @property
+    def reciever_id(self) -> UUID:
+        return self._reciever_id
+
+    async def wait_once(self, timeout: float) -> Event:
+        return await wait_for(fut=self._event_queue.get(), timeout=timeout)
+
+    async def subscribe(self) -> AsyncGenerator[Event, None]:
+        while True:
+            yield await self._event_queue.get()
+
+
+class EventBus(AbstractRabbitMQService, AsyncEventLoopMixin):
+    EVENT_BUS_EXCHANGE = "event_bus"
+
+    @classmethod
+    def set_event_bus_exchange(cls, name: str) -> None:
+        cls.EVENT_BUS_EXCHANGE = name
+
+    def __init__(self, remote_logger: BaseRemoteLogSender):
+        self._events: dict[str, dict[UUID, EventQueue]] = {}
+        self._worker_id = uuid4()
+        self._remote_logger = remote_logger
+        self._first_connect = True
+        self._event_reciever_tasks: list[Task[None]] = []
+
+    def initialize(
+        self, connection: AsyncioConnection, channel_pool: ChannelPool
+    ) -> Future[None]:
+        self._connection = connection
+        self._channel_pool = channel_pool
+        self._event_bus_future = self.loop.create_future()
+        self._event_bus_channel_adapter = EventBusChannelAdapter(
+            connection=connection,
+            on_event_bus_channel_opened=self._declare_exchange,
+            on_event_bus_consumer_cancel=self._register_consumer,
+        )
+
+        self._event_bus_channel_adapter.open().add_done_callback(
+            self._on_event_bus_channel_open_done
+        )
+
+        return cast(Future[None], self._event_bus_future)
+
+    def close(self) -> Future[None]:
+        close_future = self.loop.create_future()
+
+        close_future.set_result(None)
+
+        return cast(Future[None], close_future)
+
+    def _on_event_bus_channel_open_done(self, future: Future[UUID]) -> None:
+        if future.cancelled():
+            if not self._event_bus_future.done():
+                self._event_bus_future.cancel()
+
+            return
+
+        exception = future.exception()
+
+        if exception is not None:
+            if not self._event_bus_future.done():
+                self._event_bus_future.set_exception(exception)
+
+            return
+
+        self._event_bus_channel_id = future.result()
+
+    def create_event_reciever(
+        self,
+        event_key: str,
+    ) -> EventReciever:
+        event_reciever = EventReciever(event_key=event_key)
+
+        if event_key not in self._events:
+            self._events[event_key] = {
+                event_reciever.reciever_id: event_reciever.event_queue
+            }
+        else:
+            self._events[event_key][event_reciever.reciever_id] = Queue()
+
+        return event_reciever
+
+    async def emit(
+        self,
+        event_key: str,
+        payload: Any | None = None,
+        event_reciever: EventReciever | None = None,
+    ) -> None:
+        await self._submit_event(
+            event_key=event_key,
+            event_reciever=event_reciever,
+            payload=payload,
+        )
+
+    async def _submit_event(
+        self,
+        event_key: str,
+        globally_emitted: bool = False,
+        event_reciever: EventReciever | None = None,
+        payload: Any | None = None,
+    ) -> None:
+        event = Event(payload)
+        event_queues = self._events.get(event_key)
+
+        if event_queues is None:
+            if not globally_emitted:
+                global_event = GlobalEvent(
+                    event_key=event_key,
+                    payload=payload,
+                )
+
+                await self._emit_globally(global_event)
+
+            return
+
+        if event_reciever is not None:
+            event_queue = event_queues.get(event_reciever.reciever_id)
+
+            if event_queue is None:
+                return
+
+            return await event_queue.put(event)
+
+        for event_queue in event_queues.values():
+            await event_queue.put(event)
+
+    async def _emit_globally(self, global_event: GlobalEvent) -> None:
+        if self._channel_pool is None:
+            self._remote_logger.error(
+                message="channel pool not initialized to publish to global event recievers",
+                tags=["event_bus", "event_pool_not_initialized"],
+            )
+
+            return
+
+        try:
+            with await self._channel_pool.get() as channel:
+                channel.basic_publish(
+                    exchange=self.EVENT_BUS_EXCHANGE,
+                    routing_key="IRRELEVANT",
+                    body=global_event.model_dump_json().encode(),
+                )
+        except:
+            self._remote_logger.exception(
+                message="unable to publish event",
+                tags=["event_bus", "global_event_publishing_error"],
+            )
+
+    async def remove(self, event_reciever: EventReciever) -> None:
+        if (
+            event_reciever.event_key not in self._events
+            or event_reciever.reciever_id
+            not in self._events[event_reciever.event_key]
+        ):
+            return
+
+        del self._events[event_reciever.event_key][event_reciever.reciever_id]
+
+        if not self._events[event_reciever.event_key]:
+            del self._events[event_reciever.event_key]
+
+    def _declare_exchange(self, channel: Channel) -> None:
+        self._channel = channel
+
+        self._channel.exchange_declare(
+            exchange=self.EVENT_BUS_EXCHANGE,
+            exchange_type=ExchangeType.fanout,
+            auto_delete=True,
+            callback=self._on_exchange_declared,
+        )
+
+    def _on_exchange_declared(self, method: Method) -> None:
+        del method
+
+        self._declare_queue()
+
+    def _declare_queue(self) -> None:
+        self._consumer_queue = self._unique_consumer_queue()
+
+        try:
+            self._channel.queue_declare(
+                queue=self._consumer_queue,
+                auto_delete=True,
+                callback=self._on_queue_declared,
+            )
+        except Exception as e:
+            if not self._event_bus_future.done():
+                self._event_bus_future.set_exception(e)
+            else:
+                self._remote_logger.exception(
+                    message=f"unable to declare queue {self._consumer_queue} for event bus",
+                    tags=["event_bus", "queue_declaration_error"],
+                )
+
+    def _on_queue_declared(self, method: Method) -> None:
+        del method
+
+        try:
+            self._channel.queue_bind(
+                queue=self._consumer_queue,
+                exchange=self.EVENT_BUS_EXCHANGE,
+                callback=self._on_queue_bound,
+            )
+        except Exception as e:
+            if not self._event_bus_future.done():
+                self._event_bus_future.set_exception(e)
+            else:
+                self._remote_logger.exception(
+                    message="unable to bind queue to exchange",
+                    tags=["event_bus", "queue_binding_error"],
+                )
+
+    def _on_queue_bound(self, method: Method) -> None:
+        del method
+
+        self._register_consumer()
+
+    def _register_consumer(self) -> None:
+        try:
+            self._channel.basic_consume(
+                queue=self._consumer_queue,
+                on_message_callback=self._on_message_recieved,
+                auto_ack=True,
+                callback=self._on_consumer_registered,
+            )
+        except Exception as e:
+            if not self._event_bus_future.done():
+                self._event_bus_future.set_exception(e)
+            else:
+                self._remote_logger.exception(
+                    message=f"unable to start consuming {self._consumer_queue}",
+                    tags=["event_bus", "consumer_registration_error"],
+                )
+
+    def _on_consumer_registered(self, method: Method) -> None:
+        del method
+
+        if not self._event_bus_future.done():
+            self._event_bus_future.set_result(None)
+
+    def _on_message_recieved(
+        self,
+        channel: Channel,
+        method: Basic.Deliver,
+        properties: BasicProperties,
+        body: bytes,
+    ) -> None:
+        del channel, method, properties
+
+        try:
+            global_event = GlobalEvent.model_validate_json(body)
+        except:
+            self._remote_logger.exception(
+                message="event not deserializable",
+                tags=["event_bus", "event_deserialization_error"],
+            )
+
+            return
+
+        event_reciever_task = self.loop.create_task(
+            self._submit_event(
+                event_key=global_event.event_key,
+                globally_emitted=True,
+                payload=global_event.payload,
+            )
+        )
+
+        event_reciever_task.add_done_callback(self._on_event_emit)
+        self._event_reciever_tasks.append(event_reciever_task)
+
+    def _on_event_emit(self, task: Task[None]) -> None:
+        if task in self._event_reciever_tasks:
+            self._event_reciever_tasks.remove(task)
+
+        if task.cancelled():
+            return
+
+        e = task.exception()
+
+        if e is not None:
+            self._remote_logger.error(
+                message="unable to emit from global event",
+                tags=["event_bus", "emit_event_error", task.get_name()],
+                exception=e,
+            )
+
+    def _unique_consumer_queue(self) -> str:
+        return f"{self.EVENT_BUS_EXCHANGE}.worker.{self._worker_id.hex}"