talentro-commons 0.20.2__tar.gz → 0.20.4__tar.gz

{talentro_commons-0.20.2 → talentro_commons-0.20.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: talentro-commons
-Version: 0.20.2
+Version: 0.20.4
 Summary: This package contains all globally used code, services, models and data structures for Talentro
 License: Proprietary
 Author: Emiel van Essen

{talentro_commons-0.20.2 → talentro_commons-0.20.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "talentro-commons"
-version = "0.20.2"
+version = "0.20.4"
 description = "This package contains all globally used code, services, models and data structures for Talentro"
 authors = ["Emiel van Essen <emiel@marksmen.nl>"]
 license = "Proprietary"

talentro_commons-0.20.4/src/talentro/messaging/topology.py

@@ -0,0 +1,103 @@
+import aio_pika
+
+from typing import List, Tuple, Optional
+
+FIVE_MINUTES = 5 * 60 * 1000
+FIFTEEN_MINUTES = 15 * 60 * 1000
+
+class TopologyConfig:
+    def __init__(
+        self,
+        exchange_name: str,
+        exchange_type: aio_pika.ExchangeType,
+        bindings: List[Tuple[str, str]],
+        dlx_name: str = None,
+        retry_config: Optional[dict] = None
+    ):
+        self.exchange_name = exchange_name
+        self.exchange_type = exchange_type
+        self.bindings = bindings
+        self.dlx_name = dlx_name
+        self.retry_config = retry_config
+
+    def get_max_retries(self, queue_name: str) -> int:
+        """Get max retries for a specific queue, default 3."""
+        if not self.retry_config:
+            return 3
+        return self.retry_config.get(queue_name, {}).get("max_retries", 3)
+
+
+FRONTEND_TOPOLOGY = TopologyConfig(
+    exchange_name="x.frontend",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        ("q.frontend.updates.gateway", "frontend.update"),
+    ],
+    retry_config={
+        "q.frontend.updates.gateway": {"enabled": False}
+    }
+)
+
+
+APPLICATIONS_TOPOLOGY = TopologyConfig(
+    exchange_name="x.applications",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        # Integrations
+        ("q.applications.raw.integrations", "applications.raw"),
+        ("q.applications.send.integrations", "applications.send"),
+
+        # Acquisitions
+        ("q.applications.saved.acquisitions", "applications.saved"),
+
+        # Vacancies
+        ("q.applications.saved.vacancies", "applications.saved"),
+
+        # Candidates
+        ("q.applications.normalized.candidates", "applications.normalized"),
+        ("q.applications.external.upserted.candidates", "applications.external.upserted"),
+    ],
+    dlx_name="x.applications.dlx",
+    retry_config={
+        # Integrations
+        "q.applications.raw.integrations": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3},
+        "q.applications.send.integrations": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3},
+
+        # Acquisitions
+        "q.applications.saved.acquisitions": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3},
+
+        # Vacancies
+        "q.applications.saved.vacancies": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3},
+
+        # Candidates
+        "q.applications.normalized.candidates": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3},
+        "q.applications.external.upserted.candidates": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3}
+    }
+)
+
+
+FEEDS_TOPOLOGY = TopologyConfig(
+    exchange_name="x.feeds",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        # Acquisitions
+        ("q.feeds.sync.acquisitions", "feeds.sync"),
+    ],
+    dlx_name="x.feeds.dlx",
+    retry_config={
+        "q.feeds.sync.acquisitions": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3}
+    }
+)
+
+BILLING_TOPOLOGY = TopologyConfig(
+    exchange_name="x.billing",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        # Billing
+        ("q.billing.events.billing", "billing.event"),
+    ],
+    dlx_name="x.billing.dlx",
+    retry_config={
+        "q.billing.events.billing": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3}
+    }
+)

{talentro_commons-0.20.2 → talentro_commons-0.20.4}/src/talentro/services/rabbitmq.py

@@ -1,11 +1,14 @@
 import asyncio
 import os
+import traceback
 from typing import List, Tuple, Optional, Callable
 
 import aio_pika
 from aiormq import AMQPConnectionError
 
 from ..event import Message, Event
+from ..messaging.topology import TopologyConfig, FRONTEND_TOPOLOGY, BILLING_TOPOLOGY, FEEDS_TOPOLOGY, \
+    APPLICATIONS_TOPOLOGY
 from ..util.singleton import SingletonMeta
 
 
@@ -43,7 +46,8 @@ class RabbitMQ:
         exchange_type: aio_pika.ExchangeType = aio_pika.ExchangeType.TOPIC,
         durable: bool = True,
         bindings: List[Tuple[str, str]] = (),
-        dlx_name: Optional[str] = None
+        dlx_name: Optional[str] = None,
+        retry_config: Optional[dict] = None  # { "queue_name": {"retry_enabled": True, "ttl_ms": 300000}, ... }
     ):
         exchange = await self.channel.declare_exchange(
             exchange_name, exchange_type, durable=durable
@@ -73,7 +77,36 @@ class RabbitMQ:
                 await dlq.bind(dlx_name, routing_key=f"{queue_name}.dead")
                 print(f"  RabbitMQ - DLQ bound: {dlq.name} ← {dlx_name}")
 
-    async def consume(self, exchange_name: str, queue: str, callback: Callable, dlx_name: Optional[str] = None):
+                queue_retry_config = retry_config.get(queue_name, {})
+                if queue_retry_config.get("enabled", False):
+                    ttl_ms = queue_retry_config.get("ttl_ms", 300000)
+
+                    retry_exchange = await self.channel.declare_exchange(
+                        f"{dlx_name}.retry", aio_pika.ExchangeType.DIRECT, durable=True
+                    )
+
+                    retry_queue_args = {
+                        "x-dead-letter-exchange": exchange_name,
+                        "x-dead-letter-routing-key": queue_name,
+                        "x-message-ttl": ttl_ms
+                    }
+                    retry_queue = await self.channel.declare_queue(
+                        f"{queue_name}.retry",
+                        durable=True,
+                        arguments=retry_queue_args
+                    )
+                    await retry_queue.bind(retry_exchange, routing_key=f"{queue_name}.retry")
+                    print(f"  RabbitMQ - Retry queue enabled: {retry_queue.name} "
+                          f"(TTL: {ttl_ms}ms)")
+
+    async def consume(
+        self,
+        exchange_name: str,
+        queue: str,
+        callback: Callable,
+        dlx_name: Optional[str] = None,
+        max_retries: int = 3
+    ):
         while True:
             try:
                 print(f"  RabbitMQ - Consuming from queue: {queue}")
@@ -91,10 +124,17 @@ class RabbitMQ:
                     await queue_object.bind(exchange, routing_key=queue)
 
                 async for message in queue_object:
-                    async with message.process():
-                        event: Event = Event.decode(message.body)
-                        print(f"  RabbitMQ ({queue}) - [x] Received event: {event.meta.event_type} (trace: {event.meta.trace_id})")
-                        await callback(queue, event)
+                    try:
+                        async with message.process():
+                            event: Event = Event.decode(message.body)
+                            print(f"  RabbitMQ ({queue}) - [x] Received event: {event.meta.event_type} (trace: {event.meta.trace_id})")
+                            await callback(queue, event)
+                    except Exception as e:
+                        # Handle callback exception - send to retry queue
+                        print(f"  RabbitMQ ({queue}) - ⚠️ Error processing message: {e}")
+                        traceback.print_exc()
+                        await self._handle_failed_message(message, queue, exchange_name, dlx_name, max_retries)
+
             except (aio_pika.exceptions.AMQPConnectionError, aio_pika.exceptions.ChannelClosed):
                 print(f"  RabbitMQ - Connection lost while consuming {queue}. Retrying in 5 seconds...")
                 await asyncio.sleep(5)
@@ -102,6 +142,45 @@ class RabbitMQ:
                 print(f"  RabbitMQ - Unexpected error in consumer {queue}: {e}")
                 await asyncio.sleep(5)
 
+    @staticmethod
+    async def _handle_failed_message(
+            message: aio_pika.IncomingMessage,
+            queue_name: str,
+            exchange_name: str,
+            dlx_name: Optional[str] = None,
+            max_retries: int = 3
+    ):
+        """
+        Handle failed messages by sending them to the retry queue or DLQ.
+        """
+        try:
+            # Get death count from headers
+            x_death = message.headers.get('x-death', [])
+            death_count = 0
+
+            if x_death:
+                # x-death is a list of death info dicts
+                death_count = len(x_death)
+
+            print(f"  RabbitMQ ({queue_name}) - Retry attempt {death_count + 1}/{max_retries + 1}")
+
+            if death_count >= max_retries:
+                # Max retries exceeded - send to DLQ
+                print(f"  RabbitMQ ({queue_name}) - ❌ Max retries ({max_retries}) exceeded. Sending to DLQ.")
+                await message.nack(requeue=False)
+            else:
+                # Still have retries left - send to retry queue
+                if dlx_name:
+                    await message.nack(requeue=False)
+                    print(f"  RabbitMQ ({queue_name}) - Sending to retry queue...")
+                else:
+                    await message.nack(requeue=True)
+                    print(f"  RabbitMQ ({queue_name}) - Message requeued")
+
+        except Exception as e:
+            print(f"  RabbitMQ ({queue_name}) - Error handling failed message: {e}")
+            await message.nack(requeue=False)
+
     async def send_message(self, message: Message):
         # Haal de exchange op
         if message.exchange == "default":
@@ -131,6 +210,7 @@ class QueueContext(metaclass=SingletonMeta):
     def __init__(self):
         self._rabbit_mq = RabbitMQ()
         self._message_callbacks = []
+        self._topology_config: TopologyConfig | None = None
 
     async def connect(self):
         try:
@@ -140,22 +220,34 @@ class QueueContext(metaclass=SingletonMeta):
             await asyncio.sleep(5)
             await self.connect()
 
+    async def setup_topologies(self):
+        topologies = [FRONTEND_TOPOLOGY, APPLICATIONS_TOPOLOGY, FEEDS_TOPOLOGY, BILLING_TOPOLOGY]
+
+        for topology in topologies:
+            await self._setup_topology_with_config(topology)
+
+    async def _setup_topology_with_config(self, topology_config: TopologyConfig):
+        self._topology_config = topology_config
 
-    async def setup_topology(
-        self,
-        exchange_name: str,
-        exchange_type: aio_pika.ExchangeType = aio_pika.ExchangeType.TOPIC,
-        durable: bool = True,
-        bindings: List[Tuple[str, str]] = (),
-        dlx_name: Optional[str] = None
-    ):
         await self._rabbit_mq.setup_topology(
-            exchange_name, exchange_type, durable, bindings, dlx_name
+            topology_config.exchange_name,
+            topology_config.exchange_type,
+            bindings=topology_config.bindings,
+            dlx_name=topology_config.dlx_name,
+            retry_config=topology_config.retry_config
         )
 
-    async def start_consuming(self, exchange_name: str, queue: str, dlx_name: Optional[str] = None):
+    async def start_consuming(self, queue: str):
+        max_retries = self._topology_config.get_max_retries(queue) if self._topology_config else 3
+
         asyncio.create_task(
-            self._rabbit_mq.consume(exchange_name, queue, self._on_new_message, dlx_name=dlx_name)
+            self._rabbit_mq.consume(
+                self._topology_config.exchange_name,
+                queue,
+                self._on_new_message,
+                dlx_name=self._topology_config.dlx_name,
+                max_retries=max_retries
+            )
         )
 
     async def _on_new_message(self, queue: str, event: Event):