talentro-commons 0.20.1__py3-none-any.whl → 0.20.3__py3-none-any.whl

talentro/candidates/dataclasses.py

@@ -203,6 +203,7 @@ class ExternalLinkInfo(BaseModel):
 
     link: Optional[LinkInfo]
     external_id: Optional[str]
+    external_url: Optional[str]
     status: str
     application_id: UUID
 
@@ -222,5 +223,6 @@ class ExternalLinkInfo(BaseModel):
             link=link,
             status=model.status,
             external_id=model.external_id,
+            external_url=model.external_url,
             application_id=model.application_id,
         )

talentro/candidates/models.py

@@ -42,6 +42,7 @@ class Candidate(CandidatesOrganizationModel, table=True):
 class ExternalLink(CandidatesOrganizationModel, table=True):
     link_id: Optional[UUID] = Field(index=True, nullable=True)
     external_id: Optional[str] = Field(index=True, nullable=True)
+    external_url: Optional[str] = Field(index=True, nullable=True)
     status: str = Field(index=True)
 
     application_id: UUID = Field(foreign_key="application.id", ondelete="CASCADE", index=True)

talentro/messaging/topology.py

@@ -0,0 +1,103 @@
+import aio_pika
+
+from typing import List, Tuple, Optional
+
+FIVE_MINUTES = 5 * 60 * 1000
+FIFTEEN_MINUTES = 15 * 60 * 1000
+
+class TopologyConfig:
+    def __init__(
+        self,
+        exchange_name: str,
+        exchange_type: aio_pika.ExchangeType,
+        bindings: List[Tuple[str, str]],
+        dlx_name: str = None,
+        retry_config: Optional[dict] = None
+    ):
+        self.exchange_name = exchange_name
+        self.exchange_type = exchange_type
+        self.bindings = bindings
+        self.dlx_name = dlx_name
+        self.retry_config = retry_config
+
+    def get_max_retries(self, queue_name: str) -> int:
+        """Get max retries for a specific queue, default 3."""
+        if not self.retry_config:
+            return 3
+        return self.retry_config.get(queue_name, {}).get("max_retries", 3)
+
+
+FRONTEND_TOPOLOGY = TopologyConfig(
+    exchange_name="x.frontend",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        ("q.frontend.updates.gateway", "frontend.update"),
+    ],
+    retry_config={
+        "q.frontend.updates.gateway": {"enabled": False}
+    }
+)
+
+
+APPLICATIONS_TOPOLOGY = TopologyConfig(
+    exchange_name="x.applications",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        # Integrations
+        ("q.applications.raw.integrations", "applications.raw"),
+        ("q.applications.send.integrations", "applications.send"),
+
+        # Acquisitions
+        ("q.applications.saved.acquisitions", "applications.saved"),
+
+        # Vacancies
+        ("q.applications.saved.vacancies", "applications.saved"),
+
+        # Candidates
+        ("q.applications.normalized.candidates", "applications.normalized"),
+        ("q.applications.external.upserted.candidates", "applications.external.upserted"),
+    ],
+    dlx_name="x.applications.dlx",
+    retry_config={
+        # Integrations
+        "q.applications.raw.integrations": {"enabled": True, "ttl_ms": FIVE_MINUTES},
+        "q.applications.send.integrations": {"enabled": True, "ttl_ms": FIVE_MINUTES},
+
+        # Acquisitions
+        "q.applications.saved.acquisitions": {"enabled": True, "ttl_ms": FIVE_MINUTES},
+
+        # Vacancies
+        "q.applications.saved.vacancies": {"enabled": True, "ttl_ms": FIVE_MINUTES},
+
+        # Candidates
+        "q.applications.normalized.candidates": {"enabled": True, "ttl_ms": FIVE_MINUTES},
+        "q.applications.external.upserted.candidates": {"enabled": True, "ttl_ms": FIVE_MINUTES}
+    }
+)
+
+
+FEEDS_TOPOLOGY = TopologyConfig(
+    exchange_name="x.feeds",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        # Acquisitions
+        ("q.feeds.sync.acquisitions", "feeds.sync"),
+    ],
+    dlx_name="x.feeds.dlx",
+    retry_config={
+        "q.feeds.sync.acquisitions": {"enabled": True, "ttl_ms": FIVE_MINUTES}
+    }
+)
+
+BILLING_TOPOLOGY = TopologyConfig(
+    exchange_name="x.billing",
+    exchange_type=aio_pika.ExchangeType.TOPIC,
+    bindings=[
+        # Billing
+        ("q.billing.events.billing", "billing.event"),
+    ],
+    dlx_name="x.billing.dlx",
+    retry_config={
+        "q.billing.events.billing": {"enabled": True, "ttl_ms": FIVE_MINUTES, "max_retries": 3}
+    }
+)

talentro/services/rabbitmq.py

@@ -1,11 +1,14 @@
 import asyncio
 import os
+import traceback
 from typing import List, Tuple, Optional, Callable
 
 import aio_pika
 from aiormq import AMQPConnectionError
 
 from ..event import Message, Event
+from ..messaging.topology import TopologyConfig, FRONTEND_TOPOLOGY, BILLING_TOPOLOGY, FEEDS_TOPOLOGY, \
+    APPLICATIONS_TOPOLOGY
 from ..util.singleton import SingletonMeta
 
 
@@ -43,7 +46,8 @@ class RabbitMQ:
         exchange_type: aio_pika.ExchangeType = aio_pika.ExchangeType.TOPIC,
         durable: bool = True,
         bindings: List[Tuple[str, str]] = (),
-        dlx_name: Optional[str] = None
+        dlx_name: Optional[str] = None,
+        retry_config: Optional[dict] = None  # { "queue_name": {"retry_enabled": True, "ttl_ms": 300000}, ... }
     ):
         exchange = await self.channel.declare_exchange(
             exchange_name, exchange_type, durable=durable
@@ -73,7 +77,36 @@ class RabbitMQ:
                 await dlq.bind(dlx_name, routing_key=f"{queue_name}.dead")
                 print(f"  RabbitMQ - DLQ bound: {dlq.name} ← {dlx_name}")
 
-    async def consume(self, exchange_name: str, queue: str, callback: Callable, dlx_name: Optional[str] = None):
+                queue_retry_config = retry_config.get(queue_name, {})
+                if queue_retry_config.get("enabled", False):
+                    ttl_ms = queue_retry_config.get("ttl_ms", 300000)
+
+                    retry_exchange = await self.channel.declare_exchange(
+                        f"{dlx_name}.retry", aio_pika.ExchangeType.DIRECT, durable=True
+                    )
+
+                    retry_queue_args = {
+                        "x-dead-letter-exchange": exchange_name,
+                        "x-dead-letter-routing-key": queue_name,
+                        "x-message-ttl": ttl_ms
+                    }
+                    retry_queue = await self.channel.declare_queue(
+                        f"{queue_name}.retry",
+                        durable=True,
+                        arguments=retry_queue_args
+                    )
+                    await retry_queue.bind(retry_exchange, routing_key=f"{queue_name}.retry")
+                    print(f"  RabbitMQ - Retry queue enabled: {retry_queue.name} "
+                          f"(TTL: {ttl_ms}ms)")
+
+    async def consume(
+        self,
+        exchange_name: str,
+        queue: str,
+        callback: Callable,
+        dlx_name: Optional[str] = None,
+        max_retries: int = 3
+    ):
         while True:
             try:
                 print(f"  RabbitMQ - Consuming from queue: {queue}")
@@ -91,10 +124,17 @@ class RabbitMQ:
                     await queue_object.bind(exchange, routing_key=queue)
 
                 async for message in queue_object:
-                    async with message.process():
-                        event: Event = Event.decode(message.body)
-                        print(f"  RabbitMQ ({queue}) - [x] Received event: {event.meta.event_type} (trace: {event.meta.trace_id})")
-                        await callback(queue, event)
+                    try:
+                        async with message.process():
+                            event: Event = Event.decode(message.body)
+                            print(f"  RabbitMQ ({queue}) - [x] Received event: {event.meta.event_type} (trace: {event.meta.trace_id})")
+                            await callback(queue, event)
+                    except Exception as e:
+                        # Handle callback exception - send to retry queue
+                        print(f"  RabbitMQ ({queue}) - ⚠️ Error processing message: {e}")
+                        traceback.print_exc()
+                        await self._handle_failed_message(message, queue, exchange_name, dlx_name, max_retries)
+
             except (aio_pika.exceptions.AMQPConnectionError, aio_pika.exceptions.ChannelClosed):
                 print(f"  RabbitMQ - Connection lost while consuming {queue}. Retrying in 5 seconds...")
                 await asyncio.sleep(5)
@@ -102,6 +142,45 @@ class RabbitMQ:
                 print(f"  RabbitMQ - Unexpected error in consumer {queue}: {e}")
                 await asyncio.sleep(5)
 
+    @staticmethod
+    async def _handle_failed_message(
+            message: aio_pika.IncomingMessage,
+            queue_name: str,
+            exchange_name: str,
+            dlx_name: Optional[str] = None,
+            max_retries: int = 3
+    ):
+        """
+        Handle failed messages by sending them to the retry queue or DLQ.
+        """
+        try:
+            # Get death count from headers
+            x_death = message.headers.get('x-death', [])
+            death_count = 0
+
+            if x_death:
+                # x-death is a list of death info dicts
+                death_count = len(x_death)
+
+            print(f"  RabbitMQ ({queue_name}) - Retry attempt {death_count + 1}/{max_retries + 1}")
+
+            if death_count >= max_retries:
+                # Max retries exceeded - send to DLQ
+                print(f"  RabbitMQ ({queue_name}) - ❌ Max retries ({max_retries}) exceeded. Sending to DLQ.")
+                await message.nack(requeue=False)
+            else:
+                # Still have retries left - send to retry queue
+                if dlx_name:
+                    await message.nack(requeue=False)
+                    print(f"  RabbitMQ ({queue_name}) - Sending to retry queue...")
+                else:
+                    await message.nack(requeue=True)
+                    print(f"  RabbitMQ ({queue_name}) - Message requeued")
+
+        except Exception as e:
+            print(f"  RabbitMQ ({queue_name}) - Error handling failed message: {e}")
+            await message.nack(requeue=False)
+
     async def send_message(self, message: Message):
         # Haal de exchange op
         if message.exchange == "default":
@@ -131,6 +210,7 @@ class QueueContext(metaclass=SingletonMeta):
     def __init__(self):
         self._rabbit_mq = RabbitMQ()
         self._message_callbacks = []
+        self._topology_config: TopologyConfig | None = None
 
     async def connect(self):
         try:
@@ -140,22 +220,34 @@ class QueueContext(metaclass=SingletonMeta):
             await asyncio.sleep(5)
             await self.connect()
 
+    async def setup_topologies(self):
+        topologies = [FRONTEND_TOPOLOGY, APPLICATIONS_TOPOLOGY, FEEDS_TOPOLOGY, BILLING_TOPOLOGY]
+
+        for topology in topologies:
+            await self._setup_topology_with_config(topology)
+
+    async def _setup_topology_with_config(self, topology_config: TopologyConfig):
+        self._topology_config = topology_config
 
-    async def setup_topology(
-        self,
-        exchange_name: str,
-        exchange_type: aio_pika.ExchangeType = aio_pika.ExchangeType.TOPIC,
-        durable: bool = True,
-        bindings: List[Tuple[str, str]] = (),
-        dlx_name: Optional[str] = None
-    ):
         await self._rabbit_mq.setup_topology(
-            exchange_name, exchange_type, durable, bindings, dlx_name
+            topology_config.exchange_name,
+            topology_config.exchange_type,
+            bindings=topology_config.bindings,
+            dlx_name=topology_config.dlx_name,
+            retry_config=topology_config.retry_config
         )
 
-    async def start_consuming(self, exchange_name: str, queue: str, dlx_name: Optional[str] = None):
+    async def start_consuming(self, queue: str):
+        max_retries = self._topology_config.get_max_retries(queue) if self._topology_config else 3
+
         asyncio.create_task(
-            self._rabbit_mq.consume(exchange_name, queue, self._on_new_message, dlx_name=dlx_name)
+            self._rabbit_mq.consume(
+                self._topology_config.exchange_name,
+                queue,
+                self._on_new_message,
+                dlx_name=self._topology_config.dlx_name,
+                max_retries=max_retries
+            )
         )
 
     async def _on_new_message(self, queue: str, event: Event):

{talentro_commons-0.20.1.dist-info → talentro_commons-0.20.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: talentro-commons
-Version: 0.20.1
+Version: 0.20.3
 Summary: This package contains all globally used code, services, models and data structures for Talentro
 License: Proprietary
 Author: Emiel van Essen

{talentro_commons-0.20.1.dist-info → talentro_commons-0.20.3.dist-info}/RECORD

@@ -6,8 +6,8 @@ talentro/billing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 talentro/billing/dataclasses.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 talentro/billing/models.py,sha256=rbmcJh-FcU5I3oFRhnD413YqIade1nxvqvksvEtCvMs,825
 talentro/candidates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-talentro/candidates/dataclasses.py,sha256=YHN7qRx1TXgH-zcwkQDB5Ai8saeNSQlNrLvLSKajHoU,6619
-talentro/candidates/models.py,sha256=K4MH42ADd_L8i4jBAC35HZFZaHSvIFXS9N8k79CQ7h8,2885
+talentro/candidates/dataclasses.py,sha256=oDlI9-Pv4XAXdJ6aWW42o6mbOGGTSiyOKZy9wE0EbQ0,6696
+talentro/candidates/models.py,sha256=2FktzPjsL8YJw4JTazzby5Q4LX5UdAutaSigIXpJJmc,2952
 talentro/constants.py,sha256=08CuJ1qVbgi4IDZThw7CQt_D65jGwekGrkMEg8eDGZo,861
 talentro/event.py,sha256=Xie-nosLwEpg35Hir9yCKtJBXM-_R4O1fyknOtG_6IY,4595
 talentro/exceptions.py,sha256=-0i7G0-IF3PeWKBMMQ6eq2Hb-Q3OHt-cCqVgxPiWiiQ,1226
@@ -21,13 +21,15 @@ talentro/iam/types.py,sha256=xJjujucc_MJitwPSB3QVFR4c5YZXJLM5cS-ym6rv_dU,206
 talentro/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 talentro/integrations/dataclasses.py,sha256=FxYbtglKvp_BAufQSCw-IO3wraJUuMvoKZnwJMd5W6w,4482
 talentro/integrations/models.py,sha256=iNgmAahbonQj2EVd1776xcYzv-1mtAkzGvgDdbjocjo,1292
+talentro/messaging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+talentro/messaging/topology.py,sha256=veA1FFjygfw9KcgG0LecMNZyeT_-B6m9AWvwZCYfcFM,3158
 talentro/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 talentro/services/billing.py,sha256=8pX57-tP7HyRqB_afB-CpEJXiZzMTJyjLyDI3vLvQsI,1989
 talentro/services/caching.py,sha256=mmjhXAMJ_g8D8cJqn15YqZ7ST-G5lD9MS-PmlowI7pU,2921
 talentro/services/clients.py,sha256=vluOrdYdDAQLyGR9-EmogLjA9OUlJtHy0tYD9LhwxKg,2174
 talentro/services/db.py,sha256=cnKCrYG7GwIu7ZZhA25D-yaXaiCJqPpzfcanWquyrBY,822
 talentro/services/google_storage.py,sha256=5r5uiDZD-76Dylc7yyRG5Ni4XNTc9xK8rC0glCRG8_8,6014
-talentro/services/rabbitmq.py,sha256=RNIsLqXM48z5SRRqJ3iWqhRp6FL9TwRwEeii_m8-xDo,6854
+talentro/services/rabbitmq.py,sha256=7qEKBc5Nk2kdjQzQVbyCgPrV3doJ4jIhKmVAWh6QN-k,10745
 talentro/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 talentro/util/attributes.py,sha256=PgJnn9LMtHkiNIaMov2HQt5944HweD6gRlAHBZrJGPA,448
 talentro/util/enum.py,sha256=YftsoJKnrn8HAjA2Q1tqIYBMrNnlq6m1b34N8hfykbE,155
@@ -39,6 +41,6 @@ talentro/vacancies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 talentro/vacancies/dataclasses.py,sha256=E6H5fsZH4IwtBFSLDolzF6u39tEIrANtWiNVoB65P0c,15074
 talentro/vacancies/models.py,sha256=GoXr71CNzU6csf8gdmv84etb3Rm-Cdfigp1yqPI_jjQ,4822
 talentro/vacancies/taxanomy.py,sha256=B6DMq9Wbs0aXFwD9aZveSlLwAC9eq1iCO_KM-FmrV7M,6384
-talentro_commons-0.20.1.dist-info/METADATA,sha256=mgvU_osYdjYDXkE2BG1WnnY0zi1rE3SIz1n2OwsWZeU,1507
-talentro_commons-0.20.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-talentro_commons-0.20.1.dist-info/RECORD,,
+talentro_commons-0.20.3.dist-info/METADATA,sha256=WjYJUoNIpQ0gWYlxuYzy-VBv3_tZqH1_Q8i2_v77kAA,1507
+talentro_commons-0.20.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+talentro_commons-0.20.3.dist-info/RECORD,,