kiba-core 0.5.3.dev73__tar.gz → 0.5.3.dev74__tar.gz

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/CHANGELOG.md

@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 
 ### Added
+- [MINOR] Added `shouldNotifyRetriedFailures` (default `True`) to `MessageQueueProcessor`: when `False`, failure notifications are only sent once the queue gives up on a message, not for failures it will retry. `MessageQueue.fail_message` now returns whether the queue scheduled another attempt: `SqlMessageQueue` and `CosmosMessageQueue` return `True` until `maxAttempts` is reached, while `SqsMessageQueue` and `AqsMessageQueue` always return `False` as the platform decides on redelivery, so they keep notifying every failure
+- [MAJOR] Updated `CosmosMessageQueue` to retry failed messages like `SqlMessageQueue`: added `maxAttempts` (default 3) and `failureRetryDelaySeconds` (default 60). `fail_message` reschedules the message with linear backoff and stores `lastError`, and a visible message whose lease expired counts as an attempt when it is claimed again. Once `postCount` reaches `maxAttempts` the message is removed, along with its deduplication item. Previously failed or abandoned messages reappeared once their lease expired and were retried forever
 - [MINOR] Added `pollLogIntervalSeconds` (default 600) to `MessageQueueProcessor`: `Retrieving messages...` is now logged on the first poll and then at most once per interval instead of on every poll, so it still shows the worker is alive without flooding the logs; pass `0` to log every poll as before
 - [MAJOR] Updated `RestEthClient` to raise `RpcServerException` (a `ServerException` keeping the original status code and message, with the original exception as its cause) when the node responds with a 5xx, so callers can tell node failures apart from other server errors; `RequesterTimeoutException` is still raised unchanged. Code catching `InternalServerErrorException` (or `BadGatewayException`, `ServiceUnavailableException`, …) around `RestEthClient` calls must catch `RpcServerException` instead
 - [MINOR] Added `RestEthClient.get_code`

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev73
+Version: 0.5.3.dev74
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/core/queues/aqs.py

@@ -96,9 +96,9 @@ class AqsMessageQueue(MessageQueue[AqsMessage]):
         await self.send_message(message=message, delaySeconds=delaySeconds)
         await self.complete_message(message=message)
 
-    async def fail_message(self, message: AqsMessage, errorMessage: str | None) -> None:
+    async def fail_message(self, message: AqsMessage, errorMessage: str | None) -> bool:  # noqa: ARG002
         # NOTE(krishan711): the message reappears after its visibility timeout
-        pass
+        return False
 
     async def extend_message_lease(self, message: AqsMessage, expectedProcessingSeconds: int) -> bool:
         if not self._aqsClient:

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/core/queues/cosmos.py

@@ -48,10 +48,12 @@ class CosmosMessage(Message):
 
 
 class CosmosMessageQueue(MessageQueue[CosmosMessage]):
-    def __init__(self, container: ContainerProxy, queueName: str, pollIntervalSeconds: float = 1.0) -> None:
+    def __init__(self, container: ContainerProxy, queueName: str, pollIntervalSeconds: float = 1.0, maxAttempts: int = 3, failureRetryDelaySeconds: int = 60) -> None:
         self.container = container
         self.queueName = queueName
         self.pollIntervalSeconds = pollIntervalSeconds
+        self.maxAttempts = maxAttempts
+        self.failureRetryDelaySeconds = failureRetryDelaySeconds
 
     async def connect(self) -> None:
         # The caller supplies a shared ContainerProxy and owns CosmosClient lifecycle.
@@ -148,6 +150,12 @@ class CosmosMessageQueue(MessageQueue[CosmosMessage]):
         messages: list[CosmosMessage] = []
         async for candidate in candidates:
             item = {key: value for key, value in candidate.items() if not key.startswith('_')}
+            # NOTE(krishan711): a visible message that still has a lease was abandoned by its worker, which counts as an attempt
+            if candidate.get('leaseId') is not None:
+                if (candidate.get('postCount') or 0) >= self.maxAttempts:
+                    await self._remove_message_if_unchanged(itemId=candidate['id'], etag=candidate['_etag'], deduplicationId=candidate.get('deduplicationId'))
+                    continue
+                item['postCount'] = (candidate.get('postCount') or 0) + 1
             leaseId = uuid.uuid4().hex
             item['leaseId'] = leaseId
             item['visibleDate'] = now + expectedProcessingSeconds
@@ -170,27 +178,38 @@ class CosmosMessageQueue(MessageQueue[CosmosMessage]):
                 break
         return messages
 
-    async def complete_message(self, message: CosmosMessage) -> None:
-        if message.deduplicationId is None:
+    async def _remove_message(self, itemId: str, etag: str, deduplicationId: str | None) -> None:
+        if deduplicationId is None:
             await self.container.delete_item(
-                item=message.id,
+                item=itemId,
                 partition_key=self.queueName,
-                etag=message.etag,
+                etag=etag,
                 match_condition=MatchConditions.IfNotModified,
             )
             return
 
-        deduplicationKey = f'{self.queueName}:{message.deduplicationId}'
+        deduplicationKey = f'{self.queueName}:{deduplicationId}'
         deduplicationItemId = f'dedup-{hashlib.sha256(deduplicationKey.encode()).hexdigest()}'
         await self.container.execute_item_batch(
             batch_operations=[
-                ('delete', (message.id,), {'if_match_etag': message.etag}),
+                ('delete', (itemId,), {'if_match_etag': etag}),
                 ('delete', (deduplicationItemId,)),
             ],
             partition_key=self.queueName,
         )
 
-    async def retry_message(self, message: CosmosMessage, delaySeconds: int = 0) -> None:
+    async def _remove_message_if_unchanged(self, itemId: str, etag: str, deduplicationId: str | None) -> None:
+        try:
+            await self._remove_message(itemId=itemId, etag=etag, deduplicationId=deduplicationId)
+        except (cosmos_exceptions.CosmosHttpResponseError, cosmos_exceptions.CosmosBatchOperationError) as exception:
+            # NOTE(krishan711): 412 means another worker changed the message first, so it now owns it
+            if exception.status_code != 412:  # noqa: PLR2004
+                raise
+
+    async def complete_message(self, message: CosmosMessage) -> None:
+        await self._remove_message(itemId=message.id, etag=message.etag, deduplicationId=message.deduplicationId)
+
+    async def _reschedule_message(self, message: CosmosMessage, delaySeconds: float, extraOperations: list[dict[str, Any]]) -> None:  # type: ignore[explicit-any]
         try:
             await self.container.patch_item(
                 item=message.id,
@@ -200,6 +219,7 @@ class CosmosMessageQueue(MessageQueue[CosmosMessage]):
                     {'op': 'set', 'path': '/visibleDate', 'value': time.time() + delaySeconds},
                     {'op': 'set', 'path': '/postCount', 'value': (message.postCount or 0) + 1},
                     {'op': 'set', 'path': '/postDate', 'value': date_util.datetime_from_now().isoformat()},
+                    *extraOperations,
                 ],
                 etag=message.etag,
                 match_condition=MatchConditions.IfNotModified,
@@ -209,9 +229,17 @@ class CosmosMessageQueue(MessageQueue[CosmosMessage]):
             if exception.status_code != 412:  # noqa: PLR2004
                 raise
 
-    async def fail_message(self, message: CosmosMessage, errorMessage: str | None) -> None:
-        # NOTE(krishan711): the message reappears once its lease expires
-        pass
+    async def retry_message(self, message: CosmosMessage, delaySeconds: int = 0) -> None:
+        await self._reschedule_message(message=message, delaySeconds=delaySeconds, extraOperations=[])
+
+    async def fail_message(self, message: CosmosMessage, errorMessage: str | None) -> bool:
+        postCount = message.postCount or 0
+        if postCount >= self.maxAttempts:
+            # NOTE(krishan711): cosmos keeps no message history, so a message out of attempts is removed like a completed one
+            await self._remove_message_if_unchanged(itemId=message.id, etag=message.etag, deduplicationId=message.deduplicationId)
+            return False
+        await self._reschedule_message(message=message, delaySeconds=self.failureRetryDelaySeconds * postCount, extraOperations=[{'op': 'set', 'path': '/lastError', 'value': errorMessage}])
+        return True
 
     async def extend_message_lease(self, message: CosmosMessage, expectedProcessingSeconds: int) -> bool:
         try:

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/core/queues/message_queue.py

@@ -39,7 +39,8 @@ class MessageQueue[MessageType: Message](ABC):
         raise NotImplementedError
 
     @abc.abstractmethod
-    async def fail_message(self, message: MessageType, errorMessage: str | None) -> None:
+    async def fail_message(self, message: MessageType, errorMessage: str | None) -> bool:
+        # NOTE(krishan711): returns whether the queue itself scheduled another attempt; queues where the platform decides on redelivery return False
         raise NotImplementedError
 
     @abc.abstractmethod

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/core/queues/message_queue_processor.py

@@ -40,25 +40,29 @@ class MessageLeaseLostException(InternalServerErrorException):
 
 
 class MessageQueueProcessor[MessageType: Message]:
-    def __init__(self, queue: MessageQueue[MessageType], messageProcessor: MessageProcessor, notificationClients: list[NotificationClient], requestIdHolder: RequestIdHolder | None = None, pollLogIntervalSeconds: float = 600) -> None:
+    def __init__(self, queue: MessageQueue[MessageType], messageProcessor: MessageProcessor, notificationClients: list[NotificationClient], requestIdHolder: RequestIdHolder | None = None, pollLogIntervalSeconds: float = 600, shouldNotifyRetriedFailures: bool = True) -> None:
         self.queue = queue
         self.messageProcessor = messageProcessor
         self.notificationClients = notificationClients
         self.requestIdHolder = requestIdHolder
         self.pollLogIntervalSeconds = pollLogIntervalSeconds
+        self.shouldNotifyRetriedFailures = shouldNotifyRetriedFailures
         self._lastPollLogTime: float | None = None
 
     async def _handle_failure(self, message: MessageType, exception: Exception, requestId: str) -> int:
         logging.error('Caught exception whilst processing message:')
         logging.exception(exception)
         kibaException = KibaException.from_exception(exception=exception)
-        await self.queue.fail_message(message=message, errorMessage=kibaException.message)
-        for client in self.notificationClients:
-            try:
-                await client.post(messageText=f'Error processing message: {message.command}\n```\n{requestId}\n{message.content}\n{kibaException.exceptionType}: {kibaException.message}```')
-            except Exception as notificationException:  # noqa: BLE001
-                logging.error('Failed to send message failure notification:')
-                logging.exception(notificationException)
+        isRetryScheduled = await self.queue.fail_message(message=message, errorMessage=kibaException.message)
+        if isRetryScheduled and not self.shouldNotifyRetriedFailures:
+            logging.info(msg=f'Skipping failure notification for message:{message.command} as another attempt is scheduled')
+        else:
+            for client in self.notificationClients:
+                try:
+                    await client.post(messageText=f'Error processing message: {message.command}\n```\n{requestId}\n{message.content}\n{kibaException.exceptionType}: {kibaException.message}```')
+                except Exception as notificationException:  # noqa: BLE001
+                    logging.error('Failed to send message failure notification:')
+                    logging.exception(notificationException)
         return exception.statusCode if isinstance(exception, KibaException) else 500
 
     async def _keep_message_alive(self, message: MessageType, expectedProcessingSeconds: int, stopEvent: asyncio.Event) -> bool:

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/core/queues/sql.py

@@ -212,14 +212,15 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
     async def retry_message(self, message: SqlMessage, delaySeconds: int = 0) -> None:
         await self._reschedule_message(message=message, delaySeconds=delaySeconds, extraValues={})
 
-    async def fail_message(self, message: SqlMessage, errorMessage: str | None) -> None:
+    async def fail_message(self, message: SqlMessage, errorMessage: str | None) -> bool:
         postCount = message.postCount or 0
         if postCount >= self.maxAttempts:
             async with self.database.create_transaction() as connection:
                 query = self._owned_update(message=message).values(status=MESSAGE_STATUS_FAILED, lockToken=None, completedDate=self.database.now(), lastError=errorMessage)
                 await self.database.execute(query=query, connection=connection)
-            return
+            return False
         await self._reschedule_message(message=message, delaySeconds=self.failureRetryDelaySeconds * postCount, extraValues={'lastError': errorMessage})
+        return True
 
     async def _reschedule_message(self, message: SqlMessage, delaySeconds: float, extraValues: dict[str, Any]) -> None:  # type: ignore[explicit-any]
         async with self.database.create_transaction() as connection:

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/core/queues/sqs.py

@@ -113,9 +113,9 @@ class SqsMessageQueue(MessageQueue[SqsMessage]):
         await self.send_message(message=message, delaySeconds=delaySeconds)
         await self.complete_message(message=message)
 
-    async def fail_message(self, message: SqsMessage, errorMessage: str | None) -> None:
+    async def fail_message(self, message: SqsMessage, errorMessage: str | None) -> bool:  # noqa: ARG002
         # NOTE(krishan711): the message reappears after its visibility timeout and the queue's redrive policy decides when to dead-letter it
-        pass
+        return False
 
     async def extend_message_lease(self, message: SqsMessage, expectedProcessingSeconds: int) -> bool:
         if not self._sqsClient:

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/kiba_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev73
+Version: 0.5.3.dev74
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kiba-core"
-version = "0.5.3.dev73"
+version = "0.5.3.dev74"
 description = "Kiba Labs' python utilities"
 readme = "README.md"
 requires-python = ">=3.12, <4"

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/tests/queues/test_cosmos_queue.py

@@ -130,11 +130,46 @@ async def test_extend_after_losing_the_lease_reports_it_lost(workers: tuple[Cosm
     assert await workerA.extend_message_lease(message=staleMessage, expectedProcessingSeconds=60) is False
 
 
-async def test_failed_message_reappears_once_its_lease_expires(workers: tuple[CosmosMessageQueue, CosmosMessageQueue]):
+async def test_failed_message_backs_off_then_is_removed_after_max_attempts(workers: tuple[CosmosMessageQueue, CosmosMessageQueue]):
     workerA, _ = workers
     await workerA.send_message(message=_message())
-    message = await _claim(queue=workerA)
-    await workerA.fail_message(message=message, errorMessage='boom')
+    for attempt in range(1, 4):
+        message = await _claim(queue=workerA)
+        assert message.postCount == attempt
+        isRetryScheduled = await workerA.fail_message(message=message, errorMessage=f'boom {attempt}')
+        assert isRetryScheduled == (attempt < 3)  # noqa: PLR2004
+        if attempt < 3:  # noqa: PLR2004
+            item = await workerA.container.read_item(item=message.id, partition_key=workerA.queueName)
+            assert item['lastError'] == f'boom {attempt}'
+            assert await workerA.get_message() is None
+            await _make_visible(queue=workerA, messageId=message.id)
+    assert await workerA.get_message_count() == 0
+    assert await workerA.get_inflight_message_count() == 0
+    await workerA.send_message(message=_message())
+    assert await workerA.get_message_count() == 1
+
+
+async def test_abandoned_message_counts_as_an_attempt_and_is_removed_after_max_attempts(workers: tuple[CosmosMessageQueue, CosmosMessageQueue]):
+    workerA, _ = workers
+    await workerA.send_message(message=_message())
+    for attempt in range(1, 4):
+        message = await _claim(queue=workerA)
+        assert message.postCount == attempt
+        await _make_visible(queue=workerA, messageId=message.id)
     assert await workerA.get_message() is None
-    await _make_visible(queue=workerA, messageId=message.id)
-    assert (await _claim(queue=workerA)).id == message.id
+    assert await workerA.get_message_count() == 0
+    await workerA.send_message(message=_message())
+    assert await workerA.get_message_count() == 1
+
+
+async def test_fail_after_losing_the_lease_leaves_the_new_holder_untouched(workers: tuple[CosmosMessageQueue, CosmosMessageQueue]):
+    workerA, workerB = workers
+    await workerA.send_message(message=_message())
+    staleMessage = await _claim(queue=workerA)
+    await _make_visible(queue=workerA, messageId=staleMessage.id)
+    newMessage = await _claim(queue=workerB)
+    staleMessage.postCount = 3
+    assert await workerA.fail_message(message=staleMessage, errorMessage='boom') is False
+    assert await workerA.get_inflight_message_count() == 1
+    await workerB.complete_message(message=newMessage)
+    assert await workerA.get_inflight_message_count() == 0

{kiba_core-0.5.3.dev73 → kiba_core-0.5.3.dev74}/tests/queues/test_sql_queue.py

@@ -132,6 +132,29 @@ async def test_processor_fails_message_even_when_notifications_fail(workers: tup
     assert (await _rows(queue=workerA))[0]['status'] == 'failed'
 
 
+@pytest.mark.parametrize(('shouldNotifyRetriedFailures', 'expectedNotificationCounts'), [(True, [1, 2, 3]), (False, [0, 0, 1])])
+async def test_processor_only_notifies_retried_failures_when_enabled(workers: tuple[SqlMessageQueue, SqlMessageQueue], shouldNotifyRetriedFailures: bool, expectedNotificationCounts: list[int]):
+    workerA, _ = workers
+    notifications: list[str] = []
+
+    class BrokenProcessor(MessageProcessor):
+        async def process_message(self, message: Message) -> None:
+            raise ValueError('boom')
+
+    class RecordingNotificationClient(NotificationClient):
+        async def post(self, messageText: str) -> KibaResponse:
+            notifications.append(messageText)
+            return typing.cast(KibaResponse, None)
+
+    processor = MessageQueueProcessor(queue=workerA, messageProcessor=BrokenProcessor(), notificationClients=[RecordingNotificationClient()], shouldNotifyRetriedFailures=shouldNotifyRetriedFailures)
+    await workerA.send_message(message=_message())
+    for expectedNotificationCount in expectedNotificationCounts:
+        assert await processor.execute(longPollSeconds=0)
+        assert len(notifications) == expectedNotificationCount
+        await _make_visible(queue=workerA, messageId=(await _rows(queue=workerA))[0]['id'])
+    assert (await _rows(queue=workerA))[0]['status'] == 'failed'
+
+
 async def test_concurrent_claims_never_hand_out_the_same_message(workers: tuple[SqlMessageQueue, SqlMessageQueue]):
     await workers[0].send_messages(messages=[_message(deduplicationId=None, userId=str(index)) for index in range(20)])
     results = await asyncio.gather(*[worker.get_messages(limit=3) for _ in range(5) for worker in workers])
@@ -257,7 +280,8 @@ async def test_failed_message_backs_off_then_fails_after_max_attempts(workers: t
     for attempt in range(1, 4):
         message = await _claim(queue=workerA)
         assert message.postCount == attempt
-        await workerA.fail_message(message=message, errorMessage=f'boom {attempt}')
+        isRetryScheduled = await workerA.fail_message(message=message, errorMessage=f'boom {attempt}')
+        assert isRetryScheduled == (attempt < 3)  # noqa: PLR2004
         row = await _row(queue=workerA, messageId=message.id)
         if attempt < 3:  # noqa: PLR2004
             assert row['status'] == 'pending'