kiba-core 0.5.3.dev72__tar.gz → 0.5.3.dev73__tar.gz

{kiba_core-0.5.3.dev72 → kiba_core-0.5.3.dev73}/CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 
 ### Added
+- [MINOR] Added `pollLogIntervalSeconds` (default 600) to `MessageQueueProcessor`: `Retrieving messages...` is now logged on the first poll and then at most once per interval instead of on every poll, so it still shows the worker is alive without flooding the logs; pass `0` to log every poll as before
 - [MAJOR] Updated `RestEthClient` to raise `RpcServerException` (a `ServerException` keeping the original status code and message, with the original exception as its cause) when the node responds with a 5xx, so callers can tell node failures apart from other server errors; `RequesterTimeoutException` is still raised unchanged. Code catching `InternalServerErrorException` (or `BadGatewayException`, `ServiceUnavailableException`, …) around `RestEthClient` calls must catch `RpcServerException` instead
 - [MINOR] Added `RestEthClient.get_code`
 - [MINOR] Added optional `maxConcurrentRequestCount` and `semaphore` to `RestEthClient` to limit concurrent requests; pass the same `semaphore` to several clients to limit them together (it takes precedence over `maxConcurrentRequestCount`)

{kiba_core-0.5.3.dev72 → kiba_core-0.5.3.dev73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev72
+Version: 0.5.3.dev73
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12

{kiba_core-0.5.3.dev72 → kiba_core-0.5.3.dev73}/core/queues/message_queue_processor.py

@@ -40,11 +40,13 @@ class MessageLeaseLostException(InternalServerErrorException):
 
 
 class MessageQueueProcessor[MessageType: Message]:
-    def __init__(self, queue: MessageQueue[MessageType], messageProcessor: MessageProcessor, notificationClients: list[NotificationClient], requestIdHolder: RequestIdHolder | None = None) -> None:
+    def __init__(self, queue: MessageQueue[MessageType], messageProcessor: MessageProcessor, notificationClients: list[NotificationClient], requestIdHolder: RequestIdHolder | None = None, pollLogIntervalSeconds: float = 600) -> None:
         self.queue = queue
         self.messageProcessor = messageProcessor
         self.notificationClients = notificationClients
         self.requestIdHolder = requestIdHolder
+        self.pollLogIntervalSeconds = pollLogIntervalSeconds
+        self._lastPollLogTime: float | None = None
 
     async def _handle_failure(self, message: MessageType, exception: Exception, requestId: str) -> int:
         logging.error('Caught exception whilst processing message:')
@@ -117,7 +119,10 @@ class MessageQueueProcessor[MessageType: Message]:
             self.requestIdHolder.set_value(value=None)
 
     async def execute_batch(self, batchSize: int, expectedProcessingSeconds: int = 300, longPollSeconds: int = 20, shouldProcessInParallel: bool = False) -> int:
-        logging.info('Retrieving messages...')
+        now = time.monotonic()
+        if self._lastPollLogTime is None or now - self._lastPollLogTime >= self.pollLogIntervalSeconds:
+            logging.info('Retrieving messages...')
+            self._lastPollLogTime = now
         messages = await self.queue.get_messages(expectedProcessingSeconds=expectedProcessingSeconds, longPollSeconds=longPollSeconds, limit=batchSize)
         if shouldProcessInParallel:
             await asyncio.gather(*[self._process_message(message=message, expectedProcessingSeconds=expectedProcessingSeconds) for message in messages])

{kiba_core-0.5.3.dev72 → kiba_core-0.5.3.dev73}/kiba_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev72
+Version: 0.5.3.dev73
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12

{kiba_core-0.5.3.dev72 → kiba_core-0.5.3.dev73}/kiba_core.egg-info/SOURCES.txt

@@ -117,6 +117,7 @@ tests/locks/test_sql_lock.py
 tests/queues/conftest.py
 tests/queues/test_aqs_queue.py
 tests/queues/test_cosmos_queue.py
+tests/queues/test_message_queue_processor.py
 tests/queues/test_sql_queue.py
 tests/queues/test_sqs_queue.py
 tests/requester/__init__.py

{kiba_core-0.5.3.dev72 → kiba_core-0.5.3.dev73}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kiba-core"
-version = "0.5.3.dev72"
+version = "0.5.3.dev73"
 description = "Kiba Labs' python utilities"
 readme = "README.md"
 requires-python = ">=3.12, <4"

kiba_core-0.5.3.dev73/tests/queues/test_message_queue_processor.py

@@ -0,0 +1,46 @@
+import asyncio
+
+import pytest
+
+from core.queues.message_queue_processor import MessageProcessor
+from core.queues.message_queue_processor import MessageQueueProcessor
+from core.queues.model import Message
+
+
+class EmptyQueue:
+    async def get_messages(self, expectedProcessingSeconds: int, longPollSeconds: int, limit: int) -> list[Message]:  # noqa: ARG002
+        return []
+
+
+class NoopProcessor(MessageProcessor):
+    async def process_message(self, message: Message) -> None:
+        pass
+
+
+def _processor(pollLogIntervalSeconds: float) -> MessageQueueProcessor[Message]:
+    return MessageQueueProcessor(queue=EmptyQueue(), messageProcessor=NoopProcessor(), notificationClients=[], pollLogIntervalSeconds=pollLogIntervalSeconds)  # type: ignore[arg-type]
+
+
+def _pollLogCount(caplog: pytest.LogCaptureFixture) -> int:
+    return sum(1 for record in caplog.records if record.getMessage() == 'Retrieving messages...')
+
+
+@pytest.mark.asyncio
+async def test_poll_is_logged_once_per_interval(caplog: pytest.LogCaptureFixture) -> None:
+    caplog.set_level('INFO')
+    processor = _processor(pollLogIntervalSeconds=0.2)
+    await processor.execute_batch(batchSize=1)
+    await processor.execute_batch(batchSize=1)
+    assert _pollLogCount(caplog) == 1
+    await asyncio.sleep(0.25)
+    await processor.execute_batch(batchSize=1)
+    assert _pollLogCount(caplog) == 2  # noqa: PLR2004
+
+
+@pytest.mark.asyncio
+async def test_zero_interval_logs_every_poll(caplog: pytest.LogCaptureFixture) -> None:
+    caplog.set_level('INFO')
+    processor = _processor(pollLogIntervalSeconds=0)
+    for _ in range(3):
+        await processor.execute_batch(batchSize=1)
+    assert _pollLogCount(caplog) == 3  # noqa: PLR2004