kiba-core 0.5.3.dev85__tar.gz → 0.5.3.dev87__tar.gz

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/CHANGELOG.md

@@ -8,6 +8,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 
 ### Added
+- [MINOR] Updated `json_route` and `streaming_json_route` (and so `route`) to accept `multipart/form-data` bodies: form fields are used like JSON body fields and uploaded files are passed as `core.api.uploaded_file.UploadedFile` (`filename`, `contentType`, `content`), so request models can declare `file: UploadedFile`. A field given more than once raises `BadRequestException`. The `core-api` extra now installs `python-multipart`
+- [MAJOR] Added abstract `MessageQueue.release_message` to hand a claimed message back to be claimed again straight away without using an attempt (`SqlMessageQueue` and `CosmosMessageQueue` keep `postCount`; `SqsMessageQueue` and `AqsMessageQueue` make the message visible again, though SQS still counts the next receive towards its redrive policy)
+- [MAJOR] Added abstract `MessageQueue.delete_expired_messages`: `SqlMessageQueue` deletes `succeeded` and `deduplicated` messages older than `retentionSeconds` (the other queues remove messages on completion and return 0). `SqlMessageQueue.get_message_count` no longer deletes them and its `shouldSkipRemovingNonRetained` parameter is removed
+- [MINOR] Updated `MessageQueueProcessor` to shut down gracefully: `run`/`run_batches` handle SIGTERM and SIGINT (a second signal falls back to the default handler), as does `request_shutdown()`, by finishing the message being processed, releasing claimed messages not yet started and stopping instead of sleeping. A processed message whose task is cancelled is released rather than left until its lease expires
+- [MINOR] Added `deleteExpiredMessagesIntervalSeconds` (default 600) to `MessageQueueProcessor`, which calls `delete_expired_messages` at most this often while running, logging rather than raising on failure; added `shouldProcessInParallel` to `run_batches`
 - [MINOR] Added optional `proxyUrl` to `Requester` to send all its requests through an HTTP(S) proxy, e.g. to reach an API that allowlists a fixed egress IP
 - [MINOR] Added optional `timeoutSeconds` (default 10) to `RestEthClient` to set the timeout of each request, e.g. a shorter one for an archive node so a degraded node fails fast
 - [MINOR] Added `UNDEFINED`, `Undefined` and `UpdateValue[T]` to `core.util.typing_util` for partial updates: `UpdateValue[T]` is `T` or the `UNDEFINED` sentinel, so an update can tell "not provided" apart from an explicit `None`. Use it as a pydantic field default (`title: UpdateValue[str | None] = UNDEFINED`); request input never validates to `UNDEFINED` (a JSON `"UNDEFINED"` stays a string), and `x is not UNDEFINED` narrows `x` to `T` for type checkers

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev85
+Version: 0.5.3.dev87
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12
@@ -14,6 +14,7 @@ Requires-Dist: uvicorn[standard]>=0.54.0; extra == "api"
 Provides-Extra: core-api
 Requires-Dist: starlette>=1.7.0; extra == "core-api"
 Requires-Dist: uvicorn[standard]>=0.54.0; extra == "core-api"
+Requires-Dist: python-multipart>=0.0.29; extra == "core-api"
 Provides-Extra: storage
 Requires-Dist: aiobotocore>=3.9.2; extra == "storage"
 Provides-Extra: queues

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/api/json_route.py

@@ -7,10 +7,9 @@ from starlette.requests import Request
 
 from core.api.api_request import KibaApiRequest
 from core.api.api_response import KibaJSONResponse
+from core.api.request_body import read_request_body
 from core.exceptions import BadRequestException
 from core.exceptions import InternalServerErrorException
-from core.util import json_util
-from core.util.typing_util import JsonObject
 
 
 def json_route[ApiRequest: BaseModel, ApiResponse: BaseModel](
@@ -22,16 +21,7 @@ def json_route[ApiRequest: BaseModel, ApiResponse: BaseModel](
         async def async_wrapper(receivedRequest: Request) -> KibaJSONResponse:
             pathParams = receivedRequest.path_params
             queryParams = receivedRequest.query_params
-            bodyBytes = await receivedRequest.body()
-            if len(bodyBytes) == 0:
-                body: JsonObject = {}
-            else:
-                try:
-                    body = typing.cast(JsonObject, json_util.loads(bodyBytes.decode()))
-                except json_util.JsonDecodeException as exception:
-                    raise BadRequestException(f'Invalid JSON body: {exception}')
-                if not isinstance(body, dict):
-                    raise BadRequestException('Invalid JSON body: expected an object')
+            body = await read_request_body(request=receivedRequest)
             allParams = {**pathParams, **body, **queryParams}
             try:
                 requestParams = requestType(**allParams)

kiba_core-0.5.3.dev87/core/api/request_body.py

@@ -0,0 +1,31 @@
+import typing
+from collections.abc import Mapping
+
+from starlette.datastructures import UploadFile
+from starlette.requests import Request
+
+from core.api.uploaded_file import UploadedFile
+from core.exceptions import BadRequestException
+from core.util import json_util
+from core.util.typing_util import JsonObject
+
+
+async def read_request_body(request: Request) -> Mapping[str, object]:
+    if request.headers.get('Content-Type', '').startswith('multipart/form-data'):
+        form = await request.form()
+        body: dict[str, object] = {}
+        for key, value in form.multi_items():
+            if key in body:
+                raise BadRequestException(f'Invalid form body: {key} is given more than once')
+            body[key] = UploadedFile(filename=value.filename, contentType=value.content_type, content=await value.read()) if isinstance(value, UploadFile) else value
+        return body
+    bodyBytes = await request.body()
+    if len(bodyBytes) == 0:
+        return {}
+    try:
+        jsonBody = typing.cast(JsonObject, json_util.loads(bodyBytes.decode()))
+    except json_util.JsonDecodeException as exception:
+        raise BadRequestException(f'Invalid JSON body: {exception}')
+    if not isinstance(jsonBody, dict):
+        raise BadRequestException('Invalid JSON body: expected an object')
+    return jsonBody

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/api/streaming_json_route.py

@@ -9,10 +9,10 @@ from starlette.requests import Request
 from starlette.responses import StreamingResponse
 
 from core.api.api_request import KibaApiRequest
+from core.api.request_body import read_request_body
 from core.exceptions import BadRequestException
 from core.exceptions import InternalServerErrorException
 from core.util import json_util
-from core.util.typing_util import JsonObject
 
 
 async def _convert_to_json_generator[T: BaseModel](response_iterator: AsyncIterator[T], expectedType: typing.Type[T]) -> AsyncIterator[bytes]:
@@ -31,16 +31,7 @@ def streaming_json_route[ApiRequest: BaseModel, ApiResponse: BaseModel](
         async def async_wrapper(receivedRequest: Request) -> StreamingResponse:
             pathParams = receivedRequest.path_params
             queryParams = receivedRequest.query_params
-            bodyBytes = await receivedRequest.body()
-            if len(bodyBytes) == 0:
-                body: JsonObject = {}
-            else:
-                try:
-                    body = typing.cast(JsonObject, json_util.loads(bodyBytes.decode()))
-                except json_util.JsonDecodeException as exception:
-                    raise BadRequestException(f'Invalid JSON body: {exception}')
-                if not isinstance(body, dict):
-                    raise BadRequestException('Invalid JSON body: expected an object')
+            body = await read_request_body(request=receivedRequest)
             allParams = {**pathParams, **body, **queryParams}
             try:
                 requestParams = requestType(**allParams)

kiba_core-0.5.3.dev87/core/api/uploaded_file.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class UploadedFile(BaseModel):
+    filename: str | None
+    contentType: str | None
+    content: bytes

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/queues/aqs.py

@@ -113,6 +113,14 @@ class AqsMessageQueue(MessageQueue[AqsMessage]):
         message.popReceipt = updatedMessage.pop_receipt
         return True
 
+    async def release_message(self, message: AqsMessage) -> None:
+        # NOTE(krishan711): a message whose lease was already lost is left alone
+        await self.extend_message_lease(message=message, expectedProcessingSeconds=0)
+
+    async def delete_expired_messages(self) -> int:
+        # NOTE(krishan711): azure deletes messages when they are completed and expires the rest itself
+        return 0
+
     async def get_message_count(self) -> int:
         if not self._aqsClient:
             raise InternalServerErrorException('You need to call .connect() before trying to count messages')

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/queues/cosmos.py

@@ -258,6 +258,27 @@ class CosmosMessageQueue(MessageQueue[CosmosMessage]):
         message.etag = str(updatedItem['_etag'])
         return True
 
+    async def release_message(self, message: CosmosMessage) -> None:
+        try:
+            await self.container.patch_item(
+                item=message.id,
+                partition_key=self.queueName,
+                patch_operations=[
+                    {'op': 'set', 'path': '/leaseId', 'value': None},
+                    {'op': 'set', 'path': '/visibleDate', 'value': time.time()},
+                ],
+                etag=message.etag,
+                match_condition=MatchConditions.IfNotModified,
+            )
+        except cosmos_exceptions.CosmosHttpResponseError as exception:
+            # NOTE(krishan711): 412 means another worker now owns the message and 404 that it was already completed
+            if exception.status_code not in {404, 412}:
+                raise
+
+    async def delete_expired_messages(self) -> int:
+        # NOTE(krishan711): cosmos keeps no message history, completed messages are removed straight away
+        return 0
+
     async def get_message_count(self) -> int:
         counts = self.container.query_items(
             query='SELECT COUNT(1) AS messageCount FROM c WHERE c.itemType = @itemType AND c.queueName = @queueName AND c.visibleDate <= @now',

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/queues/message_queue.py

@@ -47,6 +47,15 @@ class MessageQueue[MessageType: Message](ABC):
     async def extend_message_lease(self, message: MessageType, expectedProcessingSeconds: int) -> bool:
         raise NotImplementedError
 
+    @abc.abstractmethod
+    async def release_message(self, message: MessageType) -> None:
+        # NOTE(krishan711): hands a claimed message back to be claimed again straight away without using an attempt, e.g. when the worker shuts down
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    async def delete_expired_messages(self) -> int:
+        raise NotImplementedError
+
     @abc.abstractmethod
     async def get_message_count(self) -> int:
         raise NotImplementedError

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/queues/message_queue_processor.py

@@ -1,6 +1,7 @@
 import abc
 import asyncio
 import contextlib
+import signal
 import sys
 import time
 import urllib.parse as urlparse
@@ -18,6 +19,7 @@ from core.util.value_holder import RequestIdHolder
 
 LOCKED_MAX_RETRY_COUNT = 3
 LOCKED_RETRY_DELAY_SECONDS = 30
+SHUTDOWN_SIGNALS = (signal.SIGTERM, signal.SIGINT)
 
 
 class MessageProcessor(ABC):
@@ -40,14 +42,30 @@ class MessageLeaseLostException(InternalServerErrorException):
 
 
 class MessageQueueProcessor[MessageType: Message]:
-    def __init__(self, queue: MessageQueue[MessageType], messageProcessor: MessageProcessor, notificationClients: list[NotificationClient], requestIdHolder: RequestIdHolder | None = None, pollLogIntervalSeconds: float = 600, shouldNotifyRetriedFailures: bool = True) -> None:
+    def __init__(
+        self,
+        queue: MessageQueue[MessageType],
+        messageProcessor: MessageProcessor,
+        notificationClients: list[NotificationClient],
+        requestIdHolder: RequestIdHolder | None = None,
+        pollLogIntervalSeconds: float = 600,
+        shouldNotifyRetriedFailures: bool = True,
+        deleteExpiredMessagesIntervalSeconds: float = 600,
+    ) -> None:
         self.queue = queue
         self.messageProcessor = messageProcessor
         self.notificationClients = notificationClients
         self.requestIdHolder = requestIdHolder
         self.pollLogIntervalSeconds = pollLogIntervalSeconds
         self.shouldNotifyRetriedFailures = shouldNotifyRetriedFailures
+        self.deleteExpiredMessagesIntervalSeconds = deleteExpiredMessagesIntervalSeconds
         self._lastPollLogTime: float | None = None
+        self._lastExpiredDeletionTime: float | None = None
+        self._shutdownEvent = asyncio.Event()
+
+    def request_shutdown(self) -> None:
+        # NOTE(krishan711): stops claiming new messages; the message being processed finishes and claimed messages not yet started are released
+        self._shutdownEvent.set()
 
     async def _handle_failure(self, message: MessageType, exception: Exception, requestId: str) -> int:
         logging.error('Caught exception whilst processing message:')
@@ -95,6 +113,7 @@ class MessageQueueProcessor[MessageType: Message]:
         # Cancellation (and other non-Exception errors) must still propagate, so those are never replaced.
         stopKeepAliveEvent = asyncio.Event()
         keepAliveTask = asyncio.create_task(self._keep_message_alive(message=message, expectedProcessingSeconds=expectedProcessingSeconds, stopEvent=stopKeepAliveEvent))
+        isLeaseLost = False
         try:
             try:
                 await self.messageProcessor.process_message(message=message)
@@ -105,6 +124,10 @@ class MessageQueueProcessor[MessageType: Message]:
                 if isLeaseLost and (inFlightException is None or isinstance(inFlightException, Exception)):
                     raise MessageLeaseLostException
             await self.queue.complete_message(message=message)
+        except asyncio.CancelledError:
+            if not isLeaseLost:
+                await self._release_messages(messages=[message])
+            raise
         except MessageNeedsReprocessingException as exception:
             logging.info(msg=f'Scheduling reprocessing for message:{message.command} due to: {exception.originalException!s}')
             await self.queue.retry_message(message=message, delaySeconds=((message.postCount or 0) * exception.delaySeconds))
@@ -122,16 +145,44 @@ class MessageQueueProcessor[MessageType: Message]:
         if self.requestIdHolder:
             self.requestIdHolder.set_value(value=None)
 
+    async def _release_messages(self, messages: list[MessageType]) -> None:
+        for message in messages:
+            try:
+                await self.queue.release_message(message=message)
+            except Exception as exception:  # noqa: BLE001
+                # NOTE(krishan711): a message that cannot be released is picked up again once its lease expires
+                logging.error(f'Failed to release message {message.command}:')
+                logging.exception(exception)
+
+    async def _delete_expired_messages_if_due(self) -> None:
+        now = time.monotonic()
+        if self._lastExpiredDeletionTime is not None and now - self._lastExpiredDeletionTime < self.deleteExpiredMessagesIntervalSeconds:
+            return
+        self._lastExpiredDeletionTime = now
+        try:
+            deletedCount = await self.queue.delete_expired_messages()
+        except Exception as exception:  # noqa: BLE001
+            logging.error('Failed to delete expired messages:')
+            logging.exception(exception)
+            return
+        if deletedCount > 0:
+            logging.info(f'Deleted {deletedCount} expired messages')
+
     async def execute_batch(self, batchSize: int, expectedProcessingSeconds: int = 300, longPollSeconds: int = 20, shouldProcessInParallel: bool = False) -> int:
         now = time.monotonic()
         if self._lastPollLogTime is None or now - self._lastPollLogTime >= self.pollLogIntervalSeconds:
             logging.info('Retrieving messages...')
             self._lastPollLogTime = now
         messages = await self.queue.get_messages(expectedProcessingSeconds=expectedProcessingSeconds, longPollSeconds=longPollSeconds, limit=batchSize)
-        if shouldProcessInParallel:
+        if self._shutdownEvent.is_set():
+            await self._release_messages(messages=messages)
+        elif shouldProcessInParallel:
             await asyncio.gather(*[self._process_message(message=message, expectedProcessingSeconds=expectedProcessingSeconds) for message in messages])
         else:
-            for message in messages:
+            for index, message in enumerate(messages):
+                if self._shutdownEvent.is_set():
+                    await self._release_messages(messages=messages[index:])
+                    break
                 await self._process_message(message=message, expectedProcessingSeconds=expectedProcessingSeconds)
         return len(messages)
 
@@ -139,14 +190,41 @@ class MessageQueueProcessor[MessageType: Message]:
         processedMessageCount = await self.execute_batch(batchSize=1, expectedProcessingSeconds=expectedProcessingSeconds, longPollSeconds=longPollSeconds)
         return processedMessageCount > 0
 
-    async def run_batches(self, batchSize: int, expectedProcessingSeconds: int = 300, longPollSeconds: int = 20, sleepTime: int = 30, totalMessageLimit: int | None = None) -> int:
+    def _on_shutdown_signal(self, signalNumber: int) -> None:
+        logging.info(f'Received {signal.Signals(signalNumber).name}, finishing the current message before shutting down')
+        self.request_shutdown()
+        # NOTE(krishan711): a second signal falls back to the default handler so an impatient interrupt still stops the process
+        loop = asyncio.get_running_loop()
+        for shutdownSignal in SHUTDOWN_SIGNALS:
+            loop.remove_signal_handler(shutdownSignal)
+
+    async def run_batches(
+        self,
+        batchSize: int,
+        expectedProcessingSeconds: int = 300,
+        longPollSeconds: int = 20,
+        sleepTime: int = 30,
+        totalMessageLimit: int | None = None,
+        shouldProcessInParallel: bool = False,
+    ) -> int:
+        loop = asyncio.get_running_loop()
+        for shutdownSignal in SHUTDOWN_SIGNALS:
+            loop.add_signal_handler(shutdownSignal, self._on_shutdown_signal, shutdownSignal)
         processedMessageCount = 0
-        while totalMessageLimit is None or processedMessageCount < totalMessageLimit:
-            innerProcessedMessageCount = await self.execute_batch(expectedProcessingSeconds=expectedProcessingSeconds, longPollSeconds=longPollSeconds, batchSize=batchSize)
-            if innerProcessedMessageCount == 0:
-                logging.info('No message received.. sleeping')
-                await asyncio.sleep(sleepTime)
-            processedMessageCount += innerProcessedMessageCount
+        try:
+            while not self._shutdownEvent.is_set() and (totalMessageLimit is None or processedMessageCount < totalMessageLimit):
+                await self._delete_expired_messages_if_due()
+                innerProcessedMessageCount = await self.execute_batch(expectedProcessingSeconds=expectedProcessingSeconds, longPollSeconds=longPollSeconds, batchSize=batchSize, shouldProcessInParallel=shouldProcessInParallel)
+                if innerProcessedMessageCount == 0:
+                    logging.info('No message received.. sleeping')
+                    with contextlib.suppress(TimeoutError):
+                        await asyncio.wait_for(self._shutdownEvent.wait(), timeout=sleepTime)
+                processedMessageCount += innerProcessedMessageCount
+        finally:
+            for shutdownSignal in SHUTDOWN_SIGNALS:
+                loop.remove_signal_handler(shutdownSignal)
+        if self._shutdownEvent.is_set():
+            logging.info('Shutdown requested, stopped processing messages')
         return processedMessageCount
 
     async def run(self, expectedProcessingSeconds: int = 300, longPollSeconds: int = 20, sleepTime: int = 30, totalMessageLimit: int | None = None) -> bool:

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/queues/sql.py

@@ -211,7 +211,7 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
             return result.first() is not None
 
     async def retry_message(self, message: SqlMessage, delaySeconds: int = 0) -> None:
-        await self._reschedule_message(message=message, delaySeconds=delaySeconds, extraValues={})
+        await self._reschedule_message(message=message, delaySeconds=delaySeconds, shouldCountAttempt=True)
 
     async def fail_message(self, message: SqlMessage, errorMessage: str | None) -> bool:
         postCount = message.postCount or 0
@@ -220,21 +220,39 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
                 query = self._owned_update(message=message).values(status=MESSAGE_STATUS_FAILED, lockToken=None, completedDate=self.database.now(), lastError=errorMessage)
                 await self.database.execute(query=query, connection=connection)
             return False
-        await self._reschedule_message(message=message, delaySeconds=self.failureRetryDelaySeconds * postCount, extraValues={'lastError': errorMessage})
+        await self._reschedule_message(message=message, delaySeconds=self.failureRetryDelaySeconds * postCount, shouldCountAttempt=True, extraValues={'lastError': errorMessage})
         return True
 
-    async def _reschedule_message(self, message: SqlMessage, delaySeconds: float, extraValues: dict[str, Any]) -> None:  # type: ignore[explicit-any]
+    async def release_message(self, message: SqlMessage) -> None:
+        await self._reschedule_message(message=message, delaySeconds=0, shouldCountAttempt=False)
+
+    async def _reschedule_message(self, message: SqlMessage, delaySeconds: float, shouldCountAttempt: bool, extraValues: dict[str, Any] | None = None) -> None:  # type: ignore[explicit-any]
+        values = {'status': MESSAGE_STATUS_PENDING, 'lockToken': None, 'visibleDate': self.database.now(seconds=delaySeconds), **(extraValues or {})}
+        if shouldCountAttempt:
+            values |= {'postCount': self.table.c.postCount + 1, 'postDate': self.database.now()}
         async with self.database.create_transaction() as connection:
             try:
                 async with connection.begin_nested():
-                    query = self._owned_update(message=message).values(status=MESSAGE_STATUS_PENDING, lockToken=None, visibleDate=self.database.now(seconds=delaySeconds), postCount=self.table.c.postCount + 1, postDate=self.database.now(), **extraValues)
+                    query = self._owned_update(message=message).values(values)
                     await self.database.execute(query=query, connection=connection)
             except sqlalchemy.exc.IntegrityError:
                 # NOTE(krishan711): an identical message was enqueued while this one ran, so it will do the work instead
                 query = self._owned_update(message=message).values(status=MESSAGE_STATUS_DEDUPLICATED, lockToken=None, completedDate=self.database.now())
                 await self.database.execute(query=query, connection=connection)
 
-    async def get_message_count(self, shouldSkipRemovingNonRetained: bool = False) -> int:
+    async def delete_expired_messages(self) -> int:
+        async with self.database.create_transaction() as connection:
+            query = (
+                sqlalchemy.delete(self.table)
+                .where(self.table.c.queueName == self.queueName)
+                .where(self.table.c.status.in_([MESSAGE_STATUS_SUCCEEDED, MESSAGE_STATUS_DEDUPLICATED]))
+                .where(self.table.c.completedDate < self.database.now(seconds=-self.retentionSeconds))
+                .returning(self.table.c.id)
+            )
+            result = await self.database.execute(query=query, connection=connection)
+            return len(result.all())
+
+    async def get_message_count(self) -> int:
         countQuery = (
             sqlalchemy.select(sqlalchemy.func.count())
             .select_from(self.table)
@@ -243,14 +261,6 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
             .where(self.table.c.visibleDate <= self.database.now())
         )
         async with self.database.create_transaction() as connection:
-            if not shouldSkipRemovingNonRetained:
-                deleteQuery = (
-                    sqlalchemy.delete(self.table)
-                    .where(self.table.c.queueName == self.queueName)
-                    .where(self.table.c.status.in_([MESSAGE_STATUS_SUCCEEDED, MESSAGE_STATUS_DEDUPLICATED]))
-                    .where(self.table.c.completedDate < self.database.now(seconds=-self.retentionSeconds))
-                )
-                await self.database.execute(query=deleteQuery, connection=connection)
             result = await self.database.execute(query=countQuery, connection=connection)
         return int(result.scalar_one())
 

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/core/queues/sqs.py

@@ -131,6 +131,14 @@ class SqsMessageQueue(MessageQueue[SqsMessage]):
             raise
         return True
 
+    async def release_message(self, message: SqsMessage) -> None:
+        # NOTE(krishan711): a message whose lease was already lost is left alone; SQS still counts the next receive towards the queue's redrive policy
+        await self.extend_message_lease(message=message, expectedProcessingSeconds=0)
+
+    async def delete_expired_messages(self) -> int:
+        # NOTE(krishan711): SQS deletes messages when they are completed and expires the rest itself
+        return 0
+
     async def get_message_count(self) -> int:
         if not self._sqsClient:
             raise InternalServerErrorException('You need to call .connect() before trying to count messages')

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/kiba_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev85
+Version: 0.5.3.dev87
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12
@@ -14,6 +14,7 @@ Requires-Dist: uvicorn[standard]>=0.54.0; extra == "api"
 Provides-Extra: core-api
 Requires-Dist: starlette>=1.7.0; extra == "core-api"
 Requires-Dist: uvicorn[standard]>=0.54.0; extra == "core-api"
+Requires-Dist: python-multipart>=0.0.29; extra == "core-api"
 Provides-Extra: storage
 Requires-Dist: aiobotocore>=3.9.2; extra == "storage"
 Provides-Extra: queues

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/kiba_core.egg-info/SOURCES.txt

@@ -26,11 +26,13 @@ core/api/health.py
 core/api/json_route.py
 core/api/openapi.py
 core/api/rate_limit.py
+core/api/request_body.py
 core/api/request_context.py
 core/api/route.py
 core/api/route_auth.py
 core/api/route_metadata.py
 core/api/streaming_json_route.py
+core/api/uploaded_file.py
 core/api/middleware/__init__.py
 core/api/middleware/database_connection_middleware.py
 core/api/middleware/exception_handling_middleware.py

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/kiba_core.egg-info/requires.txt

@@ -9,6 +9,7 @@ uvicorn[standard]>=0.54.0
 [core-api]
 starlette>=1.7.0
 uvicorn[standard]>=0.54.0
+python-multipart>=0.0.29
 
 [cron]
 apscheduler<4,>=3.11.3

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kiba-core"
-version = "0.5.3.dev85"
+version = "0.5.3.dev87"
 description = "Kiba Labs' python utilities"
 readme = "README.md"
 requires-python = ">=3.12, <4"
@@ -18,6 +18,7 @@ api = [
 core-api = [
     "starlette>=1.7.0",
     "uvicorn[standard]>=0.54.0",
+    "python-multipart>=0.0.29",
 ]
 storage = [
     "aiobotocore>=3.9.2",

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/tests/api/test_json_route.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel
 
 from core.api.json_route import json_route
 from core.api.api_request import KibaApiRequest
+from core.api.uploaded_file import UploadedFile
 from core.api.middleware.exception_handling_middleware import ExceptionHandlingMiddleware
 from core.api.request_context import RequestContext
 from core.api.request_context import RequestContextHolder
@@ -30,6 +31,18 @@ class ExampleResponse(BaseModel):
     path_echo: str | None = None
 
 
+class UploadRequest(BaseModel):
+    file: UploadedFile
+    age: int
+
+
+class UploadResponse(BaseModel):
+    filename: str | None
+    content_type: str | None
+    content: str
+    age: int
+
+
 @pytest.fixture
 def client():
     @json_route(requestType=ExampleRequest, responseType=ExampleResponse)
@@ -52,9 +65,20 @@ def client():
         )
 
 
+    @json_route(requestType=UploadRequest, responseType=UploadResponse)
+    async def test_upload_endpoint(request: KibaApiRequest[UploadRequest]) -> UploadResponse:
+        return UploadResponse(
+            filename=request.data.file.filename,
+            content_type=request.data.file.contentType,
+            content=request.data.file.content.decode(),
+            age=request.data.age,
+        )
+
+
     app = Starlette(routes=[
         Route("/test", test_endpoint, methods=["POST"]),
         Route("/test/{path_param}", test_path_endpoint, methods=["POST"]),
+        Route("/upload", test_upload_endpoint, methods=["POST"]),
     ])
     app.add_middleware(ExceptionHandlingMiddleware)
     app.add_middleware(RequestContextMiddleware, requestContextHolder=requestContextHolder, requestContextFactory=create_request_context)
@@ -145,3 +169,23 @@ def test_json_route_with_non_object_json(client, content):
     )
     assert response.status_code == 400
     assert "expected an object" in response.json().get("message", "")
+
+
+def test_json_route_with_multipart_body_passes_files_and_fields(client):
+    response = client.post(
+        "/upload",
+        files={"file": ("scan.dcm", b"file-content", "application/dicom")},
+        data={"age": "41"},
+    )
+    assert response.status_code == 200
+    assert response.json() == {"filename": "scan.dcm", "content_type": "application/dicom", "content": "file-content", "age": 41}
+
+
+def test_json_route_with_multipart_body_rejects_repeated_fields(client):
+    response = client.post(
+        "/upload",
+        files=[("file", ("first.dcm", b"first", "application/dicom")), ("file", ("second.dcm", b"second", "application/dicom"))],
+        data={"age": "41"},
+    )
+    assert response.status_code == 400
+    assert "file is given more than once" in response.json().get("message", "")

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/tests/queues/test_aqs_queue.py

@@ -43,6 +43,13 @@ async def _claim(queue: AqsMessageQueue, expectedProcessingSeconds: int = 300) -
     return message
 
 
+async def test_released_message_can_be_received_again_straight_away(workers: tuple[AqsMessageQueue, AqsMessageQueue]):
+    workerA, workerB = workers
+    await workerA.send_message(message=_message())
+    await workerA.release_message(message=await _claim(queue=workerA))
+    await _claim(queue=workerB)
+
+
 async def test_extended_message_stays_invisible_and_can_still_be_completed(workers: tuple[AqsMessageQueue, AqsMessageQueue]):
     workerA, workerB = workers
     await workerA.send_message(message=_message())

{kiba_core-0.5.3.dev85 → kiba_core-0.5.3.dev87}/tests/queues/test_cosmos_queue.py

@@ -85,6 +85,15 @@ async def test_completed_message_is_removed_and_can_be_sent_again(workers: tuple
     assert await workerA.get_message_count() == 1
 
 
+async def test_release_hands_the_message_back_without_using_an_attempt(workers: tuple[CosmosMessageQueue, CosmosMessageQueue]):
+    workerA, workerB = workers
+    await workerA.send_message(message=_message())
+    message = await _claim(queue=workerA)
+    await workerA.release_message(message=message)
+    releasedMessage = await _claim(queue=workerB)
+    assert (releasedMessage.id, releasedMessage.postCount) == (message.id, 1)
+
+
 async def test_retry_reschedules_the_same_message_with_a_delay(workers: tuple[CosmosMessageQueue, CosmosMessageQueue]):
     workerA, _ = workers
     await workerA.send_message(message=_message())