kiba-core 0.5.3.dev91__tar.gz → 0.5.3.dev93__tar.gz

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 
 ### Added
+- [MAJOR] Added `priority` to `SqlMessageQueue.send_message` and `send_messages` (default 0): visible messages are claimed highest priority first, then by `visibleDate`, and a duplicate send, or a retry folded into a pending duplicate, keeps the higher priority. `tbl_queue_messages` has a new `priority` column (`INTEGER NOT NULL DEFAULT 0`), so apps must add it with a migration before upgrading
 - [PATCH] Fixed `DatabaseConnectionMiddleware`'s commit when the response starts failing ("another operation is in progress") for streaming responses behind a `BaseHTTPMiddleware`: `Database.execute` and `Database.commit_context_connection` now wait for each other on the same connection
 - [MINOR] Added `Database.commit_context_connection` to commit the context connection's writes so far; anything run on it afterwards is committed when the context connection ends
 - [PATCH] Updated `DatabaseConnectionMiddleware` to commit before the response starts, so a client can't get the response and make its next request before the request's writes are visible (writes made while streaming are still committed when the request ends)

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.14.8-slim
+FROM python:3.14.9-slim
 
 RUN apt-get update && apt-get install --yes --no-install-recommends make git
 

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev91
+Version: 0.5.3.dev93
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12
@@ -9,36 +9,36 @@ Requires-Dist: aiofiles>=25.1.0
 Requires-Dist: pydantic>=2.14.1
 Requires-Dist: orjson>=3.13.0
 Provides-Extra: api
-Requires-Dist: fastapi>=0.146.0; extra == "api"
+Requires-Dist: fastapi>=0.162.0; extra == "api"
 Requires-Dist: uvicorn[standard]>=0.54.0; extra == "api"
 Provides-Extra: core-api
-Requires-Dist: starlette>=1.9.0; extra == "core-api"
+Requires-Dist: starlette>=1.10.0; extra == "core-api"
 Requires-Dist: uvicorn[standard]>=0.54.0; extra == "core-api"
 Requires-Dist: python-multipart>=0.0.32; extra == "core-api"
 Provides-Extra: storage
-Requires-Dist: aiobotocore>=3.9.2; extra == "storage"
+Requires-Dist: aiobotocore>=3.10.0; extra == "storage"
 Provides-Extra: queues
-Requires-Dist: aiobotocore>=3.9.2; extra == "queues"
+Requires-Dist: aiobotocore>=3.10.0; extra == "queues"
 Provides-Extra: queue-sqs
-Requires-Dist: aiobotocore>=3.9.2; extra == "queue-sqs"
+Requires-Dist: aiobotocore>=3.10.0; extra == "queue-sqs"
 Provides-Extra: queue-aqs
 Requires-Dist: azure-storage-queue[aio]>=12.18.0; extra == "queue-aqs"
 Provides-Extra: queue-cosmos
 Requires-Dist: azure-cosmos>=4.17.1; extra == "queue-cosmos"
 Provides-Extra: database-psql
-Requires-Dist: sqlalchemy[asyncio]>=2.1.5; extra == "database-psql"
+Requires-Dist: sqlalchemy[asyncio]>=2.1.6; extra == "database-psql"
 Requires-Dist: asyncpg>=0.32.0; extra == "database-psql"
 Provides-Extra: database-sqlite
-Requires-Dist: sqlalchemy[asyncio]>=2.1.5; extra == "database-sqlite"
+Requires-Dist: sqlalchemy[asyncio]>=2.1.6; extra == "database-sqlite"
 Requires-Dist: aiosqlite>=0.22.1; extra == "database-sqlite"
 Provides-Extra: requester
-Requires-Dist: httpx2>=2.14.0; extra == "requester"
+Requires-Dist: httpx2>=2.16.0; extra == "requester"
 Provides-Extra: web3
 Requires-Dist: web3<8.0.0,>=7.16.0; extra == "web3"
 Provides-Extra: cron
 Requires-Dist: apscheduler<4,>=3.11.3; extra == "cron"
 Provides-Extra: types
-Requires-Dist: types-aiobotocore[essential]>=3.9.2; extra == "types"
+Requires-Dist: types-aiobotocore[essential]>=3.10.0; extra == "types"
 Requires-Dist: types-aiofiles>=25.1.0.20260518; extra == "types"
 
 # Core-Py

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/core/queues/sql.py

@@ -52,6 +52,7 @@ QueueMessagesTable = sqlalchemy.Table(
     sqlalchemy.Column(key='startedDate', name='started_date', type_=sqlalchemy.DateTime(timezone=True), nullable=True),
     sqlalchemy.Column(key='completedDate', name='completed_date', type_=sqlalchemy.DateTime(timezone=True), nullable=True),
     sqlalchemy.Column(key='lastError', name='last_error', type_=sqlalchemy.Text, nullable=True),
+    sqlalchemy.Column(key='priority', name='priority', type_=sqlalchemy.Integer, nullable=False, server_default='0'),
     sqlalchemy.Index('tbl_queue_messages_idx_queue_name_status_visible_date', 'queueName', 'status', 'visibleDate'),
     sqlalchemy.Index('tbl_queue_messages_ux_queue_name_deduplication_id_pending', 'queueName', 'deduplicationId', unique=True, postgresql_where=_PENDING_PREDICATE, sqlite_where=_PENDING_PREDICATE),
 )
@@ -109,10 +110,10 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
             return sqlalchemy_sqlite.insert(self.table)
         raise InternalServerErrorException(message=f'SqlMessageQueue does not support dialect: {dialectName}')
 
-    async def send_message(self, message: Message, delaySeconds: int = 0) -> None:
-        await self.send_messages(messages=[message], delaySeconds=delaySeconds)
+    async def send_message(self, message: Message, delaySeconds: int = 0, priority: int = 0) -> None:
+        await self.send_messages(messages=[message], delaySeconds=delaySeconds, priority=priority)
 
-    async def send_messages(self, messages: Sequence[Message], delaySeconds: int = 0) -> None:
+    async def send_messages(self, messages: Sequence[Message], delaySeconds: int = 0, priority: int = 0) -> None:
         if not messages:
             return
         now = self.database.now()
@@ -136,13 +137,17 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
                     'status': MESSAGE_STATUS_PENDING,
                     'visibleDate': self.database.now(seconds=delaySeconds),
                     'createdDate': now,
+                    'priority': priority,
                 }
             )
         async with self.database.create_transaction() as connection:
             insertQuery = self._create_insert(dialectName=connection.dialect.name).values(rows)
             earliest = sqlalchemy.func.least if connection.dialect.name == 'postgresql' else sqlalchemy.func.min
+            highest = sqlalchemy.func.greatest if connection.dialect.name == 'postgresql' else sqlalchemy.func.max
             upsertQuery = insertQuery.on_conflict_do_update(
-                index_elements=[self.table.c.queueName, self.table.c.deduplicationId], index_where=_PENDING_PREDICATE, set_={self.table.c.visibleDate: earliest(self.table.c.visibleDate, insertQuery.excluded.visibleDate)}
+                index_elements=[self.table.c.queueName, self.table.c.deduplicationId],
+                index_where=_PENDING_PREDICATE,
+                set_={self.table.c.visibleDate: earliest(self.table.c.visibleDate, insertQuery.excluded.visibleDate), self.table.c.priority: highest(self.table.c.priority, insertQuery.excluded.priority)},
             )
             await self.database.execute(query=upsertQuery, connection=connection)
 
@@ -175,7 +180,7 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
                 .where(self.table.c.queueName == self.queueName)
                 .where(self.table.c.status.in_([MESSAGE_STATUS_PENDING, MESSAGE_STATUS_RUNNING]))
                 .where(self.table.c.visibleDate <= now)
-                .order_by(self.table.c.visibleDate, self.table.c.id)
+                .order_by(self.table.c.priority.desc(), self.table.c.visibleDate, self.table.c.id)
                 .limit(limit)
                 .with_for_update(skip_locked=True)
             )
@@ -194,7 +199,7 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
             )
             result = await self.database.execute(query=claimQuery, connection=connection)
             rows = result.mappings().all()
-        return sorted((SqlMessage.from_row(row=row, table=self.table) for row in rows), key=lambda message: message.id)
+        return [SqlMessage.from_row(row=row, table=self.table) for row in sorted(rows, key=lambda row: (-row[self.table.c.priority], row[self.table.c.id]))]
 
     def _owned_update(self, message: SqlMessage) -> sqlalchemy.Update:
         return sqlalchemy.update(self.table).where(self.table.c.id == message.id).where(self.table.c.lockToken == message.lockToken).where(self.table.c.status == MESSAGE_STATUS_RUNNING)
@@ -237,6 +242,16 @@ class SqlMessageQueue(MessageQueue[SqlMessage]):
                     await self.database.execute(query=query, connection=connection)
             except sqlalchemy.exc.IntegrityError:
                 # NOTE(krishan711): an identical message was enqueued while this one ran, so it will do the work instead
+                highest = sqlalchemy.func.greatest if connection.dialect.name == 'postgresql' else sqlalchemy.func.max
+                runningPriority = sqlalchemy.select(self.table.c.priority).where(self.table.c.id == message.id).scalar_subquery()
+                duplicateQuery = (
+                    sqlalchemy.update(self.table)
+                    .where(self.table.c.queueName == self.queueName)
+                    .where(self.table.c.deduplicationId == message.deduplicationId)
+                    .where(self.table.c.status == MESSAGE_STATUS_PENDING)
+                    .values(priority=highest(self.table.c.priority, runningPriority))
+                )
+                await self.database.execute(query=duplicateQuery, connection=connection)
                 query = self._owned_update(message=message).values(status=MESSAGE_STATUS_DEDUPLICATED, lockToken=None, completedDate=self.database.now())
                 await self.database.execute(query=query, connection=connection)
 

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/kiba_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiba-core
-Version: 0.5.3.dev91
+Version: 0.5.3.dev93
 Summary: Kiba Labs' python utilities
 Project-URL: repository, https://github.com/kibalabs/core-py
 Requires-Python: <4,>=3.12
@@ -9,36 +9,36 @@ Requires-Dist: aiofiles>=25.1.0
 Requires-Dist: pydantic>=2.14.1
 Requires-Dist: orjson>=3.13.0
 Provides-Extra: api
-Requires-Dist: fastapi>=0.146.0; extra == "api"
+Requires-Dist: fastapi>=0.162.0; extra == "api"
 Requires-Dist: uvicorn[standard]>=0.54.0; extra == "api"
 Provides-Extra: core-api
-Requires-Dist: starlette>=1.9.0; extra == "core-api"
+Requires-Dist: starlette>=1.10.0; extra == "core-api"
 Requires-Dist: uvicorn[standard]>=0.54.0; extra == "core-api"
 Requires-Dist: python-multipart>=0.0.32; extra == "core-api"
 Provides-Extra: storage
-Requires-Dist: aiobotocore>=3.9.2; extra == "storage"
+Requires-Dist: aiobotocore>=3.10.0; extra == "storage"
 Provides-Extra: queues
-Requires-Dist: aiobotocore>=3.9.2; extra == "queues"
+Requires-Dist: aiobotocore>=3.10.0; extra == "queues"
 Provides-Extra: queue-sqs
-Requires-Dist: aiobotocore>=3.9.2; extra == "queue-sqs"
+Requires-Dist: aiobotocore>=3.10.0; extra == "queue-sqs"
 Provides-Extra: queue-aqs
 Requires-Dist: azure-storage-queue[aio]>=12.18.0; extra == "queue-aqs"
 Provides-Extra: queue-cosmos
 Requires-Dist: azure-cosmos>=4.17.1; extra == "queue-cosmos"
 Provides-Extra: database-psql
-Requires-Dist: sqlalchemy[asyncio]>=2.1.5; extra == "database-psql"
+Requires-Dist: sqlalchemy[asyncio]>=2.1.6; extra == "database-psql"
 Requires-Dist: asyncpg>=0.32.0; extra == "database-psql"
 Provides-Extra: database-sqlite
-Requires-Dist: sqlalchemy[asyncio]>=2.1.5; extra == "database-sqlite"
+Requires-Dist: sqlalchemy[asyncio]>=2.1.6; extra == "database-sqlite"
 Requires-Dist: aiosqlite>=0.22.1; extra == "database-sqlite"
 Provides-Extra: requester
-Requires-Dist: httpx2>=2.14.0; extra == "requester"
+Requires-Dist: httpx2>=2.16.0; extra == "requester"
 Provides-Extra: web3
 Requires-Dist: web3<8.0.0,>=7.16.0; extra == "web3"
 Provides-Extra: cron
 Requires-Dist: apscheduler<4,>=3.11.3; extra == "cron"
 Provides-Extra: types
-Requires-Dist: types-aiobotocore[essential]>=3.9.2; extra == "types"
+Requires-Dist: types-aiobotocore[essential]>=3.10.0; extra == "types"
 Requires-Dist: types-aiofiles>=25.1.0.20260518; extra == "types"
 
 # Core-Py

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/kiba_core.egg-info/requires.txt

@@ -3,11 +3,11 @@ pydantic>=2.14.1
 orjson>=3.13.0
 
 [api]
-fastapi>=0.146.0
+fastapi>=0.162.0
 uvicorn[standard]>=0.54.0
 
 [core-api]
-starlette>=1.9.0
+starlette>=1.10.0
 uvicorn[standard]>=0.54.0
 python-multipart>=0.0.32
 
@@ -15,11 +15,11 @@ python-multipart>=0.0.32
 apscheduler<4,>=3.11.3
 
 [database-psql]
-sqlalchemy[asyncio]>=2.1.5
+sqlalchemy[asyncio]>=2.1.6
 asyncpg>=0.32.0
 
 [database-sqlite]
-sqlalchemy[asyncio]>=2.1.5
+sqlalchemy[asyncio]>=2.1.6
 aiosqlite>=0.22.1
 
 [queue-aqs]
@@ -29,19 +29,19 @@ azure-storage-queue[aio]>=12.18.0
 azure-cosmos>=4.17.1
 
 [queue-sqs]
-aiobotocore>=3.9.2
+aiobotocore>=3.10.0
 
 [queues]
-aiobotocore>=3.9.2
+aiobotocore>=3.10.0
 
 [requester]
-httpx2>=2.14.0
+httpx2>=2.16.0
 
 [storage]
-aiobotocore>=3.9.2
+aiobotocore>=3.10.0
 
 [types]
-types-aiobotocore[essential]>=3.9.2
+types-aiobotocore[essential]>=3.10.0
 types-aiofiles>=25.1.0.20260518
 
 [web3]

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kiba-core"
-version = "0.5.3.dev91"
+version = "0.5.3.dev93"
 description = "Kiba Labs' python utilities"
 readme = "README.md"
 requires-python = ">=3.12, <4"
@@ -12,23 +12,23 @@ dependencies = [
 
 [project.optional-dependencies]
 api = [
-    "fastapi>=0.146.0",
+    "fastapi>=0.162.0",
     "uvicorn[standard]>=0.54.0",
 ]
 core-api = [
-    "starlette>=1.9.0",
+    "starlette>=1.10.0",
     "uvicorn[standard]>=0.54.0",
     "python-multipart>=0.0.32",
 ]
 storage = [
-    "aiobotocore>=3.9.2",
+    "aiobotocore>=3.10.0",
 ]
 # TODO(krishan711): remove this in next major update
 queues = [
-    "aiobotocore>=3.9.2",
+    "aiobotocore>=3.10.0",
 ]
 queue-sqs = [
-    "aiobotocore>=3.9.2",
+    "aiobotocore>=3.10.0",
 ]
 queue-aqs = [
     "azure-storage-queue[aio]>=12.18.0",
@@ -37,15 +37,15 @@ queue-cosmos = [
     "azure-cosmos>=4.17.1",
 ]
 database-psql = [
-    "sqlalchemy[asyncio]>=2.1.5",
+    "sqlalchemy[asyncio]>=2.1.6",
     "asyncpg>=0.32.0",
 ]
 database-sqlite = [
-    "sqlalchemy[asyncio]>=2.1.5",
+    "sqlalchemy[asyncio]>=2.1.6",
     "aiosqlite>=0.22.1",
 ]
 requester = [
-    "httpx2>=2.14.0",
+    "httpx2>=2.16.0",
 ]
 web3 = [
     "web3>=7.16.0,<8.0.0",
@@ -54,13 +54,13 @@ cron = [
     "apscheduler>=3.11.3,<4",
 ]
 types = [
-    "types-aiobotocore[essential]>=3.9.2",
+    "types-aiobotocore[essential]>=3.10.0",
     "types-aiofiles>=25.1.0.20260518",
 ]
 
 [dependency-groups]
 dev = [
-    "kiba-build==0.1.11.dev20",
+    "kiba-build==0.1.11.dev22",
 ]
 
 [project.urls]

{kiba_core-0.5.3.dev91 → kiba_core-0.5.3.dev93}/tests/queues/test_sql_queue.py

@@ -116,6 +116,24 @@ async def test_duplicate_send_keeps_the_earliest_visibility(workers: tuple[SqlMe
     assert await workerA.get_message() is not None
 
 
+async def test_higher_priority_messages_are_claimed_before_older_ones(workers: tuple[SqlMessageQueue, SqlMessageQueue]):
+    workerA, _ = workers
+    await workerA.send_messages(messages=[_message(deduplicationId=None, userId=str(index)) for index in range(3)])
+    await workerA.send_message(message=_message(deduplicationId=None, userId='urgent'), priority=1)
+    messages = await workerA.get_messages(limit=2)
+    assert [message.content['userId'] for message in messages] == ['urgent', '0']
+
+
+async def test_duplicate_send_keeps_the_highest_priority(workers: tuple[SqlMessageQueue, SqlMessageQueue]):
+    workerA, _ = workers
+    await workerA.send_message(message=_message(deduplicationId=None, userId='other'))
+    await workerA.send_message(message=_message())
+    await workerA.send_message(message=_message(), priority=1)
+    await workerA.send_message(message=_message())
+    assert [(row['content']['userId'], row['priority']) for row in await _rows(queue=workerA)] == [('other', 0), ('user-1', 1)]
+    assert (await _claim(queue=workerA)).content['userId'] == 'user-1'
+
+
 async def test_processor_fails_message_even_when_notifications_fail(workers: tuple[SqlMessageQueue, SqlMessageQueue]):
     workerA, _ = workers
 
@@ -376,6 +394,15 @@ async def test_retry_is_folded_into_an_identical_pending_message(workers: tuple[
     assert [row['status'] for row in await _rows(queue=workerA)] == ['deduplicated', 'pending']
 
 
+async def test_retry_folded_into_a_lower_priority_duplicate_raises_its_priority(workers: tuple[SqlMessageQueue, SqlMessageQueue]):
+    workerA, _ = workers
+    await workerA.send_message(message=_message(), priority=1)
+    message = await _claim(queue=workerA)
+    await workerA.send_message(message=_message())
+    await workerA.retry_message(message=message)
+    assert [(row['status'], row['priority']) for row in await _rows(queue=workerA)] == [('deduplicated', 1), ('pending', 1)]
+
+
 async def test_release_hands_the_message_back_without_using_an_attempt(workers: tuple[SqlMessageQueue, SqlMessageQueue]):
     workerA, workerB = workers
     await workerA.send_message(message=_message())