taskiq-redis 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

taskiq_redis/list_schedule_source.py

@@ -25,6 +25,17 @@ class ListRedisScheduleSource(ScheduleSource):
         skip_past_schedules: bool = False,
         **connection_kwargs: Any,
     ) -> None:
+        """
+        Create a new schedule source.
+
+        :param url: Redis URL
+        :param prefix: Prefix for all the keys
+        :param max_connection_pool_size: Maximum size of the connection pool
+        :param serializer: Serializer to use for the schedules
+        :param buffer_size: Buffer size for getting schedules
+        :param skip_past_schedules: Skip schedules that are in the past.
+        :param connection_kwargs: Additional connection kwargs
+        """
         super().__init__()
         self._prefix = prefix
         self._buffer_size = buffer_size
@@ -179,7 +190,7 @@ class ListRedisScheduleSource(ScheduleSource):
         current_time = datetime.datetime.now(datetime.timezone.utc)
         timed: list[bytes] = []
         # Only during first run, we need to get previous time schedules
-        if self._is_first_run and not self._skip_past_schedules:
+        if not self._skip_past_schedules:
             timed = await self._get_previous_time_schedules()
             self._is_first_run = False
         async with Redis(connection_pool=self._connection_pool) as redis:

taskiq_redis/redis_broker.py

@@ -164,8 +164,10 @@ class RedisStreamBroker(BaseRedisBroker):
         consumer_name: Optional[str] = None,
         consumer_id: str = "$",
         mkstream: bool = True,
-        xread_block: int = 10000,
+        xread_block: int = 2000,
         maxlen: Optional[int] = None,
+        idle_timeout: int = 600000,  # 10 minutes
+        unacknowledged_batch_size: int = 100,
         additional_streams: Optional[Dict[str, str]] = None,
         **connection_kwargs: Any,
     ) -> None:
@@ -189,6 +191,8 @@ class RedisStreamBroker(BaseRedisBroker):
             trims (the old values of) the stream each time a new element is added
         :param additional_streams: additional streams to read from.
             Each key is a stream name, value is a consumer id.
+        :param redeliver_timeout: time in ms to wait before redelivering a message.
+        :param unacknowledged_batch_size: number of unacknowledged messages to fetch.
         """
         super().__init__(
             url,
@@ -205,6 +209,8 @@ class RedisStreamBroker(BaseRedisBroker):
         self.block = xread_block
         self.maxlen = maxlen
         self.additional_streams = additional_streams or {}
+        self.idle_timeout = idle_timeout
+        self.unacknowledged_batch_size = unacknowledged_batch_size
 
     async def _declare_consumer_group(self) -> None:
         """
@@ -260,6 +266,7 @@ class RedisStreamBroker(BaseRedisBroker):
         """Listen to incoming messages."""
         async with Redis(connection_pool=self.connection_pool) as redis_conn:
             while True:
+                logger.debug("Starting fetching new messages")
                 fetched = await redis_conn.xreadgroup(
                     self.consumer_group_name,
                     self.consumer_name,
@@ -277,3 +284,29 @@ class RedisStreamBroker(BaseRedisBroker):
                             data=msg[b"data"],
                             ack=self._ack_generator(msg_id),
                         )
+                logger.debug("Starting fetching unacknowledged messages")
+                for stream in [self.queue_name, *self.additional_streams.keys()]:
+                    lock = redis_conn.lock(
+                        f"autoclaim:{self.consumer_group_name}:{stream}",
+                    )
+                    if await lock.locked():
+                        continue
+                    async with lock:
+                        pending = await redis_conn.xautoclaim(
+                            name=stream,
+                            groupname=self.consumer_group_name,
+                            consumername=self.consumer_name,
+                            min_idle_time=self.idle_timeout,
+                            count=self.unacknowledged_batch_size,
+                        )
+                        logger.debug(
+                            "Found %d pending messages in stream %s",
+                            len(pending),
+                            stream,
+                        )
+                        for msg_id, msg in pending[1]:
+                            logger.debug("Received message: %s", msg)
+                            yield AckableMessage(
+                                data=msg[b"data"],
+                                ack=self._ack_generator(msg_id),
+                            )

taskiq_redis/schedule_source.py

@@ -1,4 +1,5 @@
 import sys
+import warnings
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Any, AsyncIterator, List, Optional, Tuple
 
@@ -53,6 +54,12 @@ class RedisScheduleSource(ScheduleSource):
         serializer: Optional[TaskiqSerializer] = None,
         **connection_kwargs: Any,
     ) -> None:
+        warnings.warn(
+            "RedisScheduleSource is deprecated. "
+            "Please switch to ListRedisScheduleSource",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self.prefix = prefix
         self.connection_pool: _BlockingConnectionPool = BlockingConnectionPool.from_url(
             url=url,

{taskiq_redis-1.0.5.dist-info → taskiq_redis-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: taskiq-redis
-Version: 1.0.5
+Version: 1.0.7
 Summary: Redis integration for taskiq
 Keywords: taskiq,tasks,distributed,async,redis,result_backend
 Author: taskiq-team

{taskiq_redis-1.0.5.dist-info → taskiq_redis-1.0.7.dist-info}/RECORD

@@ -1,13 +1,13 @@
 taskiq_redis/__init__.py,sha256=Sl4m9rKxweU1t0m289Qtf0qm4xSSkkFHoOfKq6qaz6g,1192
 taskiq_redis/exceptions.py,sha256=7buBJ7CRVWd5WqVqSjtHO8cVL7QzZg-DOM3nB87t-Sk,738
-taskiq_redis/list_schedule_source.py,sha256=w7lSJ1-n889PXOwXiA_Jp6Wj4RsZpuXwFCVhOIc5KAw,9462
+taskiq_redis/list_schedule_source.py,sha256=NlHqtvwsYmWpAXLz_0BFcdSIPJCq7ch_r27b0QVmcGE,9917
 taskiq_redis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 taskiq_redis/redis_backend.py,sha256=MLBaUN3Zx-DLvm1r-lgPU82_WZq9kc6oTxYI8LQjd6k,19882
-taskiq_redis/redis_broker.py,sha256=ep31kUxXx4XhGKjrzVjdNBmA6wLbTQoy7-DlKuwtLz4,10068
+taskiq_redis/redis_broker.py,sha256=EZqWrxD-xtMXyebmwBvliDtYeWIjqdYkm-RrDShYLQw,11829
 taskiq_redis/redis_cluster_broker.py,sha256=FuWl5fP7Fwr9FbytErmhcUGjRCdPexDK2Co2u6kpDlo,6591
 taskiq_redis/redis_sentinel_broker.py,sha256=wHnbG3xuD_ruhhwp4AXo91NNjq8v2iufUZ0i_HbBRVQ,9073
-taskiq_redis/schedule_source.py,sha256=hqpcs2D8W90KUDHREKblisnhGCE9dbVOtKtuJcOTGZw,9915
-taskiq_redis-1.0.5.dist-info/LICENSE,sha256=lEHEEE-ZxmuItxYgUMPiFWdRcAITxE8DFMNyAg4eOYE,1075
-taskiq_redis-1.0.5.dist-info/METADATA,sha256=uV2AJOyyE-f1_5A1LsvGpbf74zSueiJaNNRJ6kIqpvw,6573
-taskiq_redis-1.0.5.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-taskiq_redis-1.0.5.dist-info/RECORD,,
+taskiq_redis/schedule_source.py,sha256=mDYlAlAuzIzMICpJiQ1AwWOF-9_OHVGJWXA45Gm2Trg,10128
+taskiq_redis-1.0.7.dist-info/LICENSE,sha256=lEHEEE-ZxmuItxYgUMPiFWdRcAITxE8DFMNyAg4eOYE,1075
+taskiq_redis-1.0.7.dist-info/METADATA,sha256=7oouvMMNze9lzj8XcvGQY2DonXiT7PLkv7ErtNN1i0s,6573
+taskiq_redis-1.0.7.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+taskiq_redis-1.0.7.dist-info/RECORD,,