taskiq-redis 1.0.2__tar.gz → 1.0.4__tar.gz

taskiq_redis-1.0.4/PKG-INFO

@@ -0,0 +1,215 @@
+Metadata-Version: 2.3
+Name: taskiq-redis
+Version: 1.0.4
+Summary: Redis integration for taskiq
+Keywords: taskiq,tasks,distributed,async,redis,result_backend
+Author: taskiq-team
+Author-email: taskiq@norely.com
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Requires-Dist: redis (>=5,<6)
+Requires-Dist: taskiq (>=0.11.12,<1)
+Project-URL: Homepage, https://github.com/taskiq-python/taskiq-redis
+Project-URL: Repository, https://github.com/taskiq-python/taskiq-redis
+Description-Content-Type: text/markdown
+
+# TaskIQ-Redis
+
+Taskiq-redis is a plugin for taskiq that adds a new broker and result backend based on redis.
+
+# Installation
+
+To use this project you must have installed core taskiq library:
+```bash
+pip install taskiq
+```
+This project can be installed using pip:
+```bash
+pip install taskiq-redis
+```
+
+# Usage
+
+Let's see the example with the redis broker and redis async result:
+
+```python
+# broker.py
+import asyncio
+
+from taskiq_redis import RedisAsyncResultBackend, RedisStreamBroker
+
+result_backend = RedisAsyncResultBackend(
+    redis_url="redis://localhost:6379",
+)
+
+# Or you can use PubSubBroker if you need broadcasting
+# Or ListQueueBroker if you don't want acknowledges
+broker = RedisStreamBroker(
+    url="redis://localhost:6379",
+).with_result_backend(result_backend)
+
+
+@broker.task
+async def best_task_ever() -> None:
+    """Solve all problems in the world."""
+    await asyncio.sleep(5.5)
+    print("All problems are solved!")
+
+
+async def main():
+    task = await best_task_ever.kiq()
+    print(await task.wait_result())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+Launch the workers:
+`taskiq worker broker:broker`
+Then run the main code:
+`python3 broker.py`
+
+
+## Brokers
+
+This package contains 6 broker implementations.
+3 broker types:
+* PubSub broker
+* ListQueue broker
+* Stream broker
+
+Each of type is implemented for each redis architecture:
+* Single node
+* Cluster
+* Sentinel
+
+Here's a small breakdown of how they differ from eachother.
+
+
+### PubSub
+
+By default on old redis versions PUBSUB was the way of making redis into a queue.
+But using PUBSUB means that all messages delivered to all subscribed consumers.
+
+> [!WARNING]
+> This broker doesn't support acknowledgements. If during message processing
+> Worker was suddenly killed the message is going to be lost.
+
+### ListQueue
+
+This broker creates a list of messages at some key. Adding new tasks will be done
+by appending them from the left side using `lpush`, and taking them from the right side using `brpop`.
+
+> [!WARNING]
+> This broker doesn't support acknowledgements. If during message processing
+> Worker was suddenly killed the message is going to be lost.
+
+### Stream
+
+Stream brokers use redis [stream type](https://redis.io/docs/latest/develop/data-types/streams/) to store and fetch messages.
+
+> [!TIP]
+> This broker **supports** acknowledgements and therefore is fine to use in cases when data durability is
+> required.
+
+## RedisAsyncResultBackend configuration
+
+RedisAsyncResultBackend parameters:
+* `redis_url` - url to redis.
+* `keep_results` - flag to not remove results from Redis after reading.
+* `result_ex_time` - expire time in seconds (by default - not specified)
+* `result_px_time` - expire time in milliseconds (by default - not specified)
+* Any other keyword arguments are passed to `redis.asyncio.BlockingConnectionPool`.
+  Notably, you can use `timeout` to set custom timeout in seconds for reconnects
+  (or set it to `None` to try reconnects indefinitely).
+
+> [!WARNING]
+> **It is highly recommended to use expire time in RedisAsyncResultBackend**
+> If you want to add expiration, either `result_ex_time` or `result_px_time` must be set.
+> ```python
+> # First variant
+> redis_async_result = RedisAsyncResultBackend(
+>     redis_url="redis://localhost:6379",
+>     result_ex_time=1000,
+> )
+>
+> # Second variant
+> redis_async_result = RedisAsyncResultBackend(
+>     redis_url="redis://localhost:6379",
+>     result_px_time=1000000,
+> )
+> ```
+
+
+## Schedule sources
+
+
+You can use this package to add dynamic schedule sources. They are used to store
+schedules for taskiq scheduler.
+
+The advantage of using schedule sources from this package over default `LabelBased` source is that you can
+dynamically add schedules in it.
+
+We have two types of schedules:
+
+* `RedisScheduleSource`
+* `ListRedisScheduleSource`
+
+
+### RedisScheduleSource
+
+This source is super simple. It stores all schedules by key `{prefix}:{schedule_id}`. When scheduler requests
+schedules, it retrieves all values from redis that start with a given `prefix`.
+
+This is very ineficent and should not be used for high-volume schedules. Because if you have `1000` schedules, this scheduler will make at least `20` requests to retrieve them (we use `scan` and `mget` to minimize number of calls).
+
+### ListRedisScheduleSource
+
+This source holds values in lists.
+
+* For cron tasks it uses key `{prefix}:cron`.
+* For timed schedules it uses key `{prefix}:time:{time}` where `{time}` is actually time where schedules should run.
+
+The main advantage of this approach is that we only fetch tasks we need to run at a given time and do not perform any excesive calls to redis.
+
+
+### Migration from one source to another
+
+To migrate from `RedisScheduleSource` to `ListRedisScheduleSource` you can define the latter as this:
+
+```python
+# broker.py
+import asyncio
+import datetime
+
+from taskiq import TaskiqScheduler
+
+from taskiq_redis import ListRedisScheduleSource, RedisStreamBroker
+from taskiq_redis.schedule_source import RedisScheduleSource
+
+broker = RedisStreamBroker(url="redis://localhost:6379")
+
+old_source = RedisScheduleSource("redis://localhost/1", prefix="prefix1")
+array_source = ListRedisScheduleSource(
+    "redis://localhost/1",
+    prefix="prefix2",
+    # To migrate schedules from an old source.
+).with_migrate_from(
+    old_source,
+    # To delete schedules from an old source.
+    delete_schedules=True,
+)
+scheduler = TaskiqScheduler(broker, [array_source])
+```
+
+During startup the scheduler will try to migrate schedules from an old source to a new one. Please be sure to specify different prefixe just to avoid any kind of collision between these two.
+

taskiq_redis-1.0.4/README.md

@@ -0,0 +1,191 @@
+# TaskIQ-Redis
+
+Taskiq-redis is a plugin for taskiq that adds a new broker and result backend based on redis.
+
+# Installation
+
+To use this project you must have installed core taskiq library:
+```bash
+pip install taskiq
+```
+This project can be installed using pip:
+```bash
+pip install taskiq-redis
+```
+
+# Usage
+
+Let's see the example with the redis broker and redis async result:
+
+```python
+# broker.py
+import asyncio
+
+from taskiq_redis import RedisAsyncResultBackend, RedisStreamBroker
+
+result_backend = RedisAsyncResultBackend(
+    redis_url="redis://localhost:6379",
+)
+
+# Or you can use PubSubBroker if you need broadcasting
+# Or ListQueueBroker if you don't want acknowledges
+broker = RedisStreamBroker(
+    url="redis://localhost:6379",
+).with_result_backend(result_backend)
+
+
+@broker.task
+async def best_task_ever() -> None:
+    """Solve all problems in the world."""
+    await asyncio.sleep(5.5)
+    print("All problems are solved!")
+
+
+async def main():
+    task = await best_task_ever.kiq()
+    print(await task.wait_result())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+Launch the workers:
+`taskiq worker broker:broker`
+Then run the main code:
+`python3 broker.py`
+
+
+## Brokers
+
+This package contains 6 broker implementations.
+3 broker types:
+* PubSub broker
+* ListQueue broker
+* Stream broker
+
+Each of type is implemented for each redis architecture:
+* Single node
+* Cluster
+* Sentinel
+
+Here's a small breakdown of how they differ from eachother.
+
+
+### PubSub
+
+By default on old redis versions PUBSUB was the way of making redis into a queue.
+But using PUBSUB means that all messages delivered to all subscribed consumers.
+
+> [!WARNING]
+> This broker doesn't support acknowledgements. If during message processing
+> Worker was suddenly killed the message is going to be lost.
+
+### ListQueue
+
+This broker creates a list of messages at some key. Adding new tasks will be done
+by appending them from the left side using `lpush`, and taking them from the right side using `brpop`.
+
+> [!WARNING]
+> This broker doesn't support acknowledgements. If during message processing
+> Worker was suddenly killed the message is going to be lost.
+
+### Stream
+
+Stream brokers use redis [stream type](https://redis.io/docs/latest/develop/data-types/streams/) to store and fetch messages.
+
+> [!TIP]
+> This broker **supports** acknowledgements and therefore is fine to use in cases when data durability is
+> required.
+
+## RedisAsyncResultBackend configuration
+
+RedisAsyncResultBackend parameters:
+* `redis_url` - url to redis.
+* `keep_results` - flag to not remove results from Redis after reading.
+* `result_ex_time` - expire time in seconds (by default - not specified)
+* `result_px_time` - expire time in milliseconds (by default - not specified)
+* Any other keyword arguments are passed to `redis.asyncio.BlockingConnectionPool`.
+  Notably, you can use `timeout` to set custom timeout in seconds for reconnects
+  (or set it to `None` to try reconnects indefinitely).
+
+> [!WARNING]
+> **It is highly recommended to use expire time in RedisAsyncResultBackend**
+> If you want to add expiration, either `result_ex_time` or `result_px_time` must be set.
+> ```python
+> # First variant
+> redis_async_result = RedisAsyncResultBackend(
+>     redis_url="redis://localhost:6379",
+>     result_ex_time=1000,
+> )
+>
+> # Second variant
+> redis_async_result = RedisAsyncResultBackend(
+>     redis_url="redis://localhost:6379",
+>     result_px_time=1000000,
+> )
+> ```
+
+
+## Schedule sources
+
+
+You can use this package to add dynamic schedule sources. They are used to store
+schedules for taskiq scheduler.
+
+The advantage of using schedule sources from this package over default `LabelBased` source is that you can
+dynamically add schedules in it.
+
+We have two types of schedules:
+
+* `RedisScheduleSource`
+* `ListRedisScheduleSource`
+
+
+### RedisScheduleSource
+
+This source is super simple. It stores all schedules by key `{prefix}:{schedule_id}`. When scheduler requests
+schedules, it retrieves all values from redis that start with a given `prefix`.
+
+This is very ineficent and should not be used for high-volume schedules. Because if you have `1000` schedules, this scheduler will make at least `20` requests to retrieve them (we use `scan` and `mget` to minimize number of calls).
+
+### ListRedisScheduleSource
+
+This source holds values in lists.
+
+* For cron tasks it uses key `{prefix}:cron`.
+* For timed schedules it uses key `{prefix}:time:{time}` where `{time}` is actually time where schedules should run.
+
+The main advantage of this approach is that we only fetch tasks we need to run at a given time and do not perform any excesive calls to redis.
+
+
+### Migration from one source to another
+
+To migrate from `RedisScheduleSource` to `ListRedisScheduleSource` you can define the latter as this:
+
+```python
+# broker.py
+import asyncio
+import datetime
+
+from taskiq import TaskiqScheduler
+
+from taskiq_redis import ListRedisScheduleSource, RedisStreamBroker
+from taskiq_redis.schedule_source import RedisScheduleSource
+
+broker = RedisStreamBroker(url="redis://localhost:6379")
+
+old_source = RedisScheduleSource("redis://localhost/1", prefix="prefix1")
+array_source = ListRedisScheduleSource(
+    "redis://localhost/1",
+    prefix="prefix2",
+    # To migrate schedules from an old source.
+).with_migrate_from(
+    old_source,
+    # To delete schedules from an old source.
+    delete_schedules=True,
+)
+scheduler = TaskiqScheduler(broker, [array_source])
+```
+
+During startup the scheduler will try to migrate schedules from an old source to a new one. Please be sure to specify different prefixe just to avoid any kind of collision between these two.

{taskiq_redis-1.0.2 → taskiq_redis-1.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "taskiq-redis"
-version = "1.0.2"
+version = "1.0.4"
 description = "Redis integration for taskiq"
 authors = ["taskiq-team <taskiq@norely.com>"]
 readme = "README.md"
@@ -25,22 +25,22 @@ keywords = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.8.1"
-taskiq = ">=0.11.1,<1"
+python = "^3.9"
+taskiq = ">=0.11.12,<1"
 redis = "^5"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.0"
+pytest = "^8"
 mypy = "^1"
-black = "^22.3.0"
-pytest-cov = "^3.0.0"
-anyio = "^3.6.1"
-pytest-env = "^0.6.2"
+black = "^25"
+pytest-cov = "^6"
+anyio = "^4"
+pytest-env = "^1"
 fakeredis = "^2"
-pre-commit = "^2.20.0"
-pytest-xdist = { version = "^2.5.0", extras = ["psutil"] }
+pre-commit = "^4"
+pytest-xdist = { version = "^3", extras = ["psutil"] }
 ruff = "^0"
-types-redis = "^4.6.0.20240425"
+freezegun = "^1.5.1"
 
 [tool.mypy]
 strict = true
@@ -56,6 +56,7 @@ warn_return_any = false
 [[tool.mypy.overrides]]
 module = ['redis']
 ignore_missing_imports = true
+ignore_errors = true
 strict = false
 
 [build-system]
@@ -65,7 +66,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.ruff]
 # List of enabled rulsets.
 # See https://docs.astral.sh/ruff/rules/ for more information.
-select = [
+lint.select = [
     "E",   # Error
     "F",   # Pyflakes
     "W",   # Pycodestyle
@@ -92,24 +93,22 @@ select = [
     "PL",  # PyLint checks
     "RUF", # Specific to Ruff checks
 ]
-ignore = [
+lint.ignore = [
     "D105",    # Missing docstring in magic method
     "D107",    # Missing docstring in __init__
     "D212",    # Multi-line docstring summary should start at the first line
     "D401",    # First line should be in imperative mood
     "D104",    # Missing docstring in public package
     "D100",    # Missing docstring in public module
-    "ANN102",  # Missing type annotation for self in method
-    "ANN101",  # Missing type annotation for argument
     "ANN401",  # typing.Any are disallowed in `**kwargs
     "PLR0913", # Too many arguments for function call
     "D106",    # Missing docstring in public nested class
 ]
 exclude = [".venv/"]
-mccabe = { max-complexity = 10 }
+lint.mccabe = { max-complexity = 10 }
 line-length = 88
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/*" = [
     "S101",   # Use of assert detected
     "S301",   # Use of pickle detected
@@ -119,12 +118,12 @@ line-length = 88
     "D101",   # Missing docstring in public class
 ]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "pep257"
 ignore-decorators = ["typing.overload"]
 
-[tool.ruff.pylint]
+[tool.ruff.lint.pylint]
 allow-magic-value-types = ["int", "str", "float"]
 
-[tool.ruff.flake8-bugbear]
+[tool.ruff.lint.flake8-bugbear]
 extend-immutable-calls = ["taskiq_dependencies.Depends", "taskiq.TaskiqDepends"]

{taskiq_redis-1.0.2 → taskiq_redis-1.0.4}/taskiq_redis/__init__.py

@@ -1,14 +1,20 @@
 """Package for redis integration."""
+
+from taskiq_redis.list_schedule_source import ListRedisScheduleSource
 from taskiq_redis.redis_backend import (
     RedisAsyncClusterResultBackend,
     RedisAsyncResultBackend,
     RedisAsyncSentinelResultBackend,
 )
-from taskiq_redis.redis_broker import ListQueueBroker, PubSubBroker
-from taskiq_redis.redis_cluster_broker import ListQueueClusterBroker
+from taskiq_redis.redis_broker import ListQueueBroker, PubSubBroker, RedisStreamBroker
+from taskiq_redis.redis_cluster_broker import (
+    ListQueueClusterBroker,
+    RedisStreamClusterBroker,
+)
 from taskiq_redis.redis_sentinel_broker import (
     ListQueueSentinelBroker,
     PubSubSentinelBroker,
+    RedisStreamSentinelBroker,
 )
 from taskiq_redis.schedule_source import (
     RedisClusterScheduleSource,
@@ -17,15 +23,19 @@ from taskiq_redis.schedule_source import (
 )
 
 __all__ = [
-    "RedisAsyncClusterResultBackend",
-    "RedisAsyncResultBackend",
-    "RedisAsyncSentinelResultBackend",
     "ListQueueBroker",
-    "PubSubBroker",
     "ListQueueClusterBroker",
     "ListQueueSentinelBroker",
+    "ListRedisScheduleSource",
+    "PubSubBroker",
     "PubSubSentinelBroker",
-    "RedisScheduleSource",
+    "RedisAsyncClusterResultBackend",
+    "RedisAsyncResultBackend",
+    "RedisAsyncSentinelResultBackend",
     "RedisClusterScheduleSource",
+    "RedisScheduleSource",
     "RedisSentinelScheduleSource",
+    "RedisStreamBroker",
+    "RedisStreamClusterBroker",
+    "RedisStreamSentinelBroker",
 ]

{taskiq_redis-1.0.2 → taskiq_redis-1.0.4}/taskiq_redis/exceptions.py

@@ -8,10 +8,16 @@ class TaskIQRedisError(TaskiqError):
 class DuplicateExpireTimeSelectedError(ResultBackendError, TaskIQRedisError):
     """Error if two lifetimes are selected."""
 
+    __template__ = "Choose either result_ex_time or result_px_time."
+
 
 class ExpireTimeMustBeMoreThanZeroError(ResultBackendError, TaskIQRedisError):
     """Error if two lifetimes are less or equal zero."""
 
+    __template__ = (
+        "You must select one expire time param and it must be more than zero."
+    )
+
 
 class ResultIsMissingError(TaskIQRedisError, ResultGetError):
     """Error if there is no result when trying to get it."""