threadmill 0.7.2__tar.gz → 0.8.0__tar.gz

{threadmill-0.7.2 → threadmill-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.7.2
+Version: 0.8.0
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>
@@ -19,6 +19,8 @@ Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3.15
+Classifier: Programming Language :: Python :: Free Threading
+Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
 Classifier: Topic :: Communications :: Email
 Classifier: Topic :: Software Development
 Classifier: Topic :: Text Processing :: Markup :: Markdown
@@ -49,10 +51,18 @@ Provides-Extra: redis
   <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
 </p>
 
-# Threadmill [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
+# Threadmill [![PyPI Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/main/LICENSE)
 
 **Durable high-performance backend for Django's task framework.**
 
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker process each: Threadmill 19,954, Threadmill (GIL) 9,215, dramatiq 7,251, huey 6,254, celery 2,497, django-tasks-db 1,814, django-tasks-rq 84." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+  </picture>
+</p>
+
 ## Design Principles
 
 - **Durability** – Recover from any failures, even poorly written tasks.
@@ -64,7 +74,7 @@ Provides-Extra: redis
 You need to have [Django's Task framework][django-tasks] set up properly.
 
 ```console
-uv add threadmill[redis]
+uv add "threadmill[redis]"
 ```
 
 Add `threadmill` to your `INSTALLED_APPS` in `settings.py`
@@ -91,7 +101,7 @@ TASKS = {
 Optionally, install the inspector dependency if you want the TUI:
 
 ```console
-uv add threadmill[inspector]
+uv add "threadmill[inspector]"
 ```
 
 Then launch the worker pool:
@@ -108,13 +118,24 @@ The workers are inspired by Gunicorn, and the CLI is very similar.
 
 #### Utilization
 
-Depending on your workload, you can tweak the number of processes and threads.
-Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
+Depending on your workload, you can tweak the number of worker processes and the number of threads per process. Processes always run in parallel, because each one has its own interpreter. Threads share memory, so they are cheap, but whether they run in parallel depends on your interpreter:
+
+- On a regular GIL build, Python runs in one thread at a time. Threads still overlap waiting for IO, but CPU-bound tasks do not speed up. Raising the thread count far above the queue depth can also leave processes idle, because each process prefetches a batch sized by its thread count.
+- On a [free-threaded build](https://peps.python.org/pep-0779/), threads run truly in parallel, so CPU-bound tasks scale across cores without the cost of extra processes.
+
+A pool on a free-threaded interpreter therefore reaches the same throughput with one process running many threads as with many processes running one thread each, while using a single Redis connection pool and one copy of your application state:
 
 ```console
-uv run manage.py threadmill worker --processes 4 --threads 2
+uv run manage.py threadmill worker --workers 1 --threads 8
 ```
 
+Threads all live in one process, so a crash or a `--max-tasks` recycle takes every thread down at once. One process per core, the default, confines that to a single process. Prefer threads when memory matters more than that isolation, and keep the default when it does not.
+
+A free-threaded interpreter can be slower than a regular one for single-threaded work, so a GIL build stays the better choice for IO-bound tasks. Pick the interpreter that fits your workload rather than assuming free threading is an upgrade.
+
+> [!WARNING]
+> A C extension that does not declare free-threading support re-enables the GIL for the whole process, which silently costs you all thread parallelism. `hiredis` is a common offender: installing `redis[hiredis]` enables it on import. The worker logs a warning when it detects that the GIL was re-enabled.
+
 #### Health
 
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
@@ -125,12 +146,15 @@ uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
 
+The limit is soft. A worker drains its buffer and the batch in hand before it stops. It can then run about twice `--prefetch-count` tasks more than the configured maximum.
+
 Should a worker crash or be killed, the pool will automatically restart it.
 
 #### Shutdown
 
-A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
-All workers will finish the tasks they acquired and acknowledge them.
+A graceful shutdown is possible with `SIGTERM` or a keyboard interrupt.
+All workers finish the tasks they acquired and acknowledge them. This includes the tasks in their prefetch buffer.
+A hard kill cannot be intercepted, so the lease reaper collects the buffered tasks after the lease expires.
 
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
@@ -143,27 +167,36 @@ The optional TUI inspector lets you watch queues, tasks, and task details in rea
 Install it with the `inspector` extra and launch it from a separate terminal:
 
 ```console
-uv add threadmill[inspector]
+uv add "threadmill[inspector]"
 uv run manage.py threadmill inspector
 ```
 
 ### Redis Backend Options
 
+> [!IMPORTANT]
+> Threadmill requires a persistent Redis without eviction.
+
 The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
 `TASKS` configuration:
 
-| Option              | Default                   | Description                                                             |
-| ------------------- | ------------------------- | ----------------------------------------------------------------------- |
-| `lease_ttl`         | `timedelta(hours=1)`      | Max processing time before a started task is marked FAILED.             |
-| `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.            |
-| `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                  |
-| `batch_size`        | `100`                     | Max tasks to move or requeue per broker pass.                           |
-| `poll_interval`     | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll. |
-| `poll_max_interval` | `timedelta(seconds=1)`    | Max wait between idle acquire attempts.                                 |
-
-A task that is started but never acknowledged (lease expired) is marked FAILED
-with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
-worst-case task runtime.
+| Option              | Default                   | Description                                                                               |
+| ------------------- | ------------------------- | ----------------------------------------------------------------------------------------- |
+| `lease_ttl`         | `timedelta(hours=1)`      | Max time from acquisition to acknowledgement before the task is retried or marked FAILED. |
+| `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.                              |
+| `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                                    |
+| `batch_size`        | `100`                     | Max tasks to move or reap per broker pass.                                                |
+| `poll_interval`     | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll.                   |
+| `poll_max_interval` | `timedelta(seconds=1)`    | Max wait between idle acquire attempts.                                                   |
+
+A task whose lease expired reaches the `retry` callback as an
+`AcknowledgementTimeout` error, or is marked FAILED when nothing retries it.
+A claimed task whose stored payload cannot be read any more is dropped with the
+read error logged. A dropped task records no result, so it leaves the inspector
+and cannot be requeued.
+Keep `lease_ttl` above your worst-case runtime and above the time a task waits in a
+prefetch buffer. A task that outlives its lease can still run, so a retry can run
+at the same time. The acknowledgement of the lease holder wins. The late result
+of an expired attempt is discarded.
 
 All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
 every multi-key operation — including the cross-queue acquire — runs on a single
@@ -177,8 +210,8 @@ The callback receives a `TaskContext` — use `context.attempt` for the current
 attempt count and `context.task_result.errors[-1]` for the latest error.
 Return a `timedelta` to schedule the next attempt, or `None` to stop retrying.
 
-The worker re-queues the failed task, preserving its ID and error history;
-the broker promotes it back to the ready queue once the delay elapses.
+Failed tasks are re-queued preserving their ID and error history; the broker
+promotes them back to the ready queue once the delay elapses.
 
 #### Built-in `ExponentialBackoff`
 
@@ -193,6 +226,7 @@ import datetime
 from django.tasks import task
 from requests import HTTPError
 
+from threadmill.exceptions import AcknowledgementTimeout
 from threadmill.retry import ExponentialBackoff
 
 
@@ -202,7 +236,7 @@ from threadmill.retry import ExponentialBackoff
         max_delay=datetime.timedelta(minutes=5),
         factor=2.0,
         max_retries=5,
-        expected_exceptions=(HTTPError,),
+        expected_exceptions=(HTTPError, AcknowledgementTimeout),
     )
 )
 def fetch_github_api(url: str): ...

{threadmill-0.7.2 → threadmill-0.8.0}/README.md

@@ -11,10 +11,18 @@
   <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
 </p>
 
-# Threadmill [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
+# Threadmill [![PyPI Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/main/LICENSE)
 
 **Durable high-performance backend for Django's task framework.**
 
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker process each: Threadmill 19,954, Threadmill (GIL) 9,215, dramatiq 7,251, huey 6,254, celery 2,497, django-tasks-db 1,814, django-tasks-rq 84." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+  </picture>
+</p>
+
 ## Design Principles
 
 - **Durability** – Recover from any failures, even poorly written tasks.
@@ -26,7 +34,7 @@
 You need to have [Django's Task framework][django-tasks] set up properly.
 
 ```console
-uv add threadmill[redis]
+uv add "threadmill[redis]"
 ```
 
 Add `threadmill` to your `INSTALLED_APPS` in `settings.py`
@@ -53,7 +61,7 @@ TASKS = {
 Optionally, install the inspector dependency if you want the TUI:
 
 ```console
-uv add threadmill[inspector]
+uv add "threadmill[inspector]"
 ```
 
 Then launch the worker pool:
@@ -70,13 +78,24 @@ The workers are inspired by Gunicorn, and the CLI is very similar.
 
 #### Utilization
 
-Depending on your workload, you can tweak the number of processes and threads.
-Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
+Depending on your workload, you can tweak the number of worker processes and the number of threads per process. Processes always run in parallel, because each one has its own interpreter. Threads share memory, so they are cheap, but whether they run in parallel depends on your interpreter:
+
+- On a regular GIL build, Python runs in one thread at a time. Threads still overlap waiting for IO, but CPU-bound tasks do not speed up. Raising the thread count far above the queue depth can also leave processes idle, because each process prefetches a batch sized by its thread count.
+- On a [free-threaded build](https://peps.python.org/pep-0779/), threads run truly in parallel, so CPU-bound tasks scale across cores without the cost of extra processes.
+
+A pool on a free-threaded interpreter therefore reaches the same throughput with one process running many threads as with many processes running one thread each, while using a single Redis connection pool and one copy of your application state:
 
 ```console
-uv run manage.py threadmill worker --processes 4 --threads 2
+uv run manage.py threadmill worker --workers 1 --threads 8
 ```
 
+Threads all live in one process, so a crash or a `--max-tasks` recycle takes every thread down at once. One process per core, the default, confines that to a single process. Prefer threads when memory matters more than that isolation, and keep the default when it does not.
+
+A free-threaded interpreter can be slower than a regular one for single-threaded work, so a GIL build stays the better choice for IO-bound tasks. Pick the interpreter that fits your workload rather than assuming free threading is an upgrade.
+
+> [!WARNING]
+> A C extension that does not declare free-threading support re-enables the GIL for the whole process, which silently costs you all thread parallelism. `hiredis` is a common offender: installing `redis[hiredis]` enables it on import. The worker logs a warning when it detects that the GIL was re-enabled.
+
 #### Health
 
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
@@ -87,12 +106,15 @@ uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
 
+The limit is soft. A worker drains its buffer and the batch in hand before it stops. It can then run about twice `--prefetch-count` tasks more than the configured maximum.
+
 Should a worker crash or be killed, the pool will automatically restart it.
 
 #### Shutdown
 
-A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
-All workers will finish the tasks they acquired and acknowledge them.
+A graceful shutdown is possible with `SIGTERM` or a keyboard interrupt.
+All workers finish the tasks they acquired and acknowledge them. This includes the tasks in their prefetch buffer.
+A hard kill cannot be intercepted, so the lease reaper collects the buffered tasks after the lease expires.
 
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
@@ -105,27 +127,36 @@ The optional TUI inspector lets you watch queues, tasks, and task details in rea
 Install it with the `inspector` extra and launch it from a separate terminal:
 
 ```console
-uv add threadmill[inspector]
+uv add "threadmill[inspector]"
 uv run manage.py threadmill inspector
 ```
 
 ### Redis Backend Options
 
+> [!IMPORTANT]
+> Threadmill requires a persistent Redis without eviction.
+
 The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
 `TASKS` configuration:
 
-| Option              | Default                   | Description                                                             |
-| ------------------- | ------------------------- | ----------------------------------------------------------------------- |
-| `lease_ttl`         | `timedelta(hours=1)`      | Max processing time before a started task is marked FAILED.             |
-| `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.            |
-| `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                  |
-| `batch_size`        | `100`                     | Max tasks to move or requeue per broker pass.                           |
-| `poll_interval`     | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll. |
-| `poll_max_interval` | `timedelta(seconds=1)`    | Max wait between idle acquire attempts.                                 |
-
-A task that is started but never acknowledged (lease expired) is marked FAILED
-with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
-worst-case task runtime.
+| Option              | Default                   | Description                                                                               |
+| ------------------- | ------------------------- | ----------------------------------------------------------------------------------------- |
+| `lease_ttl`         | `timedelta(hours=1)`      | Max time from acquisition to acknowledgement before the task is retried or marked FAILED. |
+| `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.                              |
+| `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                                    |
+| `batch_size`        | `100`                     | Max tasks to move or reap per broker pass.                                                |
+| `poll_interval`     | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll.                   |
+| `poll_max_interval` | `timedelta(seconds=1)`    | Max wait between idle acquire attempts.                                                   |
+
+A task whose lease expired reaches the `retry` callback as an
+`AcknowledgementTimeout` error, or is marked FAILED when nothing retries it.
+A claimed task whose stored payload cannot be read any more is dropped with the
+read error logged. A dropped task records no result, so it leaves the inspector
+and cannot be requeued.
+Keep `lease_ttl` above your worst-case runtime and above the time a task waits in a
+prefetch buffer. A task that outlives its lease can still run, so a retry can run
+at the same time. The acknowledgement of the lease holder wins. The late result
+of an expired attempt is discarded.
 
 All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
 every multi-key operation — including the cross-queue acquire — runs on a single
@@ -139,8 +170,8 @@ The callback receives a `TaskContext` — use `context.attempt` for the current
 attempt count and `context.task_result.errors[-1]` for the latest error.
 Return a `timedelta` to schedule the next attempt, or `None` to stop retrying.
 
-The worker re-queues the failed task, preserving its ID and error history;
-the broker promotes it back to the ready queue once the delay elapses.
+Failed tasks are re-queued preserving their ID and error history; the broker
+promotes them back to the ready queue once the delay elapses.
 
 #### Built-in `ExponentialBackoff`
 
@@ -155,6 +186,7 @@ import datetime
 from django.tasks import task
 from requests import HTTPError
 
+from threadmill.exceptions import AcknowledgementTimeout
 from threadmill.retry import ExponentialBackoff
 
 
@@ -164,7 +196,7 @@ from threadmill.retry import ExponentialBackoff
         max_delay=datetime.timedelta(minutes=5),
         factor=2.0,
         max_retries=5,
-        expected_exceptions=(HTTPError,),
+        expected_exceptions=(HTTPError, AcknowledgementTimeout),
     )
 )
 def fetch_github_api(url: str): ...

{threadmill-0.7.2 → threadmill-0.8.0}/pyproject.toml

@@ -25,6 +25,8 @@ classifiers = [
   "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.14",
   "Programming Language :: Python :: 3.15",
+  "Programming Language :: Python :: Free Threading",
+  "Programming Language :: Python :: Free Threading :: 3 - Stable",
   "Topic :: Communications :: Email",
   "Topic :: Software Development",
   "Topic :: Text Processing :: Markup :: Markdown",
@@ -49,6 +51,11 @@ dev = [
   { include-group = "test" },
 ]
 test = [
+  "celery",
+  "django-tasks-db",
+  "django-tasks-rq",
+  "dramatiq[redis]>=2.2.1",
+  "huey",
   "pytest",
   "pytest-asyncio",
   "pytest-benchmark",
@@ -64,7 +71,7 @@ write_to = "threadmill/_version.py"
 module.name = "threadmill"
 
 [tool.ruff]
-src = [ "tests", "threadmill" ]
+src = [ "benchmarks", "tests", "threadmill" ]
 lint.select = [
   "B",  # flake8-bugbear
   "C",  # flake8-comprehensions
@@ -86,7 +93,7 @@ lint.pydocstyle.convention = "pep257"
 
 [tool.pytest]
 ini_options.minversion = "6.1"
-ini_options.testpaths = [ "tests" ]
+ini_options.testpaths = [ "benchmarks", "tests" ]
 ini_options.addopts = """\
   --cov --cov-report=xml --cov-report=term --tb=short -rxs --benchmark-autosave --benchmark-group-by=fullname \
   --benchmark-min-rounds=10\

{threadmill-0.7.2 → threadmill-0.8.0}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.7.2'
-__version_tuple__ = version_tuple = (0, 7, 2)
+__version__ = version = '0.8.0'
+__version_tuple__ = version_tuple = (0, 8, 0)
 
-__commit_id__ = commit_id = 'g560d3b3b5'
+__commit_id__ = commit_id = 'gd8270af5a'

{threadmill-0.7.2 → threadmill-0.8.0}/threadmill/backends/base.py

@@ -3,9 +3,11 @@ import dataclasses
 import datetime
 import enum
 import json
+import logging
 import threading
 import typing
 from abc import ABC
+from traceback import format_exception
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.tasks import DEFAULT_TASK_QUEUE_NAME, Task, TaskResult, TaskResultStatus
@@ -15,6 +17,8 @@ from django.tasks.exceptions import InvalidTask
 from django.utils.inspect import is_module_level_function
 from django.utils.module_loading import import_string
 
+logger = logging.getLogger(__name__)
+
 
 @dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
 class RetryTask(Task):
@@ -61,6 +65,24 @@ class RetryTask(Task):
         return (reconstructor, (kwargs,))
 
 
+@dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
+class ThreadmillTaskResult(TaskResult):
+    lease_token: str | None = None
+
+    @classmethod
+    def from_result(
+        cls, task_result: TaskResult, *, lease_token: str | None
+    ) -> ThreadmillTaskResult:
+        return cls(
+            **{
+                field.name: getattr(task_result, field.name)
+                for field in dataclasses.fields(TaskResult)
+                if field.init
+            },
+            lease_token=lease_token,
+        )
+
+
 @dataclasses.dataclass(kw_only=True, slots=True)
 class QueueCounts:
     """Point-in-time cardinality of each queue segment."""
@@ -151,10 +173,15 @@ class TaskResultEncoder(DjangoJSONEncoder):
     """JSON encoder for TaskResult and TaskError objects."""
 
     def default(self, o):
-        if isinstance(o, (TaskResult, TaskError)):
+        if isinstance(o, TaskResult):
+            return {
+                field.name: getattr(o, field.name)
+                for field in dataclasses.fields(TaskResult)
+            }
+        if isinstance(o, TaskError):
             return {
                 field.name: getattr(o, field.name)
-                for field in dataclasses.fields(type(o))
+                for field in dataclasses.fields(TaskError)
             }
         if isinstance(o, RetryTask):
             data = {
@@ -208,23 +235,53 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
                 "Task's retry function must be defined at a module level or be a deconstructible callable."
             )
 
+    @staticmethod
+    def create_task_error(exception: BaseException) -> TaskError:
+        exception_type = type(exception)
+        return TaskError(
+            exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}",
+            traceback="".join(format_exception(exception)),
+        )
+
+    @staticmethod
+    def retry_delay(task_result: TaskResult) -> datetime.timedelta | None:
+        if task_result.task.retry:
+            try:
+                return task_result.task.retry(TaskContext(task_result=task_result))
+            except Exception:
+                logger.exception(
+                    "Retry callback failed for task '%s@%s'",
+                    task_result.id,
+                    task_result.task.module_path,
+                )
+
     def acquire(
         self,
         *queue_names: str,
+        count: int = 1,
         timeout: datetime.timedelta | None = None,
         worker: str = "",
-    ) -> TaskResult:
+    ) -> list[ThreadmillTaskResult]:
         """
-        Return and lock the next task to be processed without removing it from the queue.
+        Return and lock up to `count` tasks. The tasks stay in the queue.
+
+        The method waits up to `timeout` for the first task. It fills the
+        remaining tasks without a wait.
 
         Args:
             queue_names: The names of the queues to acquire tasks from.
-            timeout: The maximum time to wait for a task. If None, wait indefinitely.
-            worker: The name of the worker thread acquiring the task.
+            count: The maximum number of tasks to acquire, at least 1.
+            timeout: The maximum time to wait for the first task. If None, the method waits without a limit.
+            worker: The name of the worker thread that acquires the tasks.
 
         Raises:
-            TimeoutError: If no task is available within the specified timeout.
-            queue.Empty: If no task is available and timeout is None.
+            TimeoutError: The first task does not arrive within the specified timeout.
+            queue.Empty: The first task is unavailable and the timeout is None.
+
+        Returns:
+            Between one and `count` leased task results in lease order. The list is
+            never empty. Each result is a `ThreadmillTaskResult` that carries the
+            lease token for its acknowledgement.
         """
         raise NotImplementedError
 

{threadmill-0.7.2 → threadmill-0.8.0}/threadmill/backends/lua/acknowledge.lua

@@ -20,6 +20,11 @@
 -- ARGV[7]  -- queue name
 -- Returns: 1 on success, 0 if task was not in the running set
 
+local lease_token = redis.call('HGET', KEYS[3], 'lease_token')
+if lease_token and lease_token ~= ARGV[8] then
+  return 0
+end
+
 local removed = redis.call('ZREM', KEYS[1], ARGV[1])
 if removed == 0 then
   return 0  -- Task already reaped, skip

threadmill-0.8.0/threadmill/backends/lua/acquire.lua

@@ -0,0 +1,57 @@
+-- Lease up to ARGV[8] ready tasks for the calling worker in one round-trip:
+-- each task's stored payload comes back beside the lease token stamped for it.
+-- Queues are tried round-robin from ARGV[7], one queue per pop, so a batch
+-- spreads across queues instead of draining one and a backlogged queue cannot
+-- starve its neighbours.
+--
+-- The payload comes back as it was enqueued. Apply the lease stamped beside it
+-- (lease_worker, lease_started_at, lease_token) to report the task as RUNNING.
+--
+-- KEYS[1..N]  -- interleaved running keys and queue keys, one pair per queue:
+--                KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
+--                KEYS[4] = queue, etc.
+-- ARGV[1]     -- current time in milliseconds
+-- ARGV[2]     -- current time as ISO-8601 string, stamped as lease_started_at
+-- ARGV[3]     -- task key prefix (e.g. "threadmill:task:")
+-- ARGV[4]     -- number of queue pairs (N/2)
+-- ARGV[5]     -- worker name, stamped as lease_worker
+-- ARGV[6]     -- lease TTL in milliseconds
+-- ARGV[7]     -- start_index; 0-based index of the queue pair to scan first, so
+--                start_index 0 is the pair at KEYS[1] and KEYS[2]
+-- ARGV[8]     -- maximum number of tasks to lease
+-- Returns: array of {payload, lease_token} pairs, empty when no queue yields a
+-- task. A queue whose head entry holds no payload yields nothing for that pop
+-- and the scan moves on.
+
+local num_queues = tonumber(ARGV[4])
+local lease_ttl_ms = tonumber(ARGV[6])
+local start_index = tonumber(ARGV[7])
+local max_count = tonumber(ARGV[8])
+local task_key_prefix = ARGV[3]
+local now_ms = tonumber(ARGV[1])
+local tasks = {}
+local misses = 0
+local queue_index = start_index % num_queues + 1
+-- joe: one full round without a task ends the scan, so a queue whose head
+-- entry is a ghost yields a short batch; retry the queue if ghosts ever dominate
+while #tasks < max_count and misses < num_queues do
+  local popped = redis.call('ZPOPMIN', KEYS[queue_index * 2])
+  local data = #popped > 0
+    and redis.call('HGET', task_key_prefix .. popped[1], 'data')
+  if data then
+    local lease_token = string.format(
+      '%06x%06x%06x%06x',
+      math.random(0, 0xffffff), math.random(0, 0xffffff),
+      math.random(0, 0xffffff), math.random(0, 0xffffff))
+    local deadline = now_ms + lease_ttl_ms
+    redis.call('ZADD', KEYS[queue_index * 2 - 1], deadline, popped[1])
+    redis.call('HSET', task_key_prefix .. popped[1],
+      'lease_worker', ARGV[5],
+      'lease_started_at', ARGV[2],
+      'lease_token', lease_token)
+    table.insert(tasks, { data, lease_token })
+  end
+  misses = data and 0 or (misses + 1)
+  queue_index = queue_index % num_queues + 1
+end
+return tasks

threadmill-0.8.0/threadmill/backends/lua/reaper.lua

@@ -0,0 +1,23 @@
+-- Claim expired tasks for the broker to decide. Undecided claims are picked up
+-- again, and running entries without task data are dropped.
+--
+-- KEYS[1]  -- running set (ZSET, scored by lease deadline in milliseconds)
+-- ARGV[1]  -- claim TTL in milliseconds
+-- ARGV[2]  -- maximum number of tasks to claim per call (batch size)
+-- ARGV[3]  -- task key prefix (e.g. "threadmill:default:task:")
+-- Returns: list of claimed task IDs
+
+local clock = redis.call('TIME')
+local now_ms = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
+local claim_deadline_ms = now_ms + tonumber(ARGV[1])
+local expired = redis.call('ZRANGEBYSCORE', KEYS[1], 0, now_ms, 'LIMIT', 0, tonumber(ARGV[2]))
+local claimed = {}
+for _, task_id in ipairs(expired) do
+  if redis.call('EXISTS', ARGV[3] .. task_id) == 1 then
+    redis.call('ZADD', KEYS[1], claim_deadline_ms, task_id)
+    table.insert(claimed, task_id)
+  else
+    redis.call('ZREM', KEYS[1], task_id)
+  end
+end
+return claimed