threadmill 0.1.0__tar.gz → 0.2.0__tar.gz

{threadmill-0.1.0 → threadmill-0.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.1.0
-Summary: A queue agnostic worker for Django's task framework.
+Version: 0.2.0
+Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>
 Requires-Python: >=3.12
@@ -27,6 +27,7 @@ Classifier: Framework :: Django
 Classifier: Framework :: Django :: 6.1
 License-File: LICENSE
 Requires-Dist: django>=6.1a1
+Requires-Dist: redis>=5.0 ; extra == "redis"
 Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
 Project-URL: Documentation, https://github.com/codingjoe/threadmill/
 Project-URL: Funding, https://github.com/sponsors/codingjoe
@@ -34,6 +35,7 @@ Project-URL: Homepage, https://github.com/codingjoe/threadmill
 Project-URL: Issues, https://github.com/codingjoe/threadmill/issues
 Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/latest
 Project-URL: Source, https://github.com/codingjoe/threadmill
+Provides-Extra: redis
 
 # Threadmill
 
@@ -41,7 +43,7 @@ Project-URL: Source, https://github.com/codingjoe/threadmill
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
-    <img alt="Django Grinder: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
+    <img alt="Threadmill: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
   </picture>
 <br>
   <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
@@ -65,22 +67,37 @@ Project-URL: Source, https://github.com/codingjoe/threadmill
 [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill)
 [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
 
+## Sponsors
+
+[![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
+
 ## Setup
 
-You need to have [Django's Task framework][django-tasks] setup properly.
+You need to have [Django's Task framework][django-tasks] set up properly.
 
 ```console
-uv add threadmill
+uv add threadmill[redis]
 ```
 
-Add `threadmill` to your `INSTALLED_APPS` in `settings.py`:
+Add `threadmill` to your `INSTALLED_APPS` in `settings.py`
+and configure the task backend:
 
 ```python
 # settings.py
+import os
+
 INSTALLED_APPS = [
     "threadmill",
     # ...
 ]
+
+TASKS = {
+    "default": {
+        "BACKEND": "threadmill.backends.redis.RedisTaskBackend",
+        "REDIS_URL": os.getenv("REDIS_URL", "redis://localhost:6379/0"),
+    },
+    # ...
+}
 ```
 
 Finally, you launch the worker pool:
@@ -91,9 +108,11 @@ uv run manage.py threadmill
 
 ## Usage
 
+### Workers
+
 The workers are inspired by Gunicorn, and the CLI is very similar.
 
-### Utilization
+#### Utilization
 
 Depending on your workload, you can tweak the number of processes and threads.
 Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
@@ -102,7 +121,7 @@ Processes allow for parallel compute (no GIL) while threads are great for low-me
 uv run manage.py threadmill --processes 4 --threads 2
 ```
 
-### Health
+#### Health
 
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
 
@@ -114,62 +133,34 @@ This will restart the workers after 1000 tasks have been processed, with a rando
 
 Should a worker crash or be killed, the pool will automatically restart it.
 
-### Shutdown
+#### Shutdown
 
 A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
-All workers will finish the tasks they acquired and publish them.
+All workers will finish the tasks they acquired and acknowledge them.
 
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
 
-### Task Backlog
-
-You can prefetch tasks from a queue to avoid IO latency bottlenecks.
-However, this will increase the memory usage of the worker pool.
-
-```console
-uv run manage.py threadmill --prefetch 100
-```
-
-### Task Timeouts
-
-> [!WARNING]
-> Work in progress, this feature is not yet stable.
-
-Task timeouts are important to ensure the long-term health of your pool.
-However, they need to be aligned with your queueing system's timeout settings.
-The message queue needs to requeue a task that hasn't been acknowledged within the timeout.
+### Redis Backend Options
 
-## Integration
+The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
+`TASKS` configuration:
 
-> [!NOTE]
-> This section is for people who want to integrate Threadmill into their queueing system.
+| Option            | Default                | Description                                                  |
+| ----------------- | ---------------------- | ------------------------------------------------------------ |
+| `lease_ttl`       | `timedelta(hours=1)`   | Max processing time before a started task is marked FAILED.  |
+| `result_ttl`      | `timedelta(days=1)`    | How long task results are retained before automatic removal. |
+| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes.       |
+| `batch_size`      | `100`                  | Max tasks to move or requeue per broker pass.                |
 
-Threadmill is designed to be durable and requires a queueing system to support late acknowledgement.
+A task that is started but never acknowledged (lease expired) is marked FAILED
+with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
+worst-case task runtime.
 
-To use Threadmill, your backend will need to inherit from `threadmill.backends.AcknowledgeableTaskBackend` and implement the following methods:
+All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
+every multi-key operation — including the cross-queue acquire — runs on a single
+shard. Scale horizontally by running additional backend aliases, not by relying
+on cross-slot operations.
 
-```python
-class AcknowledgeableTaskBackend(BaseTaskBackend, ABC):
-    """Provide an interface for tasks queues to be processed by the executor."""
-
-    def acquire(
-        self, *queue_names: str, timeout: datetime.timedelta | None = None
-    ) -> TaskResult:
-        """
-        Return and lock the next task to be processed without removing it from the queue.
-
-        Args:
-            queue_names: The names of the queues to acquire tasks from.
-            timeout: The maximum time to wait for a task. If None, wait indefinitely.
-
-        Raises:
-            TimeoutError: If no task is available within the specified timeout.
-        """
-        raise NotImplementedError
-
-    def acknowledge(self, task_result: TaskResult) -> None:
-        """Remove the task from the queue and publish the result."""
-        raise NotImplementedError
-```
+[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
 

{threadmill-0.1.0 → threadmill-0.2.0}/README.md

@@ -4,7 +4,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
-    <img alt="Django Grinder: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
+    <img alt="Threadmill: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
   </picture>
 <br>
   <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
@@ -28,22 +28,37 @@
 [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill)
 [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
 
+## Sponsors
+
+[![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
+
 ## Setup
 
-You need to have [Django's Task framework][django-tasks] setup properly.
+You need to have [Django's Task framework][django-tasks] set up properly.
 
 ```console
-uv add threadmill
+uv add threadmill[redis]
 ```
 
-Add `threadmill` to your `INSTALLED_APPS` in `settings.py`:
+Add `threadmill` to your `INSTALLED_APPS` in `settings.py`
+and configure the task backend:
 
 ```python
 # settings.py
+import os
+
 INSTALLED_APPS = [
     "threadmill",
     # ...
 ]
+
+TASKS = {
+    "default": {
+        "BACKEND": "threadmill.backends.redis.RedisTaskBackend",
+        "REDIS_URL": os.getenv("REDIS_URL", "redis://localhost:6379/0"),
+    },
+    # ...
+}
 ```
 
 Finally, you launch the worker pool:
@@ -54,9 +69,11 @@ uv run manage.py threadmill
 
 ## Usage
 
+### Workers
+
 The workers are inspired by Gunicorn, and the CLI is very similar.
 
-### Utilization
+#### Utilization
 
 Depending on your workload, you can tweak the number of processes and threads.
 Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
@@ -65,7 +82,7 @@ Processes allow for parallel compute (no GIL) while threads are great for low-me
 uv run manage.py threadmill --processes 4 --threads 2
 ```
 
-### Health
+#### Health
 
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
 
@@ -77,61 +94,33 @@ This will restart the workers after 1000 tasks have been processed, with a rando
 
 Should a worker crash or be killed, the pool will automatically restart it.
 
-### Shutdown
+#### Shutdown
 
 A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
-All workers will finish the tasks they acquired and publish them.
+All workers will finish the tasks they acquired and acknowledge them.
 
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
 
-### Task Backlog
-
-You can prefetch tasks from a queue to avoid IO latency bottlenecks.
-However, this will increase the memory usage of the worker pool.
-
-```console
-uv run manage.py threadmill --prefetch 100
-```
-
-### Task Timeouts
-
-> [!WARNING]
-> Work in progress, this feature is not yet stable.
-
-Task timeouts are important to ensure the long-term health of your pool.
-However, they need to be aligned with your queueing system's timeout settings.
-The message queue needs to requeue a task that hasn't been acknowledged within the timeout.
+### Redis Backend Options
 
-## Integration
+The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
+`TASKS` configuration:
 
-> [!NOTE]
-> This section is for people who want to integrate Threadmill into their queueing system.
+| Option            | Default                | Description                                                  |
+| ----------------- | ---------------------- | ------------------------------------------------------------ |
+| `lease_ttl`       | `timedelta(hours=1)`   | Max processing time before a started task is marked FAILED.  |
+| `result_ttl`      | `timedelta(days=1)`    | How long task results are retained before automatic removal. |
+| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes.       |
+| `batch_size`      | `100`                  | Max tasks to move or requeue per broker pass.                |
 
-Threadmill is designed to be durable and requires a queueing system to support late acknowledgement.
+A task that is started but never acknowledged (lease expired) is marked FAILED
+with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
+worst-case task runtime.
 
-To use Threadmill, your backend will need to inherit from `threadmill.backends.AcknowledgeableTaskBackend` and implement the following methods:
+All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
+every multi-key operation — including the cross-queue acquire — runs on a single
+shard. Scale horizontally by running additional backend aliases, not by relying
+on cross-slot operations.
 
-```python
-class AcknowledgeableTaskBackend(BaseTaskBackend, ABC):
-    """Provide an interface for tasks queues to be processed by the executor."""
-
-    def acquire(
-        self, *queue_names: str, timeout: datetime.timedelta | None = None
-    ) -> TaskResult:
-        """
-        Return and lock the next task to be processed without removing it from the queue.
-
-        Args:
-            queue_names: The names of the queues to acquire tasks from.
-            timeout: The maximum time to wait for a task. If None, wait indefinitely.
-
-        Raises:
-            TimeoutError: If no task is available within the specified timeout.
-        """
-        raise NotImplementedError
-
-    def acknowledge(self, task_result: TaskResult) -> None:
-        """Remove the task from the queue and publish the result."""
-        raise NotImplementedError
-```
+[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/

{threadmill-0.1.0 → threadmill-0.2.0}/pyproject.toml

@@ -35,6 +35,9 @@ classifiers = [
 requires-python = ">=3.12"
 dependencies = ["django>=6.1a1"]
 
+[project.optional-dependencies]
+redis = ["redis>=5.0"]
+
 [project.urls]
 # https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels
 Homepage = "https://github.com/codingjoe/threadmill"
@@ -56,9 +59,9 @@ minversion = "6.0"
 addopts = "--cov --cov-report=xml --cov-report=term --tb=short -rxs --benchmark-autosave --benchmark-group-by=fullname --benchmark-min-rounds=10"
 testpaths = ["tests"]
 DJANGO_SETTINGS_MODULE = "tests.testapp.settings"
+asyncio_mode = "auto"
 markers = [
   "benchmark: mark benchmark tests.",
-  "integration: mark integration tests.",
 ]
 
 [tool.coverage.run]
@@ -91,6 +94,7 @@ combine-as-imports = true
 split-on-trailing-comma = true
 section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
 force-wrap-aliases = true
+known-first-party = ["threadmill", "tests"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "pep257"
@@ -105,4 +109,5 @@ test = [
   "pytest-asyncio",
   "pytest-cov",
   "pytest-django",
+  "redis>=5.0",
 ]

{threadmill-0.1.0 → threadmill-0.2.0}/threadmill/__init__.py

@@ -1,4 +1,4 @@
-"""A queue agnostic worker for Django's task framework."""
+"""The most reliable backend for Django's task framework."""
 
 from . import _version
 

{threadmill-0.1.0 → threadmill-0.2.0}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
-__commit_id__ = commit_id = 'g7270fd320'
+__commit_id__ = commit_id = 'gb135558ff'

threadmill-0.2.0/threadmill/backends/base.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import collections.abc
+import dataclasses
+import datetime
+import json
+import threading
+from abc import ABC
+
+from django.core.serializers.json import DjangoJSONEncoder
+from django.tasks import DEFAULT_TASK_QUEUE_NAME, Task, TaskResult, TaskResultStatus
+from django.tasks.backends.base import BaseTaskBackend
+from django.tasks.base import TaskError
+from django.utils.module_loading import import_string
+
+
+class Broker(threading.Thread):
+    """Backend maintenance thread launched by the task executor."""
+
+    def __init__(
+        self,
+        backend: ThreadmillTaskBackend | None = None,
+        *,
+        interval: datetime.timedelta = datetime.timedelta(seconds=1),
+    ) -> None:
+        super().__init__(daemon=True)
+        self.backend = backend
+        self.interval = interval
+        self.shutdown_requested = threading.Event()
+
+    def main(self) -> None:
+        """Perform one maintenance pass."""
+
+    def run(self) -> None:
+        while not self.shutdown_requested.wait(self.interval.total_seconds()):
+            self.main()
+
+    def shutdown(self) -> None:
+        """Request graceful shutdown."""
+        self.shutdown_requested.set()
+
+
+def _parse_datetime(value: object) -> object:
+    """Parse an ISO datetime string, returning the value unchanged if not parseable."""
+    if isinstance(value, str):
+        try:
+            return datetime.datetime.fromisoformat(value)
+        except ValueError:
+            return value
+    return value
+
+
+class TaskResultEncoder(DjangoJSONEncoder):
+    """JSON encoder for TaskResult and TaskError objects."""
+
+    def default(self, o):
+        if isinstance(o, (TaskResult, TaskError)):
+            return {
+                field.name: getattr(o, field.name)
+                for field in dataclasses.fields(type(o))
+            }
+        if isinstance(o, Task):
+            return {
+                field.name: getattr(o, field.name)
+                for field in dataclasses.fields(Task)
+                if field.name != "func"  # Exclude the function object itself
+            } | {"func": o.module_path}
+        return super().default(o)
+
+
+class ThreadmillTaskBackend(BaseTaskBackend, ABC):
+    """Interface for task queues to be processed by the executor."""
+
+    supports_async_task = True
+    supports_get_result = True
+    broker_class: type[Broker] | None = None
+
+    result_ttl: datetime.timedelta | None = None
+
+    @staticmethod
+    def serialize_task_result(task_result: TaskResult) -> str:
+        return json.dumps(task_result, cls=TaskResultEncoder)
+
+    @classmethod
+    def deserialize_task_result(cls, data: str) -> TaskResult:
+        def _object_hook(d: dict) -> dict | TaskResult:
+            if "task" in d and isinstance(d["task"], dict) and "func" in d["task"]:
+                task_data = d["task"]
+                func = import_string(task_data["func"])
+                if isinstance(func, cls.task_class):
+                    func = func.func
+                d["task"] = cls.task_class(
+                    func=func,
+                    **{
+                        field.name: _parse_datetime(task_data[field.name])
+                        for field in dataclasses.fields(cls.task_class)
+                        if field.name not in {"func", "takes_context"}
+                        and field.name in task_data
+                    },
+                )
+                d["status"] = TaskResultStatus(d["status"])
+                d["errors"] = [TaskError(**error) for error in d["errors"]]
+                return_value = d.pop("_return_value", None)
+                for key, value in d.items():
+                    d[key] = _parse_datetime(value)
+                result = TaskResult(**d)
+                object.__setattr__(result, "_return_value", return_value)
+                return result
+            return d
+
+        return json.loads(data, object_hook=_object_hook)
+
+    def acquire(
+        self,
+        *queue_names: str,
+        timeout: datetime.timedelta | None = None,
+        worker: str = "",
+    ) -> TaskResult:
+        """
+        Return and lock the next task to be processed without removing it from the queue.
+
+        Args:
+            queue_names: The names of the queues to acquire tasks from.
+            timeout: The maximum time to wait for a task. If None, wait indefinitely.
+            worker: The name of the worker thread acquiring the task.
+
+        Raises:
+            TimeoutError: If no task is available within the specified timeout.
+            queue.Empty: If no task is available and timeout is None.
+        """
+        raise NotImplementedError
+
+    def acknowledge(self, task_result: TaskResult) -> None:
+        """Remove the task from the queue and publish the result."""
+        raise NotImplementedError
+
+    def peek(
+        self,
+        queue_name: str = DEFAULT_TASK_QUEUE_NAME,
+        *,
+        status: TaskResultStatus | None = None,
+        count: int = 1,
+    ) -> collections.abc.Generator[TaskResult, None, None]:
+        """Yield acknowledged results from a queue, optionally filtered by status."""
+        raise NotImplementedError

threadmill-0.2.0/threadmill/backends/lua/acknowledge.lua

@@ -0,0 +1,24 @@
+-- Finalize a completed task: remove it from the running set, persist the
+-- result with a TTL, delete the task data hash, and add the result to the
+-- results history set. Also evict results whose finish score falls outside the
+-- retention window.
+--
+-- KEYS[1]  -- running set (ZSET)
+-- KEYS[2]  -- result key (STRING, stores serialized TaskResult)
+-- KEYS[3]  -- task data key (HASH, deleted after acknowledge)
+-- KEYS[4]  -- results history set (ZSET, ordered by finish time)
+-- ARGV[1]  -- task ID
+-- ARGV[2]  -- serialized TaskResult JSON
+-- ARGV[3]  -- result TTL in seconds
+-- ARGV[4]  -- finish timestamp in milliseconds (score for the results set)
+-- Returns: 1 on success, 0 if task was not in the running set
+
+local removed = redis.call('ZREM', KEYS[1], ARGV[1])
+if removed == 0 then
+  return 0  -- Task already reaped, skip
+end
+redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
+redis.call('DEL', KEYS[3])
+redis.call('ZADD', KEYS[4], ARGV[4], ARGV[1])
+redis.call('ZREMRANGEBYSCORE', KEYS[4], 0, tonumber(ARGV[4]) - tonumber(ARGV[3]) * 1000)
+return 1

threadmill-0.2.0/threadmill/backends/lua/acquire.lua

@@ -0,0 +1,44 @@
+-- Atomically pop the lowest-scored task from any of the given priority queues,
+-- update its JSON data with worker info, and move it directly to the running
+-- set. Iterates queues in key order and returns the first available task.
+--
+-- KEYS[1..N]  -- interleaved running keys and queue keys, one pair per queue:
+--                KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
+--                KEYS[4] = queue, etc.
+-- ARGV[1]     -- current time in milliseconds
+-- ARGV[2]     -- current time as ISO-8601 string
+-- ARGV[3]     -- task key prefix (e.g. "threadmill:default:task:")
+-- ARGV[4]     -- number of queue pairs (N/2)
+-- ARGV[5]     -- worker name
+-- ARGV[6]     -- lease TTL in milliseconds
+-- Returns: updated serialized data on success, nil if all queues are empty.
+
+local num_queues = tonumber(ARGV[4])
+local lease_ttl_ms = tonumber(ARGV[6])
+for i = 1, num_queues do
+  local result = redis.call('ZPOPMIN', KEYS[i * 2])
+  if #result > 0 then
+    local task_id = result[1]
+    local data = redis.call('HGET', ARGV[3] .. task_id, 'data')
+    if data then
+      local ok, parsed = pcall(cjson.decode, data)
+      if ok then
+        parsed.status = 'RUNNING'
+        parsed.last_attempted_at = ARGV[2]
+        if not parsed.started_at then
+          parsed.started_at = ARGV[2]
+        end
+        if not parsed.worker_ids then
+          parsed.worker_ids = {}
+        end
+        table.insert(parsed.worker_ids, ARGV[5])
+        local updated_data = cjson.encode(parsed)
+        local deadline = tonumber(ARGV[1]) + lease_ttl_ms
+        redis.call('ZADD', KEYS[i * 2 - 1], deadline, task_id)
+        redis.call('HSET', ARGV[3] .. task_id, 'data', updated_data)
+        return updated_data
+      end
+    end
+  end
+end
+return nil

threadmill-0.2.0/threadmill/backends/lua/mover.lua

@@ -0,0 +1,19 @@
+-- Move tasks whose scheduled time has passed from the deferred set to the active
+-- priority queue. Only processes up to a batch limit per call.
+--
+-- KEYS[1]  -- deferred set (ZSET, scored by run_after timestamp)
+-- KEYS[2]  -- active priority queue (ZSET, scored by priority+time)
+-- ARGV[1]  -- current time in milliseconds (all scores <= this are due)
+-- ARGV[2]  -- task key prefix (e.g. "threadmill:default:task:")
+-- ARGV[3]  -- maximum number of tasks to move per call (batch size)
+-- Returns: number of tasks moved
+
+local due = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1], 'LIMIT', 0, tonumber(ARGV[3]))
+for _, task_id in ipairs(due) do
+  redis.call('ZREM', KEYS[1], task_id)
+  local score = redis.call('HGET', ARGV[2] .. task_id, 'score')
+  if score then
+    redis.call('ZADD', KEYS[2], score, task_id)
+  end
+end
+return #due

threadmill-0.2.0/threadmill/backends/lua/reaper.lua

@@ -0,0 +1,36 @@
+-- Fail tasks whose processing lease has expired.
+--
+-- KEYS[1]  -- running set (ZSET)
+-- KEYS[2]  -- results history set (ZSET)
+-- ARGV[1]  -- current time in milliseconds (for score comparison)
+-- ARGV[2]  -- task key prefix (e.g. "threadmill:default:task:")
+-- ARGV[3]  -- result key prefix (e.g. "threadmill:default:result:")
+-- ARGV[4]  -- batch size
+-- ARGV[5]  -- result TTL in seconds
+-- ARGV[6]  -- finished_at as ISO format string
+-- Returns: number of tasks failed
+
+local stale = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1], 'LIMIT', 0, tonumber(ARGV[4]))
+for _, task_id in ipairs(stale) do
+  local data = redis.call('HGET', ARGV[2] .. task_id, 'data')
+  if data then
+    local ok, parsed = pcall(cjson.decode, data)
+    if ok then
+      parsed.status = 'FAILED'
+      parsed.finished_at = ARGV[6]
+      if not parsed.errors then
+        parsed.errors = {}
+      end
+      table.insert(parsed.errors, {
+        exception_class_path = 'threadmill.exceptions.AcknowledgementTimeout',
+        traceback = 'Task processing lease expired.'
+      })
+      local failed_data = cjson.encode(parsed)
+      redis.call('ZREM', KEYS[1], task_id)
+      redis.call('SET', ARGV[3] .. task_id, failed_data, 'EX', ARGV[5])
+      redis.call('DEL', ARGV[2] .. task_id)
+      redis.call('ZADD', KEYS[2], tonumber(ARGV[1]), task_id)
+    end
+  end
+end
+return #stale