threadmill 0.1.0__tar.gz → 0.3.1__tar.gz

threadmill-0.3.1/PKG-INFO

@@ -0,0 +1,180 @@
+Metadata-Version: 2.4
+Name: threadmill
+Version: 0.3.1
+Summary: The most reliable backend for Django's task framework.
+Keywords: Django,tasks,worker
+Author-email: Johannes Maron <johannes@maron.family>
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Environment :: Web Environment
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX
+Classifier: Topic :: Communications :: Email
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Software Development
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 6.0
+Classifier: Framework :: Django :: 6.1
+License-File: LICENSE
+Requires-Dist: django>=6.0
+Requires-Dist: textual>=8.2.7 ; extra == "inspector"
+Requires-Dist: redis>=5.0 ; extra == "redis"
+Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
+Project-URL: Documentation, https://github.com/codingjoe/threadmill/
+Project-URL: Funding, https://github.com/sponsors/codingjoe
+Project-URL: Homepage, https://github.com/codingjoe/threadmill
+Project-URL: Issues, https://github.com/codingjoe/threadmill/issues
+Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/latest
+Project-URL: Source, https://github.com/codingjoe/threadmill
+Provides-Extra: inspector
+Provides-Extra: redis
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
+    <img alt="Threadmill: Durable high-performance backend for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
+  </picture>
+<br>
+  <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
+  <a href="https://github.com/codingjoe/threadmill/issues/new/choose">Issues</a> |
+  <a href="https://github.com/codingjoe/threadmill/releases">Changelog</a> |
+  <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
+</p>
+
+# Threadmill [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
+
+**Durable high-performance backend for Django's task framework.**
+
+## Design Principles
+
+- **Durability** – We recover from any failures, even poorly written tasks.
+- **Consistency** – We never lose data, even if someone unplugs the power or network.
+- **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks.
+
+## Sponsors
+
+[![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
+
+## Setup
+
+You need to have [Django's Task framework][django-tasks] set up properly.
+
+```console
+uv add threadmill[redis]
+```
+
+Add `threadmill` to your `INSTALLED_APPS` in `settings.py`
+and configure the task backend:
+
+```python
+# settings.py
+import os
+
+INSTALLED_APPS = [
+    "threadmill",
+    # ...
+]
+
+TASKS = {
+    "default": {
+        "BACKEND": "threadmill.backends.redis.RedisTaskBackend",
+        "REDIS_URL": os.getenv("REDIS_URL", "redis://localhost:6379/0"),
+    },
+    # ...
+}
+```
+
+Optionally, install the inspector dependency if you want the TUI:
+
+```console
+uv add threadmill[inspector]
+```
+
+Then launch the worker pool:
+
+```console
+uv run manage.py threadmill worker
+```
+
+## Usage
+
+### Workers
+
+The workers are inspired by Gunicorn, and the CLI is very similar.
+
+#### Utilization
+
+Depending on your workload, you can tweak the number of processes and threads.
+Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
+
+```console
+uv run manage.py threadmill worker --processes 4 --threads 2
+```
+
+#### Health
+
+If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
+
+```console
+uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
+```
+
+This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
+
+Should a worker crash or be killed, the pool will automatically restart it.
+
+#### Shutdown
+
+A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
+All workers will finish the tasks they acquired and acknowledge them.
+
+You can use `--exit-empty` to exit immediately after all tasks have been processed,
+which might be useful for draining a one-off queue.
+
+### Inspector
+
+![Inspector TUI screenshot](https://github.com/codingjoe/threadmill/raw/main/docs/images/TUI-screenshot.svg)
+
+The optional TUI inspector lets you watch queues, tasks, and task details in real-time.
+Install it with the `inspector` extra and launch it from a separate terminal:
+
+```console
+uv add threadmill[inspector]
+uv run manage.py threadmill inspector
+```
+
+### Redis Backend Options
+
+The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
+`TASKS` configuration:
+
+| Option            | Default                | Description                                                  |
+| ----------------- | ---------------------- | ------------------------------------------------------------ |
+| `lease_ttl`       | `timedelta(hours=1)`   | Max processing time before a started task is marked FAILED.  |
+| `result_ttl`      | `timedelta(days=1)`    | How long task results are retained before automatic removal. |
+| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes.       |
+| `batch_size`      | `100`                  | Max tasks to move or requeue per broker pass.                |
+
+A task that is started but never acknowledged (lease expired) is marked FAILED
+with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
+worst-case task runtime.
+
+All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
+every multi-key operation — including the cross-queue acquire — runs on a single
+shard. Scale horizontally by running additional backend aliases, not by relying
+on cross-slot operations.
+
+[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
+

threadmill-0.3.1/README.md

@@ -0,0 +1,137 @@
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
+    <img alt="Threadmill: Durable high-performance backend for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
+  </picture>
+<br>
+  <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
+  <a href="https://github.com/codingjoe/threadmill/issues/new/choose">Issues</a> |
+  <a href="https://github.com/codingjoe/threadmill/releases">Changelog</a> |
+  <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
+</p>
+
+# Threadmill [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
+
+**Durable high-performance backend for Django's task framework.**
+
+## Design Principles
+
+- **Durability** – We recover from any failures, even poorly written tasks.
+- **Consistency** – We never lose data, even if someone unplugs the power or network.
+- **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks.
+
+## Sponsors
+
+[![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
+
+## Setup
+
+You need to have [Django's Task framework][django-tasks] set up properly.
+
+```console
+uv add threadmill[redis]
+```
+
+Add `threadmill` to your `INSTALLED_APPS` in `settings.py`
+and configure the task backend:
+
+```python
+# settings.py
+import os
+
+INSTALLED_APPS = [
+    "threadmill",
+    # ...
+]
+
+TASKS = {
+    "default": {
+        "BACKEND": "threadmill.backends.redis.RedisTaskBackend",
+        "REDIS_URL": os.getenv("REDIS_URL", "redis://localhost:6379/0"),
+    },
+    # ...
+}
+```
+
+Optionally, install the inspector dependency if you want the TUI:
+
+```console
+uv add threadmill[inspector]
+```
+
+Then launch the worker pool:
+
+```console
+uv run manage.py threadmill worker
+```
+
+## Usage
+
+### Workers
+
+The workers are inspired by Gunicorn, and the CLI is very similar.
+
+#### Utilization
+
+Depending on your workload, you can tweak the number of processes and threads.
+Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
+
+```console
+uv run manage.py threadmill worker --processes 4 --threads 2
+```
+
+#### Health
+
+If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
+
+```console
+uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
+```
+
+This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
+
+Should a worker crash or be killed, the pool will automatically restart it.
+
+#### Shutdown
+
+A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
+All workers will finish the tasks they acquired and acknowledge them.
+
+You can use `--exit-empty` to exit immediately after all tasks have been processed,
+which might be useful for draining a one-off queue.
+
+### Inspector
+
+![Inspector TUI screenshot](https://github.com/codingjoe/threadmill/raw/main/docs/images/TUI-screenshot.svg)
+
+The optional TUI inspector lets you watch queues, tasks, and task details in real-time.
+Install it with the `inspector` extra and launch it from a separate terminal:
+
+```console
+uv add threadmill[inspector]
+uv run manage.py threadmill inspector
+```
+
+### Redis Backend Options
+
+The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
+`TASKS` configuration:
+
+| Option            | Default                | Description                                                  |
+| ----------------- | ---------------------- | ------------------------------------------------------------ |
+| `lease_ttl`       | `timedelta(hours=1)`   | Max processing time before a started task is marked FAILED.  |
+| `result_ttl`      | `timedelta(days=1)`    | How long task results are retained before automatic removal. |
+| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes.       |
+| `batch_size`      | `100`                  | Max tasks to move or requeue per broker pass.                |
+
+A task that is started but never acknowledged (lease expired) is marked FAILED
+with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
+worst-case task runtime.
+
+All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
+every multi-key operation — including the cross-queue acquire — runs on a single
+shard. Scale horizontally by running additional backend aliases, not by relying
+on cross-slot operations.
+
+[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/

{threadmill-0.1.0 → threadmill-0.3.1}/pyproject.toml

@@ -30,10 +30,17 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: 3.14",
   "Framework :: Django",
+  "Framework :: Django :: 6.0",
   "Framework :: Django :: 6.1",
 ]
 requires-python = ">=3.12"
-dependencies = ["django>=6.1a1"]
+dependencies = ["django>=6.0"]
+
+[project.optional-dependencies]
+redis = ["redis>=5.0"]
+inspector = [
+    "textual>=8.2.7",
+]
 
 [project.urls]
 # https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels
@@ -56,9 +63,9 @@ minversion = "6.0"
 addopts = "--cov --cov-report=xml --cov-report=term --tb=short -rxs --benchmark-autosave --benchmark-group-by=fullname --benchmark-min-rounds=10"
 testpaths = ["tests"]
 DJANGO_SETTINGS_MODULE = "tests.testapp.settings"
+asyncio_mode = "auto"
 markers = [
   "benchmark: mark benchmark tests.",
-  "integration: mark integration tests.",
 ]
 
 [tool.coverage.run]
@@ -91,6 +98,7 @@ combine-as-imports = true
 split-on-trailing-comma = true
 section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
 force-wrap-aliases = true
+known-first-party = ["threadmill", "tests"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "pep257"
@@ -105,4 +113,5 @@ test = [
   "pytest-asyncio",
   "pytest-cov",
   "pytest-django",
+  "redis>=5.0",
 ]

{threadmill-0.1.0 → threadmill-0.3.1}/threadmill/__init__.py

@@ -1,4 +1,4 @@
-"""A queue agnostic worker for Django's task framework."""
+"""The most reliable backend for Django's task framework."""
 
 from . import _version
 

{threadmill-0.1.0 → threadmill-0.3.1}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.3.1'
+__version_tuple__ = version_tuple = (0, 3, 1)
 
-__commit_id__ = commit_id = 'g7270fd320'
+__commit_id__ = commit_id = 'g32b680b8b'

threadmill-0.3.1/threadmill/backends/base.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import collections.abc
+import dataclasses
+import datetime
+import json
+import threading
+from abc import ABC
+
+import django
+from django.core.serializers.json import DjangoJSONEncoder
+from django.tasks import DEFAULT_TASK_QUEUE_NAME, Task, TaskResult, TaskResultStatus
+from django.tasks.backends.base import BaseTaskBackend
+from django.tasks.base import TaskError
+from django.utils.module_loading import import_string
+
+if django.VERSION == (6, 0):
+    # https://github.com/django/django/commit/8c8b833d32c02d3ae6f43b04bb1e45968796b402
+    @dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
+    class Task(Task):
+        @classmethod
+        def _reconstruct(cls, kwargs):
+            func_path = kwargs["func"]
+            try:
+                func = import_string(func_path)
+                kwargs["func"] = func.func
+            except (ImportError, AttributeError) as e:
+                msg = f"Expected {func_path!r} to point to a Task instance."
+                raise ValueError(msg) from e
+            return cls(**kwargs)
+
+        def __reduce__(self):
+            kwargs = {f.name: getattr(self, f.name) for f in dataclasses.fields(self)}
+            kwargs["func"] = self.module_path
+
+            return (self.__class__._reconstruct, (kwargs,))
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class QueueCounts:
+    """Point-in-time cardinality of each queue segment."""
+
+    ready: int
+    running: int
+    deferred: int
+    successful: int
+    failed: int
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class QueueRates:
+    """Rolling ingress/egress throughput over a time window (time-series data)."""
+
+    interval: datetime.timedelta
+    ingress: int
+    egress: int
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class QueueStats:
+    """Telemetry for a single queue: point-in-time counts plus rolling rates."""
+
+    counts: QueueCounts
+    rates: QueueRates
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class BackendTelemetry:
+    """Snapshot of counts and rates across a backend's queues."""
+
+    queues: dict[str, QueueStats]
+
+
+class Broker(threading.Thread):
+    """Backend maintenance thread launched by the task executor."""
+
+    def __init__(
+        self,
+        backend: ThreadmillTaskBackend | None = None,
+        *,
+        interval: datetime.timedelta = datetime.timedelta(seconds=1),
+    ) -> None:
+        super().__init__(daemon=True)
+        self.backend = backend
+        self.interval = interval
+        self.shutdown_requested = threading.Event()
+
+    def main(self) -> None:
+        """Perform one maintenance pass."""
+
+    def run(self) -> None:
+        while not self.shutdown_requested.wait(self.interval.total_seconds()):
+            self.main()
+
+    def shutdown(self) -> None:
+        """Request graceful shutdown."""
+        self.shutdown_requested.set()
+
+
+def _parse_datetime(value: object) -> object:
+    """Parse an ISO datetime string, returning the value unchanged if not parseable."""
+    if isinstance(value, str):
+        try:
+            return datetime.datetime.fromisoformat(value)
+        except ValueError:
+            return value
+    return value
+
+
+class TaskResultEncoder(DjangoJSONEncoder):
+    """JSON encoder for TaskResult and TaskError objects."""
+
+    def default(self, o):
+        if isinstance(o, (TaskResult, TaskError)):
+            return {
+                field.name: getattr(o, field.name)
+                for field in dataclasses.fields(type(o))
+            }
+        if isinstance(o, Task):
+            return {
+                field.name: getattr(o, field.name)
+                for field in dataclasses.fields(Task)
+                if field.name != "func"  # Exclude the function object itself
+            } | {"func": o.module_path}
+        return super().default(o)
+
+
+class ThreadmillTaskBackend(BaseTaskBackend, ABC):
+    """Interface for task queues to be processed by the executor."""
+
+    task_class = Task  # can be removed in the future when Django 6.0 support is dropped
+    supports_async_task = True
+    supports_get_result = True
+    broker_class: type[Broker] | None = None
+
+    result_ttl: datetime.timedelta | None = None
+
+    @staticmethod
+    def serialize_task_result(task_result: TaskResult) -> str:
+        return json.dumps(task_result, cls=TaskResultEncoder)
+
+    @classmethod
+    def deserialize_task_result(cls, data: str) -> TaskResult:
+        def _object_hook(d: dict) -> dict | TaskResult:
+            if "task" in d and isinstance(d["task"], dict) and "func" in d["task"]:
+                task_data = d["task"]
+                func = import_string(task_data["func"])
+                if isinstance(func, cls.task_class):
+                    func = func.func
+                d["task"] = cls.task_class(
+                    func=func,
+                    **{
+                        field.name: _parse_datetime(task_data[field.name])
+                        for field in dataclasses.fields(cls.task_class)
+                        if field.name not in {"func", "takes_context"}
+                        and field.name in task_data
+                    },
+                )
+                d["status"] = TaskResultStatus(d["status"])
+                d["errors"] = [TaskError(**error) for error in d["errors"]]
+                return_value = d.pop("_return_value", None)
+                for key, value in d.items():
+                    d[key] = _parse_datetime(value)
+                result = TaskResult(**d)
+                object.__setattr__(result, "_return_value", return_value)
+                return result
+            return d
+
+        return json.loads(data, object_hook=_object_hook)
+
+    def acquire(
+        self,
+        *queue_names: str,
+        timeout: datetime.timedelta | None = None,
+        worker: str = "",
+    ) -> TaskResult:
+        """
+        Return and lock the next task to be processed without removing it from the queue.
+
+        Args:
+            queue_names: The names of the queues to acquire tasks from.
+            timeout: The maximum time to wait for a task. If None, wait indefinitely.
+            worker: The name of the worker thread acquiring the task.
+
+        Raises:
+            TimeoutError: If no task is available within the specified timeout.
+            queue.Empty: If no task is available and timeout is None.
+        """
+        raise NotImplementedError
+
+    def acknowledge(self, task_result: TaskResult) -> None:
+        """Remove the task from the queue and publish the result."""
+        raise NotImplementedError
+
+    def peek(
+        self,
+        queue_name: str = DEFAULT_TASK_QUEUE_NAME,
+        *,
+        status: TaskResultStatus,
+        count: int = 1,
+    ) -> collections.abc.Generator[TaskResult, None, None]:
+        """
+        Yield up to ``count`` tasks from a queue in the given status segment.
+
+        Args:
+            queue_name: The name of the queue to peek into.
+            status: The status of the tasks to yield.
+            count: The maximum number of tasks to yield. If 0, yield all available tasks.
+        """
+        raise NotImplementedError
+
+    def telemetry(
+        self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
+    ) -> BackendTelemetry:
+        """Return a snapshot of stats for all configured queues.
+
+        Args:
+            interval: The time window for rolling rates.
+        """
+        raise NotImplementedError

threadmill-0.3.1/threadmill/backends/lua/acknowledge.lua

@@ -0,0 +1,34 @@
+-- Finalize a completed task: remove it from the running set, persist the
+-- result with a TTL, delete the task data hash, and add the result to the
+-- per-status results history set. Evicts results whose finish score falls
+-- outside the retention window (result_ttl). The per-status history sets are
+-- the time series the inspector counts over for telemetry, so no separate
+-- egress window or status counters are needed here.
+--
+-- KEYS[1]  -- running set (ZSET)
+-- KEYS[2]  -- result key (STRING, stores serialized TaskResult)
+-- KEYS[3]  -- task data key (HASH, deleted after acknowledge)
+-- KEYS[4]  -- successful results history (ZSET, scored by finish time)
+-- KEYS[5]  -- failed results history (ZSET, scored by finish time)
+-- ARGV[1]  -- task ID
+-- ARGV[2]  -- serialized TaskResult JSON
+-- ARGV[3]  -- result TTL in seconds
+-- ARGV[4]  -- finish timestamp in milliseconds (score for the history set)
+-- ARGV[5]  -- status (SUCCESSFUL or FAILED)
+-- Returns: 1 on success, 0 if task was not in the running set
+
+local removed = redis.call('ZREM', KEYS[1], ARGV[1])
+if removed == 0 then
+  return 0  -- Task already reaped, skip
+end
+redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
+redis.call('DEL', KEYS[3])
+local finish = tonumber(ARGV[4])
+local cutoff = finish - tonumber(ARGV[3]) * 1000
+local results_key = KEYS[4]
+if ARGV[5] ~= 'SUCCESSFUL' then
+  results_key = KEYS[5]
+end
+redis.call('ZADD', results_key, finish, ARGV[1])
+redis.call('ZREMRANGEBYSCORE', results_key, 0, cutoff)
+return 1

threadmill-0.3.1/threadmill/backends/lua/acquire.lua

@@ -0,0 +1,44 @@
+-- Atomically pop the lowest-scored task from any of the given priority queues,
+-- update its JSON data with worker info, and move it directly to the running
+-- set. Iterates queues in key order and returns the first available task.
+--
+-- KEYS[1..N]  -- interleaved running keys and queue keys, one pair per queue:
+--                KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
+--                KEYS[4] = queue, etc.
+-- ARGV[1]     -- current time in milliseconds
+-- ARGV[2]     -- current time as ISO-8601 string
+-- ARGV[3]     -- task key prefix (e.g. "threadmill:default:task:")
+-- ARGV[4]     -- number of queue pairs (N/2)
+-- ARGV[5]     -- worker name
+-- ARGV[6]     -- lease TTL in milliseconds
+-- Returns: updated serialized data on success, nil if all queues are empty.
+
+local num_queues = tonumber(ARGV[4])
+local lease_ttl_ms = tonumber(ARGV[6])
+for i = 1, num_queues do
+  local result = redis.call('ZPOPMIN', KEYS[i * 2])
+  if #result > 0 then
+    local task_id = result[1]
+    local data = redis.call('HGET', ARGV[3] .. task_id, 'data')
+    if data then
+      local ok, parsed = pcall(cjson.decode, data)
+      if ok then
+        parsed.status = 'RUNNING'
+        parsed.last_attempted_at = ARGV[2]
+        if not parsed.started_at then
+          parsed.started_at = ARGV[2]
+        end
+        if not parsed.worker_ids then
+          parsed.worker_ids = {}
+        end
+        table.insert(parsed.worker_ids, ARGV[5])
+        local updated_data = cjson.encode(parsed)
+        local deadline = tonumber(ARGV[1]) + lease_ttl_ms
+        redis.call('ZADD', KEYS[i * 2 - 1], deadline, task_id)
+        redis.call('HSET', ARGV[3] .. task_id, 'data', updated_data)
+        return updated_data
+      end
+    end
+  end
+end
+return nil