threadmill 0.7.0__tar.gz → 0.7.2__tar.gz

{threadmill-0.7.0 → threadmill-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.7.0
+Version: 0.7.2
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>
@@ -18,6 +18,7 @@ Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Communications :: Email
 Classifier: Topic :: Software Development
 Classifier: Topic :: Text Processing :: Markup :: Markdown
@@ -151,12 +152,14 @@ uv run manage.py threadmill inspector
 The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
 `TASKS` configuration:
 
-| Option            | Default                | Description                                                  |
-| ----------------- | ---------------------- | ------------------------------------------------------------ |
-| `lease_ttl`       | `timedelta(hours=1)`   | Max processing time before a started task is marked FAILED.  |
-| `result_ttl`      | `timedelta(days=1)`    | How long task results are retained before automatic removal. |
-| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes.       |
-| `batch_size`      | `100`                  | Max tasks to move or requeue per broker pass.                |
+| Option              | Default                   | Description                                                             |
+| ------------------- | ------------------------- | ----------------------------------------------------------------------- |
+| `lease_ttl`         | `timedelta(hours=1)`      | Max processing time before a started task is marked FAILED.             |
+| `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.            |
+| `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                  |
+| `batch_size`        | `100`                     | Max tasks to move or requeue per broker pass.                           |
+| `poll_interval`     | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll. |
+| `poll_max_interval` | `timedelta(seconds=1)`    | Max wait between idle acquire attempts.                                 |
 
 A task that is started but never acknowledged (lease expired) is marked FAILED
 with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your

{threadmill-0.7.0 → threadmill-0.7.2}/README.md

@@ -114,12 +114,14 @@ uv run manage.py threadmill inspector
 The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
 `TASKS` configuration:
 
-| Option            | Default                | Description                                                  |
-| ----------------- | ---------------------- | ------------------------------------------------------------ |
-| `lease_ttl`       | `timedelta(hours=1)`   | Max processing time before a started task is marked FAILED.  |
-| `result_ttl`      | `timedelta(days=1)`    | How long task results are retained before automatic removal. |
-| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes.       |
-| `batch_size`      | `100`                  | Max tasks to move or requeue per broker pass.                |
+| Option              | Default                   | Description                                                             |
+| ------------------- | ------------------------- | ----------------------------------------------------------------------- |
+| `lease_ttl`         | `timedelta(hours=1)`      | Max processing time before a started task is marked FAILED.             |
+| `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.            |
+| `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                  |
+| `batch_size`        | `100`                     | Max tasks to move or requeue per broker pass.                           |
+| `poll_interval`     | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll. |
+| `poll_max_interval` | `timedelta(seconds=1)`    | Max wait between idle acquire attempts.                                 |
 
 A task that is started but never acknowledged (lease expired) is marked FAILED
 with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your

{threadmill-0.7.0 → threadmill-0.7.2}/pyproject.toml

@@ -24,6 +24,7 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.14",
+  "Programming Language :: Python :: 3.15",
   "Topic :: Communications :: Email",
   "Topic :: Software Development",
   "Topic :: Text Processing :: Markup :: Markdown",

{threadmill-0.7.0 → threadmill-0.7.2}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.7.0'
-__version_tuple__ = version_tuple = (0, 7, 0)
+__version__ = version = '0.7.2'
+__version_tuple__ = version_tuple = (0, 7, 2)
 
-__commit_id__ = commit_id = 'g0fc4d7ade'
+__commit_id__ = commit_id = 'g560d3b3b5'

{threadmill-0.7.0 → threadmill-0.7.2}/threadmill/backends/lua/acquire.lua

@@ -1,6 +1,7 @@
 -- Atomically pop the lowest-scored task from any of the given priority queues,
 -- update its JSON data with worker info, and move it directly to the running
--- set. Iterates queues in key order and returns the first available task.
+-- set. Scans the queues round-robin from ARGV[7] and returns the first available
+-- task, so a backlogged queue cannot starve its neighbours.
 --
 -- KEYS[1..N]  -- interleaved running keys and queue keys, one pair per queue:
 --                KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
@@ -11,12 +12,16 @@
 -- ARGV[4]     -- number of queue pairs (N/2)
 -- ARGV[5]     -- worker name
 -- ARGV[6]     -- lease TTL in milliseconds
+-- ARGV[7]     -- start_index; 0-based index of the queue pair to scan first, so
+--                start_index 0 is the pair at KEYS[1] and KEYS[2]
 -- Returns: updated serialized data on success, nil if all queues are empty.
 
 local num_queues = tonumber(ARGV[4])
 local lease_ttl_ms = tonumber(ARGV[6])
-for i = 1, num_queues do
-  local result = redis.call('ZPOPMIN', KEYS[i * 2])
+local start_index = tonumber(ARGV[7])
+for offset = 0, num_queues - 1 do
+  local queue_index = (start_index + offset) % num_queues + 1
+  local result = redis.call('ZPOPMIN', KEYS[queue_index * 2])
   if #result > 0 then
     local task_id = result[1]
     local data = redis.call('HGET', ARGV[3] .. task_id, 'data')
@@ -34,7 +39,7 @@ for i = 1, num_queues do
         table.insert(parsed.worker_ids, ARGV[5])
         local updated_data = cjson.encode(parsed)
         local deadline = tonumber(ARGV[1]) + lease_ttl_ms
-        redis.call('ZADD', KEYS[i * 2 - 1], deadline, task_id)
+        redis.call('ZADD', KEYS[queue_index * 2 - 1], deadline, task_id)
         redis.call('HSET', ARGV[3] .. task_id, 'data', updated_data)
         return updated_data
       end

{threadmill-0.7.0 → threadmill-0.7.2}/threadmill/backends/redis.py

@@ -5,6 +5,7 @@ import dataclasses
 import datetime
 import logging
 import queue
+import random
 import time
 import uuid
 from collections.abc import Generator, Sequence
@@ -155,6 +156,14 @@ class RedisTaskBackend(ThreadmillTaskBackend):
         self.lease_ttl = self.options.get("lease_ttl", datetime.timedelta(hours=1))
         self.result_ttl = self.options.get("result_ttl", datetime.timedelta(days=1))
         self.batch_size = self.options.get("batch_size", 100)
+        self.poll_interval = self.options.get(
+            "poll_interval", datetime.timedelta(seconds=0.01)
+        )
+        self.poll_max_interval = self.options.get(
+            "poll_max_interval", datetime.timedelta(seconds=1)
+        )
+        self._miss_count = 0
+        self._rotation_offset = random.randrange(len(self.queues))  # noqa: S311
         self._acquire_script = self.client.register_script(self.ACQUIRE_SCRIPT)
         self._acknowledge_script = self.client.register_script(self.ACKNOWLEDGE_SCRIPT)
 
@@ -268,19 +277,29 @@ class RedisTaskBackend(ThreadmillTaskBackend):
                     str(len(queue_names)),
                     worker,
                     str(int(self.lease_ttl.total_seconds() * 1000)),
+                    str(self._rotation_offset),
                 ],
             ):
+                self._miss_count = 0
+                self._rotation_offset = (self._rotation_offset + 1) % len(queue_names)
                 return self.deserialize_task_result(data)
 
             try:
-                if deadline - time.monotonic() <= 0:
-                    raise TimeoutError(
-                        "No task available within the specified timeout."
-                    )
+                remaining = deadline - time.monotonic()
             except TypeError:
                 raise queue.Empty("No task available.")
-            else:
-                time.sleep(0.01)
+            if remaining <= 0:
+                raise TimeoutError("No task available within the specified timeout.")
+            # Stop doubling once the interval reaches poll_max_interval; larger
+            # exponents would only overflow the float math.
+            cap = int(self.poll_max_interval / self.poll_interval).bit_length()
+            interval_secs = min(
+                self.poll_interval.total_seconds() * 2 ** min(self._miss_count, cap),
+                self.poll_max_interval.total_seconds(),
+                remaining,
+            )
+            self._miss_count += 1
+            time.sleep(interval_secs)
 
     def acknowledge(self, task_result: TaskResult) -> None:
         serialized = self.serialize_task_result(task_result)

{threadmill-0.7.0 → threadmill-0.7.2}/threadmill/executor.py

@@ -96,6 +96,8 @@ class TaskExecutor:
     threads: int = 1
     max_tasks: int = 0
     max_tasks_jitter: int = 0
+    poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
+    poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
     is_publishing: bool = dataclasses.field(default=True, init=False)
     worker_processes: list[WorkerProcess] = dataclasses.field(
         default_factory=list, init=False
@@ -127,6 +129,8 @@ class TaskExecutor:
             backend_alias=self.backend.alias,
             queues=self.queues,
             exit_empty=self.exit_empty,
+            poll_interval=self.poll_interval,
+            poll_max_interval=self.poll_max_interval,
             log_formatter=self.log_formatter,
         )
         worker.start()
@@ -188,6 +192,8 @@ class WorkerProcess(multiprocessing.Process):
         backend_alias: str = "",
         queues: tuple[str, ...] = (),
         exit_empty: bool = False,
+        poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01),
+        poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1),
         log_formatter: logging.Formatter,
     ) -> None:
         """Create process with dedicated thread pool for task execution."""
@@ -198,6 +204,8 @@ class WorkerProcess(multiprocessing.Process):
         self.backend_alias = backend_alias
         self.queues = queues
         self.exit_empty = exit_empty
+        self.poll_interval = poll_interval
+        self.poll_max_interval = poll_max_interval
         self.log_formatter = log_formatter
         self.task_count = 0
         self.lock: threading.Lock | None = None
@@ -211,6 +219,8 @@ class WorkerProcess(multiprocessing.Process):
         self.lock = threading.Lock()
         self.expired = threading.Event()
         backend = task_backends[self.backend_alias]
+        backend.poll_interval = self.poll_interval
+        backend.poll_max_interval = self.poll_max_interval
         consumer_threads = [
             WorkerThread(worker=self, index=index, backend=backend)
             for index in range(self.thread_count)

{threadmill-0.7.0 → threadmill-0.7.2}/threadmill/management/commands/threadmill.py

@@ -1,3 +1,4 @@
+import datetime
 import logging
 import signal
 import sys
@@ -37,7 +38,7 @@ class WorkerCommand(DjangoBaseCommand):
             "--queues",
             nargs="+",
             default=[DEFAULT_TASK_QUEUE_NAME],
-            help="Queue names to listen to and process tasks from.",
+            help="Queue names to listen to and process tasks from (round-robin).",
         )
         parser.add_argument(
             "-w",
@@ -67,6 +68,18 @@ class WorkerCommand(DjangoBaseCommand):
             default=0,
             help="Maximum random jitter to add to the max-tasks value by randint(0, max_tasks_jitter).",
         )
+        parser.add_argument(
+            "--poll-interval",
+            type=float,
+            default=0.01,
+            help="Base wait between idle acquire attempts in seconds.",
+        )
+        parser.add_argument(
+            "--poll-max-interval",
+            type=float,
+            default=1,
+            help="Maximum wait between idle acquire attempts in seconds.",
+        )
         parser.add_argument(
             "--exit-empty",
             action="store_true",
@@ -90,6 +103,8 @@ class WorkerCommand(DjangoBaseCommand):
         threads,
         max_tasks,
         max_tasks_jitter,
+        poll_interval,
+        poll_max_interval,
         exit_empty,
         log_format,
         **options,
@@ -127,6 +142,8 @@ class WorkerCommand(DjangoBaseCommand):
             threads=threads,
             max_tasks=max_tasks,
             max_tasks_jitter=max_tasks_jitter,
+            poll_interval=datetime.timedelta(seconds=poll_interval),
+            poll_max_interval=datetime.timedelta(seconds=poll_max_interval),
             exit_empty=exit_empty,
             queues=queues,
             log_formatter=log_formatter,