threadmill 0.7.1__tar.gz → 0.7.3__tar.gz

{threadmill-0.7.1 → threadmill-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.7.1
+Version: 0.7.3
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>
@@ -18,6 +18,7 @@ Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Communications :: Email
 Classifier: Topic :: Software Development
 Classifier: Topic :: Text Processing :: Markup :: Markdown

{threadmill-0.7.1 → threadmill-0.7.3}/pyproject.toml

@@ -24,6 +24,7 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.14",
+  "Programming Language :: Python :: 3.15",
   "Topic :: Communications :: Email",
   "Topic :: Software Development",
   "Topic :: Text Processing :: Markup :: Markdown",

{threadmill-0.7.1 → threadmill-0.7.3}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.7.1'
-__version_tuple__ = version_tuple = (0, 7, 1)
+__version__ = version = '0.7.3'
+__version_tuple__ = version_tuple = (0, 7, 3)
 
-__commit_id__ = commit_id = 'gc60eae976'
+__commit_id__ = commit_id = 'g0eee0b8b7'

{threadmill-0.7.1 → threadmill-0.7.3}/threadmill/backends/lua/acquire.lua

@@ -1,6 +1,7 @@
 -- Atomically pop the lowest-scored task from any of the given priority queues,
 -- update its JSON data with worker info, and move it directly to the running
--- set. Iterates queues in key order and returns the first available task.
+-- set. Scans the queues round-robin from ARGV[7] and returns the first available
+-- task, so a backlogged queue cannot starve its neighbours.
 --
 -- KEYS[1..N]  -- interleaved running keys and queue keys, one pair per queue:
 --                KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
@@ -11,12 +12,16 @@
 -- ARGV[4]     -- number of queue pairs (N/2)
 -- ARGV[5]     -- worker name
 -- ARGV[6]     -- lease TTL in milliseconds
+-- ARGV[7]     -- start_index; 0-based index of the queue pair to scan first, so
+--                start_index 0 is the pair at KEYS[1] and KEYS[2]
 -- Returns: updated serialized data on success, nil if all queues are empty.
 
 local num_queues = tonumber(ARGV[4])
 local lease_ttl_ms = tonumber(ARGV[6])
-for i = 1, num_queues do
-  local result = redis.call('ZPOPMIN', KEYS[i * 2])
+local start_index = tonumber(ARGV[7])
+for offset = 0, num_queues - 1 do
+  local queue_index = (start_index + offset) % num_queues + 1
+  local result = redis.call('ZPOPMIN', KEYS[queue_index * 2])
   if #result > 0 then
     local task_id = result[1]
     local data = redis.call('HGET', ARGV[3] .. task_id, 'data')
@@ -34,7 +39,7 @@ for i = 1, num_queues do
         table.insert(parsed.worker_ids, ARGV[5])
         local updated_data = cjson.encode(parsed)
         local deadline = tonumber(ARGV[1]) + lease_ttl_ms
-        redis.call('ZADD', KEYS[i * 2 - 1], deadline, task_id)
+        redis.call('ZADD', KEYS[queue_index * 2 - 1], deadline, task_id)
         redis.call('HSET', ARGV[3] .. task_id, 'data', updated_data)
         return updated_data
       end

{threadmill-0.7.1 → threadmill-0.7.3}/threadmill/backends/redis.py

@@ -5,6 +5,7 @@ import dataclasses
 import datetime
 import logging
 import queue
+import random
 import time
 import uuid
 from collections.abc import Generator, Sequence
@@ -162,6 +163,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             "poll_max_interval", datetime.timedelta(seconds=1)
         )
         self._miss_count = 0
+        self._rotation_offset = random.randrange(len(self.queues))  # noqa: S311
         self._acquire_script = self.client.register_script(self.ACQUIRE_SCRIPT)
         self._acknowledge_script = self.client.register_script(self.ACKNOWLEDGE_SCRIPT)
 
@@ -275,9 +277,11 @@ class RedisTaskBackend(ThreadmillTaskBackend):
                     str(len(queue_names)),
                     worker,
                     str(int(self.lease_ttl.total_seconds() * 1000)),
+                    str(self._rotation_offset),
                 ],
             ):
                 self._miss_count = 0
+                self._rotation_offset = (self._rotation_offset + 1) % len(queue_names)
                 return self.deserialize_task_result(data)
 
             try:

{threadmill-0.7.1 → threadmill-0.7.3}/threadmill/executor.py

@@ -8,6 +8,7 @@ import logging
 import multiprocessing
 import random
 import socket
+import sys
 import threading
 import time
 import typing
@@ -81,10 +82,21 @@ class JsonFormatter(logging.Formatter):
 
 
 logger = multiprocessing.get_logger()
-handler = logging.StreamHandler()
+handler = logging.StreamHandler(sys.stdout)
 handler.setFormatter(JsonFormatter())
-logger.addHandler(handler)
-logger.setLevel(logging.INFO)
+
+
+def configure_logging(formatter: logging.Formatter) -> None:
+    """Route every log record of this process through the threadmill handler."""
+    handler.setFormatter(formatter)
+    for existing_logger in logging.root.manager.loggerDict.values():
+        if isinstance(existing_logger, logging.Logger):
+            existing_logger.handlers.clear()
+            existing_logger.propagate = True
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(logging.INFO)
 
 
 @dataclasses.dataclass(kw_only=True, slots=True)
@@ -138,7 +150,7 @@ class TaskExecutor:
 
     def run(self) -> None:
         """Start consuming tasks until shutdown is requested."""
-        handler.setFormatter(self.log_formatter)
+        configure_logging(self.log_formatter)
         self.worker_processes = [
             self.create_worker_process() for _ in range(self.process_count)
         ]
@@ -213,8 +225,8 @@ class WorkerProcess(multiprocessing.Process):
 
     def run(self) -> None:
         """Start consumer execution inside this process."""
-        handler.setFormatter(self.log_formatter)
         django.setup()
+        configure_logging(self.log_formatter)
         logger.info("Starting worker process %s", self.name)
         self.lock = threading.Lock()
         self.expired = threading.Event()

{threadmill-0.7.1 → threadmill-0.7.3}/threadmill/management/commands/threadmill.py

@@ -38,7 +38,7 @@ class WorkerCommand(DjangoBaseCommand):
             "--queues",
             nargs="+",
             default=[DEFAULT_TASK_QUEUE_NAME],
-            help="Queue names to listen to and process tasks from.",
+            help="Queue names to listen to and process tasks from (round-robin).",
         )
         parser.add_argument(
             "-w",
@@ -88,8 +88,8 @@ class WorkerCommand(DjangoBaseCommand):
         parser.add_argument(
             "--log-format",
             help=(
-                "Logging format string for worker log records, e.g."
-                " '%%(levelname)s %%(message)s'. Defaults to JSON."
+                "Logging format string for all log records of the worker process,"
+                " e.g. '%%(levelname)s %%(message)s'. Defaults to JSON."
             ),
         )