threadmill 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadmill/__init__.py +9 -0
- threadmill/_version.py +24 -0
- threadmill/backends.py +34 -0
- threadmill/executor.py +307 -0
- threadmill/management/__init__.py +0 -0
- threadmill/management/commands/__init__.py +0 -0
- threadmill/management/commands/threadmill.py +132 -0
- threadmill-0.1.0.dist-info/METADATA +175 -0
- threadmill-0.1.0.dist-info/RECORD +11 -0
- threadmill-0.1.0.dist-info/WHEEL +4 -0
- threadmill-0.1.0.dist-info/licenses/LICENSE +24 -0
threadmill/__init__.py
ADDED
threadmill/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g7270fd320'
|
threadmill/backends.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
from abc import ABC
|
|
5
|
+
|
|
6
|
+
from django.tasks import TaskResult
|
|
7
|
+
from django.tasks.backends.base import BaseTaskBackend
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AcknowledgeableTaskBackend(BaseTaskBackend, ABC):
|
|
11
|
+
"""Provide an interface for tasks queues to be processed by the executor."""
|
|
12
|
+
|
|
13
|
+
supports_async_task = True
|
|
14
|
+
supports_get_result = True
|
|
15
|
+
|
|
16
|
+
def acquire(
|
|
17
|
+
self, *queue_names: str, timeout: datetime.timedelta | None = None
|
|
18
|
+
) -> TaskResult:
|
|
19
|
+
"""
|
|
20
|
+
Return and lock the next task to be processed without removing it from the queue.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
queue_names: The names of the queues to acquire tasks from.
|
|
24
|
+
timeout: The maximum time to wait for a task. If None, wait indefinitely.
|
|
25
|
+
|
|
26
|
+
Raises:
|
|
27
|
+
TimeoutError: If no task is available within the specified timeout.
|
|
28
|
+
queue.Empty: If no task is available and timeout is None.
|
|
29
|
+
"""
|
|
30
|
+
raise NotImplementedError
|
|
31
|
+
|
|
32
|
+
def acknowledge(self, task_result: TaskResult) -> None:
|
|
33
|
+
"""Remove the task from the queue and publish the result."""
|
|
34
|
+
raise NotImplementedError
|
threadmill/executor.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"""Task worker executor implementation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import dataclasses
|
|
7
|
+
import datetime
|
|
8
|
+
import logging
|
|
9
|
+
import multiprocessing
|
|
10
|
+
import random
|
|
11
|
+
import socket
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
import typing
|
|
15
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
16
|
+
from contextlib import suppress
|
|
17
|
+
from inspect import iscoroutinefunction
|
|
18
|
+
from multiprocessing.queues import JoinableQueue
|
|
19
|
+
from queue import Empty
|
|
20
|
+
from traceback import format_exception
|
|
21
|
+
|
|
22
|
+
from django.tasks import TaskResult
|
|
23
|
+
from django.tasks.base import TaskContext, TaskError, TaskResultStatus
|
|
24
|
+
from django.tasks.signals import task_finished, task_started
|
|
25
|
+
from django.utils import timezone
|
|
26
|
+
from django.utils.json import normalize_json
|
|
27
|
+
|
|
28
|
+
if typing.TYPE_CHECKING:
|
|
29
|
+
from .backends import AcknowledgeableTaskBackend
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
logger = multiprocessing.get_logger()
|
|
33
|
+
formatter = logging.Formatter(
|
|
34
|
+
"%(levelname)s: %(asctime)s - pid=%(process)s - %(message)s"
|
|
35
|
+
)
|
|
36
|
+
handler = logging.StreamHandler()
|
|
37
|
+
handler.setFormatter(formatter)
|
|
38
|
+
logger.addHandler(handler)
|
|
39
|
+
logger.setLevel(logging.INFO)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclasses.dataclass(kw_only=True, slots=True)
|
|
43
|
+
class TaskExecutor:
|
|
44
|
+
"""Consume tasks from shared joinable queues with process and thread pools."""
|
|
45
|
+
|
|
46
|
+
backend: AcknowledgeableTaskBackend
|
|
47
|
+
workers: int | None = None
|
|
48
|
+
threads: int = 1
|
|
49
|
+
max_tasks: int = 0
|
|
50
|
+
max_tasks_jitter: int = 0
|
|
51
|
+
task_timeout: datetime.timedelta = datetime.timedelta(hours=1)
|
|
52
|
+
is_acquiring: bool = dataclasses.field(default=True, init=False)
|
|
53
|
+
is_publishing: bool = dataclasses.field(default=True, init=False)
|
|
54
|
+
worker_processes: list[WorkerProcess] = dataclasses.field(
|
|
55
|
+
default_factory=list, init=False
|
|
56
|
+
)
|
|
57
|
+
process_count: int = dataclasses.field(init=False)
|
|
58
|
+
thread_count: int = dataclasses.field(init=False)
|
|
59
|
+
queues: tuple[str]
|
|
60
|
+
shared_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field(
|
|
61
|
+
init=False
|
|
62
|
+
)
|
|
63
|
+
processed_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field(
|
|
64
|
+
init=False
|
|
65
|
+
)
|
|
66
|
+
exit_empty: bool = False
|
|
67
|
+
|
|
68
|
+
def __post_init__(self) -> None:
|
|
69
|
+
"""Initialize derived orchestration fields and queues."""
|
|
70
|
+
self.process_count = self.workers or max(multiprocessing.cpu_count() - 1, 1)
|
|
71
|
+
self.thread_count = max(self.threads, 1)
|
|
72
|
+
self.shared_task_queue = multiprocessing.JoinableQueue(
|
|
73
|
+
maxsize=self.process_count * self.thread_count,
|
|
74
|
+
)
|
|
75
|
+
self.processed_task_queue = multiprocessing.JoinableQueue()
|
|
76
|
+
|
|
77
|
+
def get_maximum_tasks_per_child(self) -> int | None:
|
|
78
|
+
"""Return worker recycling limit based on config and thread count."""
|
|
79
|
+
if self.max_tasks:
|
|
80
|
+
return (
|
|
81
|
+
self.max_tasks + random.randint(0, self.max_tasks_jitter) # noqa: S311
|
|
82
|
+
) // self.thread_count
|
|
83
|
+
|
|
84
|
+
def create_worker_process(self) -> WorkerProcess:
|
|
85
|
+
"""Create and start a new worker process."""
|
|
86
|
+
worker = WorkerProcess(
|
|
87
|
+
self.shared_task_queue,
|
|
88
|
+
self.processed_task_queue,
|
|
89
|
+
self.thread_count,
|
|
90
|
+
self.task_timeout,
|
|
91
|
+
self.get_maximum_tasks_per_child(),
|
|
92
|
+
)
|
|
93
|
+
worker.start()
|
|
94
|
+
return worker
|
|
95
|
+
|
|
96
|
+
def run(self) -> None:
|
|
97
|
+
"""Start consuming tasks until shutdown is requested."""
|
|
98
|
+
self.worker_processes = [
|
|
99
|
+
self.create_worker_process() for _ in range(self.process_count)
|
|
100
|
+
]
|
|
101
|
+
threads = [
|
|
102
|
+
threading.Thread(target=self.acknowledge_tasks, daemon=True),
|
|
103
|
+
threading.Thread(target=self.maintain_worker_pool, daemon=True),
|
|
104
|
+
threading.Thread(target=self.acquire_tasks, daemon=True),
|
|
105
|
+
]
|
|
106
|
+
for thread in threads:
|
|
107
|
+
thread.start()
|
|
108
|
+
for thread in threads:
|
|
109
|
+
thread.join()
|
|
110
|
+
|
|
111
|
+
def acquire_tasks(self) -> None:
|
|
112
|
+
"""Buffer tasks in shared task queue."""
|
|
113
|
+
while self.is_acquiring:
|
|
114
|
+
try:
|
|
115
|
+
work = self.backend.acquire(*self.queues)
|
|
116
|
+
except Empty:
|
|
117
|
+
if self.exit_empty:
|
|
118
|
+
logger.info("No more tasks to solve. Shutting down.")
|
|
119
|
+
self.shutdown()
|
|
120
|
+
return
|
|
121
|
+
time.sleep(0.01)
|
|
122
|
+
else:
|
|
123
|
+
self.shared_task_queue.put(work)
|
|
124
|
+
|
|
125
|
+
def acknowledge_tasks(self) -> None:
|
|
126
|
+
"""Acknowledge processed tasks and publish updated results in main process."""
|
|
127
|
+
while self.is_publishing:
|
|
128
|
+
try:
|
|
129
|
+
task = self.processed_task_queue.get_nowait()
|
|
130
|
+
except Empty:
|
|
131
|
+
time.sleep(0.01)
|
|
132
|
+
else:
|
|
133
|
+
self.backend.acknowledge(task)
|
|
134
|
+
self.processed_task_queue.task_done()
|
|
135
|
+
|
|
136
|
+
def shutdown(self) -> None:
|
|
137
|
+
"""Stop queue consumption and terminate all worker processes."""
|
|
138
|
+
logger.info("Shutting down task executor")
|
|
139
|
+
self.is_acquiring = False
|
|
140
|
+
with suppress(ValueError):
|
|
141
|
+
self.shared_task_queue.join()
|
|
142
|
+
with suppress(ValueError):
|
|
143
|
+
self.processed_task_queue.join()
|
|
144
|
+
with ThreadPoolExecutor(max_workers=self.process_count) as executor:
|
|
145
|
+
executor.map(lambda worker: worker.shutdown(), self.worker_processes)
|
|
146
|
+
self.is_publishing = False
|
|
147
|
+
|
|
148
|
+
def maintain_worker_pool(self) -> None:
|
|
149
|
+
"""Restart worker processes that have exited."""
|
|
150
|
+
while self.is_publishing:
|
|
151
|
+
for index, worker in enumerate(self.worker_processes):
|
|
152
|
+
if worker.is_alive():
|
|
153
|
+
continue
|
|
154
|
+
worker.join(timeout=0)
|
|
155
|
+
self.worker_processes[index] = self.create_worker_process()
|
|
156
|
+
time.sleep(1)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class WorkerProcess(multiprocessing.Process):
|
|
160
|
+
"""Single worker process running thread_count consumer threads."""
|
|
161
|
+
|
|
162
|
+
def __init__(
|
|
163
|
+
self,
|
|
164
|
+
task_queue: JoinableQueue[TaskResult],
|
|
165
|
+
processed_task_queue: JoinableQueue[TaskResult],
|
|
166
|
+
thread_count: int,
|
|
167
|
+
task_timeout: datetime.timedelta,
|
|
168
|
+
max_tasks: int | None = None,
|
|
169
|
+
) -> None:
|
|
170
|
+
"""Create process with dedicated thread pool for task execution."""
|
|
171
|
+
self.shutdown_requested = multiprocessing.Event()
|
|
172
|
+
super().__init__(daemon=True)
|
|
173
|
+
self.task_queue = task_queue
|
|
174
|
+
self.processed_task_queue = processed_task_queue
|
|
175
|
+
self.thread_count = thread_count
|
|
176
|
+
self.task_timeout = task_timeout
|
|
177
|
+
self.max_tasks = max_tasks
|
|
178
|
+
self.task_count = 0
|
|
179
|
+
self.lock: threading.Lock | None = None
|
|
180
|
+
self.expired: threading.Event | None = None
|
|
181
|
+
|
|
182
|
+
def run(self) -> None:
|
|
183
|
+
"""Start consumer execution inside this process."""
|
|
184
|
+
logger.info("Starting worker process %s", self.name)
|
|
185
|
+
self.lock = threading.Lock()
|
|
186
|
+
self.expired = threading.Event()
|
|
187
|
+
consumer_threads = [
|
|
188
|
+
WorkerThread(worker=self, index=index) for index in range(self.thread_count)
|
|
189
|
+
]
|
|
190
|
+
for consumer_thread in consumer_threads:
|
|
191
|
+
consumer_thread.start()
|
|
192
|
+
for consumer_thread in consumer_threads:
|
|
193
|
+
consumer_thread.join(self.task_timeout.total_seconds())
|
|
194
|
+
|
|
195
|
+
def record_task(self) -> None:
|
|
196
|
+
"""Record one processed task and stop when max_tasks is reached."""
|
|
197
|
+
if self.max_tasks is None:
|
|
198
|
+
return
|
|
199
|
+
if self.lock is None or self.expired is None:
|
|
200
|
+
return
|
|
201
|
+
with self.lock:
|
|
202
|
+
self.task_count += 1
|
|
203
|
+
if self.task_count >= self.max_tasks:
|
|
204
|
+
self.expired.set()
|
|
205
|
+
|
|
206
|
+
def shutdown(self) -> None:
|
|
207
|
+
"""Request graceful worker stop and wait for process exit."""
|
|
208
|
+
logger.info("Stopping worker process %s", self.name)
|
|
209
|
+
self.shutdown_requested.set()
|
|
210
|
+
self.join()
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class WorkerThread(threading.Thread):
|
|
214
|
+
"""Single worker thread consuming tasks from the process queue."""
|
|
215
|
+
|
|
216
|
+
def __init__(
|
|
217
|
+
self,
|
|
218
|
+
*,
|
|
219
|
+
worker: WorkerProcess,
|
|
220
|
+
index: int,
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Create worker thread bound to process worker state."""
|
|
223
|
+
super().__init__(name=f"{socket.gethostname()}:{worker.pid}-{index}")
|
|
224
|
+
self.worker = worker
|
|
225
|
+
|
|
226
|
+
def run(self) -> None:
|
|
227
|
+
"""Start consuming tasks for this thread."""
|
|
228
|
+
while self.worker.expired is None or not self.worker.expired.is_set():
|
|
229
|
+
try:
|
|
230
|
+
task_result = self.worker.task_queue.get(timeout=1.0)
|
|
231
|
+
except Empty:
|
|
232
|
+
if self.worker.shutdown_requested.is_set():
|
|
233
|
+
return
|
|
234
|
+
continue
|
|
235
|
+
try:
|
|
236
|
+
self.worker.processed_task_queue.put(
|
|
237
|
+
self.execute_task_result(
|
|
238
|
+
task_result,
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
finally:
|
|
242
|
+
self.worker.task_queue.task_done()
|
|
243
|
+
self.worker.record_task()
|
|
244
|
+
|
|
245
|
+
def execute_task_result(self, task_result: TaskResult) -> TaskResult:
|
|
246
|
+
"""Execute task from task result and update result lifecycle state."""
|
|
247
|
+
logger.info("Executing task %r", task_result.id)
|
|
248
|
+
started_at = timezone.now()
|
|
249
|
+
task_result = dataclasses.replace(
|
|
250
|
+
task_result,
|
|
251
|
+
status=TaskResultStatus.RUNNING,
|
|
252
|
+
started_at=started_at,
|
|
253
|
+
last_attempted_at=started_at,
|
|
254
|
+
worker_ids=[*task_result.worker_ids, self.name],
|
|
255
|
+
)
|
|
256
|
+
task_started.send(TaskExecutor, task_result=task_result)
|
|
257
|
+
|
|
258
|
+
try:
|
|
259
|
+
return_value = WorkerThread.call_task(task_result)
|
|
260
|
+
except Exception as exception:
|
|
261
|
+
task_result = dataclasses.replace(
|
|
262
|
+
task_result,
|
|
263
|
+
status=TaskResultStatus.FAILED,
|
|
264
|
+
errors=[*task_result.errors, WorkerThread.create_task_error(exception)],
|
|
265
|
+
)
|
|
266
|
+
logger.exception("Task failed %r", task_result.id)
|
|
267
|
+
else:
|
|
268
|
+
task_result = dataclasses.replace(
|
|
269
|
+
task_result,
|
|
270
|
+
status=TaskResultStatus.SUCCESSFUL,
|
|
271
|
+
)
|
|
272
|
+
object.__setattr__(
|
|
273
|
+
task_result, "_return_value", normalize_json(return_value)
|
|
274
|
+
)
|
|
275
|
+
logger.info("Task successful %r", task_result.id)
|
|
276
|
+
finally:
|
|
277
|
+
task_result = dataclasses.replace(
|
|
278
|
+
task_result,
|
|
279
|
+
finished_at=timezone.now(),
|
|
280
|
+
)
|
|
281
|
+
task_finished.send(TaskExecutor, task_result=task_result)
|
|
282
|
+
|
|
283
|
+
return task_result
|
|
284
|
+
|
|
285
|
+
@staticmethod
|
|
286
|
+
def call_task(task_result: TaskResult) -> typing.Any:
|
|
287
|
+
"""Call a task with context when required."""
|
|
288
|
+
task = task_result.task
|
|
289
|
+
if task.takes_context:
|
|
290
|
+
args = [TaskContext(task_result=task_result), *task_result.args]
|
|
291
|
+
else:
|
|
292
|
+
args = task_result.args
|
|
293
|
+
if iscoroutinefunction(task.func):
|
|
294
|
+
return asyncio.run(task.func(*args, **task_result.kwargs))
|
|
295
|
+
return task.func(
|
|
296
|
+
*args,
|
|
297
|
+
**task_result.kwargs,
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
@staticmethod
|
|
301
|
+
def create_task_error(exception: BaseException) -> TaskError:
|
|
302
|
+
"""Build a task error payload for failed execution."""
|
|
303
|
+
exception_type = type(exception)
|
|
304
|
+
return TaskError(
|
|
305
|
+
exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}",
|
|
306
|
+
traceback="".join(format_exception(exception)),
|
|
307
|
+
)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import signal
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from django.core.management import BaseCommand, CommandError
|
|
6
|
+
from django.tasks import (
|
|
7
|
+
DEFAULT_TASK_BACKEND_ALIAS,
|
|
8
|
+
DEFAULT_TASK_QUEUE_NAME,
|
|
9
|
+
InvalidTaskBackend,
|
|
10
|
+
task_backends,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
from ...executor import TaskExecutor
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def kill_softly(signum, frame):
|
|
17
|
+
"""Raise a KeyboardInterrupt to stop the worker gracefully."""
|
|
18
|
+
signame = signal.Signals(signum).name
|
|
19
|
+
raise KeyboardInterrupt(f"Received {signame} ({signum}), shutting down…")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Command(BaseCommand):
|
|
23
|
+
"""Run task workers to process enqueued tasks from the specified backends and queues."""
|
|
24
|
+
|
|
25
|
+
help = __doc__
|
|
26
|
+
|
|
27
|
+
def add_arguments(self, parser):
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"-b",
|
|
30
|
+
"--backend",
|
|
31
|
+
default=DEFAULT_TASK_BACKEND_ALIAS,
|
|
32
|
+
help="Alias of the tasks backend to use.",
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"-q",
|
|
36
|
+
"--queues",
|
|
37
|
+
nargs="+",
|
|
38
|
+
default=[DEFAULT_TASK_QUEUE_NAME],
|
|
39
|
+
help="Queue names to listen to and process tasks from.",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"-w",
|
|
43
|
+
"--workers",
|
|
44
|
+
type=int,
|
|
45
|
+
help="Number of worker processes to use. Defaults to the number of CPU cores minus one.",
|
|
46
|
+
)
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"-t",
|
|
49
|
+
"--threads",
|
|
50
|
+
type=int,
|
|
51
|
+
default=1,
|
|
52
|
+
help="Number of threads to use. Defaults to 1. ",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--max-tasks",
|
|
56
|
+
type=int,
|
|
57
|
+
default=0,
|
|
58
|
+
help=(
|
|
59
|
+
"Number of the maximum number of tasks to run until a worker is recycled."
|
|
60
|
+
" Defaults to 0, which means no limit."
|
|
61
|
+
),
|
|
62
|
+
)
|
|
63
|
+
parser.add_argument(
|
|
64
|
+
"--max-tasks-jitter",
|
|
65
|
+
type=int,
|
|
66
|
+
default=0,
|
|
67
|
+
help="Maximum random jitter to add to the max-tasks value by randint(0, max_tasks_jitter).",
|
|
68
|
+
)
|
|
69
|
+
parser.add_argument(
|
|
70
|
+
"--task-timeout",
|
|
71
|
+
type=float,
|
|
72
|
+
default=3600.0,
|
|
73
|
+
help="Kill hung tasks after timeout seconds. Defaults to one hour.",
|
|
74
|
+
)
|
|
75
|
+
parser.add_argument(
|
|
76
|
+
"--task-backlog-size",
|
|
77
|
+
type=int,
|
|
78
|
+
default=1,
|
|
79
|
+
help="The number of tasks to prefetch from the message queue while all workers are busy. Defaults to 1.",
|
|
80
|
+
)
|
|
81
|
+
parser.add_argument(
|
|
82
|
+
"--exit-empty",
|
|
83
|
+
action="store_true",
|
|
84
|
+
help="Drain the task queue and exit with 0.",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def handle(
|
|
88
|
+
self,
|
|
89
|
+
*,
|
|
90
|
+
verbosity,
|
|
91
|
+
backend,
|
|
92
|
+
queues,
|
|
93
|
+
workers,
|
|
94
|
+
threads,
|
|
95
|
+
max_tasks,
|
|
96
|
+
max_tasks_jitter,
|
|
97
|
+
task_timeout,
|
|
98
|
+
exit_empty,
|
|
99
|
+
**options,
|
|
100
|
+
):
|
|
101
|
+
match sys.platform:
|
|
102
|
+
case "win32":
|
|
103
|
+
signal.signal(signal.SIGBREAK, kill_softly)
|
|
104
|
+
case _:
|
|
105
|
+
signal.signal(signal.SIGHUP, kill_softly)
|
|
106
|
+
signal.signal(signal.SIGTERM, kill_softly)
|
|
107
|
+
signal.signal(signal.SIGINT, kill_softly)
|
|
108
|
+
self.stdout.write(self.style.SUCCESS("Starting workers…"))
|
|
109
|
+
try:
|
|
110
|
+
backend = task_backends[backend]
|
|
111
|
+
except InvalidTaskBackend:
|
|
112
|
+
raise CommandError(f"Invalid backend: {backend!r}")
|
|
113
|
+
if _non_queues := set(queues) - set(backend.queues):
|
|
114
|
+
raise CommandError(
|
|
115
|
+
f"Backend does not support all specified queues: {_non_queues!r}"
|
|
116
|
+
)
|
|
117
|
+
exe = TaskExecutor(
|
|
118
|
+
backend=backend,
|
|
119
|
+
workers=workers,
|
|
120
|
+
threads=threads,
|
|
121
|
+
max_tasks=max_tasks,
|
|
122
|
+
max_tasks_jitter=max_tasks_jitter,
|
|
123
|
+
task_timeout=datetime.timedelta(seconds=task_timeout),
|
|
124
|
+
exit_empty=exit_empty,
|
|
125
|
+
queues=queues,
|
|
126
|
+
)
|
|
127
|
+
try:
|
|
128
|
+
exe.run()
|
|
129
|
+
except KeyboardInterrupt as e:
|
|
130
|
+
self.stdout.write(self.style.WARNING(str(e)))
|
|
131
|
+
self.stdout.write(self.style.NOTICE("Shutting down workers…"))
|
|
132
|
+
exe.shutdown()
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: threadmill
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A queue agnostic worker for Django's task framework.
|
|
5
|
+
Keywords: Django,tasks,worker
|
|
6
|
+
Author-email: Johannes Maron <johannes@maron.family>
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Classifier: Environment :: Web Environment
|
|
12
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
16
|
+
Classifier: Operating System :: POSIX
|
|
17
|
+
Classifier: Topic :: Communications :: Email
|
|
18
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
19
|
+
Classifier: Topic :: Software Development
|
|
20
|
+
Classifier: Programming Language :: Python
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
26
|
+
Classifier: Framework :: Django
|
|
27
|
+
Classifier: Framework :: Django :: 6.1
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: django>=6.1a1
|
|
30
|
+
Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
|
|
31
|
+
Project-URL: Documentation, https://github.com/codingjoe/threadmill/
|
|
32
|
+
Project-URL: Funding, https://github.com/sponsors/codingjoe
|
|
33
|
+
Project-URL: Homepage, https://github.com/codingjoe/threadmill
|
|
34
|
+
Project-URL: Issues, https://github.com/codingjoe/threadmill/issues
|
|
35
|
+
Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/latest
|
|
36
|
+
Project-URL: Source, https://github.com/codingjoe/threadmill
|
|
37
|
+
|
|
38
|
+
# Threadmill
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<picture>
|
|
42
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-dark.svg">
|
|
43
|
+
<source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
|
|
44
|
+
<img alt="Django Grinder: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
|
|
45
|
+
</picture>
|
|
46
|
+
<br>
|
|
47
|
+
<a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
|
|
48
|
+
<a href="https://github.com/codingjoe/threadmill/issues/new/choose">Issues</a> |
|
|
49
|
+
<a href="https://github.com/codingjoe/threadmill/releases">Changelog</a> |
|
|
50
|
+
<a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
|
|
51
|
+
</p>
|
|
52
|
+
|
|
53
|
+
**A queue agnostic worker for Django's task framework.**
|
|
54
|
+
|
|
55
|
+
## Design Principles
|
|
56
|
+
|
|
57
|
+
- **Durability** – We recover from any failures, even poorly written tasks.
|
|
58
|
+
- **Consistency** – We never lose data, even if someone unplugs the power or network.
|
|
59
|
+
- **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks.
|
|
60
|
+
|
|
61
|
+
> [!WARNING]
|
|
62
|
+
> Threadmill requires a development version of Django and is in a preview stage.
|
|
63
|
+
|
|
64
|
+
[](https://pypi.python.org/pypi/threadmill/)
|
|
65
|
+
[](https://codecov.io/gh/codingjoe/threadmill)
|
|
66
|
+
[](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
|
|
67
|
+
|
|
68
|
+
## Setup
|
|
69
|
+
|
|
70
|
+
You need to have [Django's Task framework][django-tasks] setup properly.
|
|
71
|
+
|
|
72
|
+
```console
|
|
73
|
+
uv add threadmill
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Add `threadmill` to your `INSTALLED_APPS` in `settings.py`:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
# settings.py
|
|
80
|
+
INSTALLED_APPS = [
|
|
81
|
+
"threadmill",
|
|
82
|
+
# ...
|
|
83
|
+
]
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Finally, you launch the worker pool:
|
|
87
|
+
|
|
88
|
+
```console
|
|
89
|
+
uv run manage.py threadmill
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Usage
|
|
93
|
+
|
|
94
|
+
The workers are inspired by Gunicorn, and the CLI is very similar.
|
|
95
|
+
|
|
96
|
+
### Utilization
|
|
97
|
+
|
|
98
|
+
Depending on your workload, you can tweak the number of processes and threads.
|
|
99
|
+
Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
|
|
100
|
+
|
|
101
|
+
```console
|
|
102
|
+
uv run manage.py threadmill --processes 4 --threads 2
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Health
|
|
106
|
+
|
|
107
|
+
If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
|
|
108
|
+
|
|
109
|
+
```console
|
|
110
|
+
uv run manage.py threadmill --max-tasks 1000 --max-tasks-jitter 100
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
|
|
114
|
+
|
|
115
|
+
Should a worker crash or be killed, the pool will automatically restart it.
|
|
116
|
+
|
|
117
|
+
### Shutdown
|
|
118
|
+
|
|
119
|
+
A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
|
|
120
|
+
All workers will finish the tasks they acquired and publish them.
|
|
121
|
+
|
|
122
|
+
You can use `--exit-empty` to exit immediately after all tasks have been processed,
|
|
123
|
+
which might be useful for draining a one-off queue.
|
|
124
|
+
|
|
125
|
+
### Task Backlog
|
|
126
|
+
|
|
127
|
+
You can prefetch tasks from a queue to avoid IO latency bottlenecks.
|
|
128
|
+
However, this will increase the memory usage of the worker pool.
|
|
129
|
+
|
|
130
|
+
```console
|
|
131
|
+
uv run manage.py threadmill --prefetch 100
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Task Timeouts
|
|
135
|
+
|
|
136
|
+
> [!WARNING]
|
|
137
|
+
> Work in progress, this feature is not yet stable.
|
|
138
|
+
|
|
139
|
+
Task timeouts are important to ensure the long-term health of your pool.
|
|
140
|
+
However, they need to be aligned with your queueing system's timeout settings.
|
|
141
|
+
The message queue needs to requeue a task that hasn't been acknowledged within the timeout.
|
|
142
|
+
|
|
143
|
+
## Integration
|
|
144
|
+
|
|
145
|
+
> [!NOTE]
|
|
146
|
+
> This section is for people who want to integrate Threadmill into their queueing system.
|
|
147
|
+
|
|
148
|
+
Threadmill is designed to be durable and requires a queueing system to support late acknowledgement.
|
|
149
|
+
|
|
150
|
+
To use Threadmill, your backend will need to inherit from `threadmill.backends.AcknowledgeableTaskBackend` and implement the following methods:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
class AcknowledgeableTaskBackend(BaseTaskBackend, ABC):
|
|
154
|
+
"""Provide an interface for tasks queues to be processed by the executor."""
|
|
155
|
+
|
|
156
|
+
def acquire(
|
|
157
|
+
self, *queue_names: str, timeout: datetime.timedelta | None = None
|
|
158
|
+
) -> TaskResult:
|
|
159
|
+
"""
|
|
160
|
+
Return and lock the next task to be processed without removing it from the queue.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
queue_names: The names of the queues to acquire tasks from.
|
|
164
|
+
timeout: The maximum time to wait for a task. If None, wait indefinitely.
|
|
165
|
+
|
|
166
|
+
Raises:
|
|
167
|
+
TimeoutError: If no task is available within the specified timeout.
|
|
168
|
+
"""
|
|
169
|
+
raise NotImplementedError
|
|
170
|
+
|
|
171
|
+
def acknowledge(self, task_result: TaskResult) -> None:
|
|
172
|
+
"""Remove the task from the queue and publish the result."""
|
|
173
|
+
raise NotImplementedError
|
|
174
|
+
```
|
|
175
|
+
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
threadmill/__init__.py,sha256=d74cITlnqi-fy8kpkAxnuko9LL3HNcJMdCLWF-1wo1w,187
|
|
2
|
+
threadmill/_version.py,sha256=WfyBXn1Ch4X0PrFrN6x2PrPqz8ypEAzFgY_XX00HkLI,528
|
|
3
|
+
threadmill/backends.py,sha256=YVEpPmweKxtfpLqXJXbHx9tOpqvTyetFeJMko4exrIc,1132
|
|
4
|
+
threadmill/executor.py,sha256=iaEs3VCxeRc7JAqTspCZq-n367m8LmegWFOM0c0hP_0,11243
|
|
5
|
+
threadmill/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
threadmill/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
threadmill/management/commands/threadmill.py,sha256=XiO4V0RF00tq1rHP__MdtYDcqDPAF-DbeELOG8yTqro,4109
|
|
8
|
+
threadmill-0.1.0.dist-info/licenses/LICENSE,sha256=bUZFuK9dEiNN8YcEyAgR8ULo17q2FHLGr-tcqGTIr8A,1303
|
|
9
|
+
threadmill-0.1.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
|
10
|
+
threadmill-0.1.0.dist-info/METADATA,sha256=XPuuFGsrefIXdn8Vgu0HwDGTm90aQ29k99KYzwQv4h0,6483
|
|
11
|
+
threadmill-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Johannes Maron
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|