threadmill 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
threadmill/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """A queue agnostic worker for Django's task framework."""
2
+
3
+ from . import _version
4
+
5
+ __version__ = _version.version
6
+ VERSION = _version.version_tuple
7
+
8
+
9
+ __all__ = ["VERSION", "__version__"]
threadmill/_version.py ADDED
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.1.0'
22
+ __version_tuple__ = version_tuple = (0, 1, 0)
23
+
24
+ __commit_id__ = commit_id = 'g7270fd320'
threadmill/backends.py ADDED
@@ -0,0 +1,34 @@
1
+ from __future__ import annotations
2
+
3
+ import datetime
4
+ from abc import ABC
5
+
6
+ from django.tasks import TaskResult
7
+ from django.tasks.backends.base import BaseTaskBackend
8
+
9
+
10
+ class AcknowledgeableTaskBackend(BaseTaskBackend, ABC):
11
+ """Provide an interface for tasks queues to be processed by the executor."""
12
+
13
+ supports_async_task = True
14
+ supports_get_result = True
15
+
16
+ def acquire(
17
+ self, *queue_names: str, timeout: datetime.timedelta | None = None
18
+ ) -> TaskResult:
19
+ """
20
+ Return and lock the next task to be processed without removing it from the queue.
21
+
22
+ Args:
23
+ queue_names: The names of the queues to acquire tasks from.
24
+ timeout: The maximum time to wait for a task. If None, wait indefinitely.
25
+
26
+ Raises:
27
+ TimeoutError: If no task is available within the specified timeout.
28
+ queue.Empty: If no task is available and timeout is None.
29
+ """
30
+ raise NotImplementedError
31
+
32
+ def acknowledge(self, task_result: TaskResult) -> None:
33
+ """Remove the task from the queue and publish the result."""
34
+ raise NotImplementedError
threadmill/executor.py ADDED
@@ -0,0 +1,307 @@
1
+ """Task worker executor implementation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import dataclasses
7
+ import datetime
8
+ import logging
9
+ import multiprocessing
10
+ import random
11
+ import socket
12
+ import threading
13
+ import time
14
+ import typing
15
+ from concurrent.futures import ThreadPoolExecutor
16
+ from contextlib import suppress
17
+ from inspect import iscoroutinefunction
18
+ from multiprocessing.queues import JoinableQueue
19
+ from queue import Empty
20
+ from traceback import format_exception
21
+
22
+ from django.tasks import TaskResult
23
+ from django.tasks.base import TaskContext, TaskError, TaskResultStatus
24
+ from django.tasks.signals import task_finished, task_started
25
+ from django.utils import timezone
26
+ from django.utils.json import normalize_json
27
+
28
+ if typing.TYPE_CHECKING:
29
+ from .backends import AcknowledgeableTaskBackend
30
+
31
+
32
+ logger = multiprocessing.get_logger()
33
+ formatter = logging.Formatter(
34
+ "%(levelname)s: %(asctime)s - pid=%(process)s - %(message)s"
35
+ )
36
+ handler = logging.StreamHandler()
37
+ handler.setFormatter(formatter)
38
+ logger.addHandler(handler)
39
+ logger.setLevel(logging.INFO)
40
+
41
+
42
+ @dataclasses.dataclass(kw_only=True, slots=True)
43
+ class TaskExecutor:
44
+ """Consume tasks from shared joinable queues with process and thread pools."""
45
+
46
+ backend: AcknowledgeableTaskBackend
47
+ workers: int | None = None
48
+ threads: int = 1
49
+ max_tasks: int = 0
50
+ max_tasks_jitter: int = 0
51
+ task_timeout: datetime.timedelta = datetime.timedelta(hours=1)
52
+ is_acquiring: bool = dataclasses.field(default=True, init=False)
53
+ is_publishing: bool = dataclasses.field(default=True, init=False)
54
+ worker_processes: list[WorkerProcess] = dataclasses.field(
55
+ default_factory=list, init=False
56
+ )
57
+ process_count: int = dataclasses.field(init=False)
58
+ thread_count: int = dataclasses.field(init=False)
59
+ queues: tuple[str]
60
+ shared_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field(
61
+ init=False
62
+ )
63
+ processed_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field(
64
+ init=False
65
+ )
66
+ exit_empty: bool = False
67
+
68
+ def __post_init__(self) -> None:
69
+ """Initialize derived orchestration fields and queues."""
70
+ self.process_count = self.workers or max(multiprocessing.cpu_count() - 1, 1)
71
+ self.thread_count = max(self.threads, 1)
72
+ self.shared_task_queue = multiprocessing.JoinableQueue(
73
+ maxsize=self.process_count * self.thread_count,
74
+ )
75
+ self.processed_task_queue = multiprocessing.JoinableQueue()
76
+
77
+ def get_maximum_tasks_per_child(self) -> int | None:
78
+ """Return worker recycling limit based on config and thread count."""
79
+ if self.max_tasks:
80
+ return (
81
+ self.max_tasks + random.randint(0, self.max_tasks_jitter) # noqa: S311
82
+ ) // self.thread_count
83
+
84
+ def create_worker_process(self) -> WorkerProcess:
85
+ """Create and start a new worker process."""
86
+ worker = WorkerProcess(
87
+ self.shared_task_queue,
88
+ self.processed_task_queue,
89
+ self.thread_count,
90
+ self.task_timeout,
91
+ self.get_maximum_tasks_per_child(),
92
+ )
93
+ worker.start()
94
+ return worker
95
+
96
+ def run(self) -> None:
97
+ """Start consuming tasks until shutdown is requested."""
98
+ self.worker_processes = [
99
+ self.create_worker_process() for _ in range(self.process_count)
100
+ ]
101
+ threads = [
102
+ threading.Thread(target=self.acknowledge_tasks, daemon=True),
103
+ threading.Thread(target=self.maintain_worker_pool, daemon=True),
104
+ threading.Thread(target=self.acquire_tasks, daemon=True),
105
+ ]
106
+ for thread in threads:
107
+ thread.start()
108
+ for thread in threads:
109
+ thread.join()
110
+
111
+ def acquire_tasks(self) -> None:
112
+ """Buffer tasks in shared task queue."""
113
+ while self.is_acquiring:
114
+ try:
115
+ work = self.backend.acquire(*self.queues)
116
+ except Empty:
117
+ if self.exit_empty:
118
+ logger.info("No more tasks to solve. Shutting down.")
119
+ self.shutdown()
120
+ return
121
+ time.sleep(0.01)
122
+ else:
123
+ self.shared_task_queue.put(work)
124
+
125
+ def acknowledge_tasks(self) -> None:
126
+ """Acknowledge processed tasks and publish updated results in main process."""
127
+ while self.is_publishing:
128
+ try:
129
+ task = self.processed_task_queue.get_nowait()
130
+ except Empty:
131
+ time.sleep(0.01)
132
+ else:
133
+ self.backend.acknowledge(task)
134
+ self.processed_task_queue.task_done()
135
+
136
+ def shutdown(self) -> None:
137
+ """Stop queue consumption and terminate all worker processes."""
138
+ logger.info("Shutting down task executor")
139
+ self.is_acquiring = False
140
+ with suppress(ValueError):
141
+ self.shared_task_queue.join()
142
+ with suppress(ValueError):
143
+ self.processed_task_queue.join()
144
+ with ThreadPoolExecutor(max_workers=self.process_count) as executor:
145
+ executor.map(lambda worker: worker.shutdown(), self.worker_processes)
146
+ self.is_publishing = False
147
+
148
+ def maintain_worker_pool(self) -> None:
149
+ """Restart worker processes that have exited."""
150
+ while self.is_publishing:
151
+ for index, worker in enumerate(self.worker_processes):
152
+ if worker.is_alive():
153
+ continue
154
+ worker.join(timeout=0)
155
+ self.worker_processes[index] = self.create_worker_process()
156
+ time.sleep(1)
157
+
158
+
159
+ class WorkerProcess(multiprocessing.Process):
160
+ """Single worker process running thread_count consumer threads."""
161
+
162
+ def __init__(
163
+ self,
164
+ task_queue: JoinableQueue[TaskResult],
165
+ processed_task_queue: JoinableQueue[TaskResult],
166
+ thread_count: int,
167
+ task_timeout: datetime.timedelta,
168
+ max_tasks: int | None = None,
169
+ ) -> None:
170
+ """Create process with dedicated thread pool for task execution."""
171
+ self.shutdown_requested = multiprocessing.Event()
172
+ super().__init__(daemon=True)
173
+ self.task_queue = task_queue
174
+ self.processed_task_queue = processed_task_queue
175
+ self.thread_count = thread_count
176
+ self.task_timeout = task_timeout
177
+ self.max_tasks = max_tasks
178
+ self.task_count = 0
179
+ self.lock: threading.Lock | None = None
180
+ self.expired: threading.Event | None = None
181
+
182
+ def run(self) -> None:
183
+ """Start consumer execution inside this process."""
184
+ logger.info("Starting worker process %s", self.name)
185
+ self.lock = threading.Lock()
186
+ self.expired = threading.Event()
187
+ consumer_threads = [
188
+ WorkerThread(worker=self, index=index) for index in range(self.thread_count)
189
+ ]
190
+ for consumer_thread in consumer_threads:
191
+ consumer_thread.start()
192
+ for consumer_thread in consumer_threads:
193
+ consumer_thread.join(self.task_timeout.total_seconds())
194
+
195
+ def record_task(self) -> None:
196
+ """Record one processed task and stop when max_tasks is reached."""
197
+ if self.max_tasks is None:
198
+ return
199
+ if self.lock is None or self.expired is None:
200
+ return
201
+ with self.lock:
202
+ self.task_count += 1
203
+ if self.task_count >= self.max_tasks:
204
+ self.expired.set()
205
+
206
+ def shutdown(self) -> None:
207
+ """Request graceful worker stop and wait for process exit."""
208
+ logger.info("Stopping worker process %s", self.name)
209
+ self.shutdown_requested.set()
210
+ self.join()
211
+
212
+
213
+ class WorkerThread(threading.Thread):
214
+ """Single worker thread consuming tasks from the process queue."""
215
+
216
+ def __init__(
217
+ self,
218
+ *,
219
+ worker: WorkerProcess,
220
+ index: int,
221
+ ) -> None:
222
+ """Create worker thread bound to process worker state."""
223
+ super().__init__(name=f"{socket.gethostname()}:{worker.pid}-{index}")
224
+ self.worker = worker
225
+
226
+ def run(self) -> None:
227
+ """Start consuming tasks for this thread."""
228
+ while self.worker.expired is None or not self.worker.expired.is_set():
229
+ try:
230
+ task_result = self.worker.task_queue.get(timeout=1.0)
231
+ except Empty:
232
+ if self.worker.shutdown_requested.is_set():
233
+ return
234
+ continue
235
+ try:
236
+ self.worker.processed_task_queue.put(
237
+ self.execute_task_result(
238
+ task_result,
239
+ )
240
+ )
241
+ finally:
242
+ self.worker.task_queue.task_done()
243
+ self.worker.record_task()
244
+
245
+ def execute_task_result(self, task_result: TaskResult) -> TaskResult:
246
+ """Execute task from task result and update result lifecycle state."""
247
+ logger.info("Executing task %r", task_result.id)
248
+ started_at = timezone.now()
249
+ task_result = dataclasses.replace(
250
+ task_result,
251
+ status=TaskResultStatus.RUNNING,
252
+ started_at=started_at,
253
+ last_attempted_at=started_at,
254
+ worker_ids=[*task_result.worker_ids, self.name],
255
+ )
256
+ task_started.send(TaskExecutor, task_result=task_result)
257
+
258
+ try:
259
+ return_value = WorkerThread.call_task(task_result)
260
+ except Exception as exception:
261
+ task_result = dataclasses.replace(
262
+ task_result,
263
+ status=TaskResultStatus.FAILED,
264
+ errors=[*task_result.errors, WorkerThread.create_task_error(exception)],
265
+ )
266
+ logger.exception("Task failed %r", task_result.id)
267
+ else:
268
+ task_result = dataclasses.replace(
269
+ task_result,
270
+ status=TaskResultStatus.SUCCESSFUL,
271
+ )
272
+ object.__setattr__(
273
+ task_result, "_return_value", normalize_json(return_value)
274
+ )
275
+ logger.info("Task successful %r", task_result.id)
276
+ finally:
277
+ task_result = dataclasses.replace(
278
+ task_result,
279
+ finished_at=timezone.now(),
280
+ )
281
+ task_finished.send(TaskExecutor, task_result=task_result)
282
+
283
+ return task_result
284
+
285
+ @staticmethod
286
+ def call_task(task_result: TaskResult) -> typing.Any:
287
+ """Call a task with context when required."""
288
+ task = task_result.task
289
+ if task.takes_context:
290
+ args = [TaskContext(task_result=task_result), *task_result.args]
291
+ else:
292
+ args = task_result.args
293
+ if iscoroutinefunction(task.func):
294
+ return asyncio.run(task.func(*args, **task_result.kwargs))
295
+ return task.func(
296
+ *args,
297
+ **task_result.kwargs,
298
+ )
299
+
300
+ @staticmethod
301
+ def create_task_error(exception: BaseException) -> TaskError:
302
+ """Build a task error payload for failed execution."""
303
+ exception_type = type(exception)
304
+ return TaskError(
305
+ exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}",
306
+ traceback="".join(format_exception(exception)),
307
+ )
File without changes
File without changes
@@ -0,0 +1,132 @@
1
+ import datetime
2
+ import signal
3
+ import sys
4
+
5
+ from django.core.management import BaseCommand, CommandError
6
+ from django.tasks import (
7
+ DEFAULT_TASK_BACKEND_ALIAS,
8
+ DEFAULT_TASK_QUEUE_NAME,
9
+ InvalidTaskBackend,
10
+ task_backends,
11
+ )
12
+
13
+ from ...executor import TaskExecutor
14
+
15
+
16
+ def kill_softly(signum, frame):
17
+ """Raise a KeyboardInterrupt to stop the worker gracefully."""
18
+ signame = signal.Signals(signum).name
19
+ raise KeyboardInterrupt(f"Received {signame} ({signum}), shutting down…")
20
+
21
+
22
+ class Command(BaseCommand):
23
+ """Run task workers to process enqueued tasks from the specified backends and queues."""
24
+
25
+ help = __doc__
26
+
27
+ def add_arguments(self, parser):
28
+ parser.add_argument(
29
+ "-b",
30
+ "--backend",
31
+ default=DEFAULT_TASK_BACKEND_ALIAS,
32
+ help="Alias of the tasks backend to use.",
33
+ )
34
+ parser.add_argument(
35
+ "-q",
36
+ "--queues",
37
+ nargs="+",
38
+ default=[DEFAULT_TASK_QUEUE_NAME],
39
+ help="Queue names to listen to and process tasks from.",
40
+ )
41
+ parser.add_argument(
42
+ "-w",
43
+ "--workers",
44
+ type=int,
45
+ help="Number of worker processes to use. Defaults to the number of CPU cores minus one.",
46
+ )
47
+ parser.add_argument(
48
+ "-t",
49
+ "--threads",
50
+ type=int,
51
+ default=1,
52
+ help="Number of threads to use. Defaults to 1. ",
53
+ )
54
+ parser.add_argument(
55
+ "--max-tasks",
56
+ type=int,
57
+ default=0,
58
+ help=(
59
+ "Number of the maximum number of tasks to run until a worker is recycled."
60
+ " Defaults to 0, which means no limit."
61
+ ),
62
+ )
63
+ parser.add_argument(
64
+ "--max-tasks-jitter",
65
+ type=int,
66
+ default=0,
67
+ help="Maximum random jitter to add to the max-tasks value by randint(0, max_tasks_jitter).",
68
+ )
69
+ parser.add_argument(
70
+ "--task-timeout",
71
+ type=float,
72
+ default=3600.0,
73
+ help="Kill hung tasks after timeout seconds. Defaults to one hour.",
74
+ )
75
+ parser.add_argument(
76
+ "--task-backlog-size",
77
+ type=int,
78
+ default=1,
79
+ help="The number of tasks to prefetch from the message queue while all workers are busy. Defaults to 1.",
80
+ )
81
+ parser.add_argument(
82
+ "--exit-empty",
83
+ action="store_true",
84
+ help="Drain the task queue and exit with 0.",
85
+ )
86
+
87
+ def handle(
88
+ self,
89
+ *,
90
+ verbosity,
91
+ backend,
92
+ queues,
93
+ workers,
94
+ threads,
95
+ max_tasks,
96
+ max_tasks_jitter,
97
+ task_timeout,
98
+ exit_empty,
99
+ **options,
100
+ ):
101
+ match sys.platform:
102
+ case "win32":
103
+ signal.signal(signal.SIGBREAK, kill_softly)
104
+ case _:
105
+ signal.signal(signal.SIGHUP, kill_softly)
106
+ signal.signal(signal.SIGTERM, kill_softly)
107
+ signal.signal(signal.SIGINT, kill_softly)
108
+ self.stdout.write(self.style.SUCCESS("Starting workers…"))
109
+ try:
110
+ backend = task_backends[backend]
111
+ except InvalidTaskBackend:
112
+ raise CommandError(f"Invalid backend: {backend!r}")
113
+ if _non_queues := set(queues) - set(backend.queues):
114
+ raise CommandError(
115
+ f"Backend does not support all specified queues: {_non_queues!r}"
116
+ )
117
+ exe = TaskExecutor(
118
+ backend=backend,
119
+ workers=workers,
120
+ threads=threads,
121
+ max_tasks=max_tasks,
122
+ max_tasks_jitter=max_tasks_jitter,
123
+ task_timeout=datetime.timedelta(seconds=task_timeout),
124
+ exit_empty=exit_empty,
125
+ queues=queues,
126
+ )
127
+ try:
128
+ exe.run()
129
+ except KeyboardInterrupt as e:
130
+ self.stdout.write(self.style.WARNING(str(e)))
131
+ self.stdout.write(self.style.NOTICE("Shutting down workers…"))
132
+ exe.shutdown()
@@ -0,0 +1,175 @@
1
+ Metadata-Version: 2.4
2
+ Name: threadmill
3
+ Version: 0.1.0
4
+ Summary: A queue agnostic worker for Django's task framework.
5
+ Keywords: Django,tasks,worker
6
+ Author-email: Johannes Maron <johannes@maron.family>
7
+ Requires-Python: >=3.12
8
+ Description-Content-Type: text/markdown
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Programming Language :: Python
11
+ Classifier: Environment :: Web Environment
12
+ Classifier: License :: OSI Approved :: BSD License
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: Microsoft :: Windows
15
+ Classifier: Operating System :: MacOS :: MacOS X
16
+ Classifier: Operating System :: POSIX
17
+ Classifier: Topic :: Communications :: Email
18
+ Classifier: Topic :: Text Processing :: Markup :: Markdown
19
+ Classifier: Topic :: Software Development
20
+ Classifier: Programming Language :: Python
21
+ Classifier: Programming Language :: Python :: 3
22
+ Classifier: Programming Language :: Python :: 3 :: Only
23
+ Classifier: Programming Language :: Python :: 3.12
24
+ Classifier: Programming Language :: Python :: 3.13
25
+ Classifier: Programming Language :: Python :: 3.14
26
+ Classifier: Framework :: Django
27
+ Classifier: Framework :: Django :: 6.1
28
+ License-File: LICENSE
29
+ Requires-Dist: django>=6.1a1
30
+ Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
31
+ Project-URL: Documentation, https://github.com/codingjoe/threadmill/
32
+ Project-URL: Funding, https://github.com/sponsors/codingjoe
33
+ Project-URL: Homepage, https://github.com/codingjoe/threadmill
34
+ Project-URL: Issues, https://github.com/codingjoe/threadmill/issues
35
+ Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/latest
36
+ Project-URL: Source, https://github.com/codingjoe/threadmill
37
+
38
+ # Threadmill
39
+
40
+ <p align="center">
41
+ <picture>
42
+ <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-dark.svg">
43
+ <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
44
+ <img alt="Django Grinder: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
45
+ </picture>
46
+ <br>
47
+ <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
48
+ <a href="https://github.com/codingjoe/threadmill/issues/new/choose">Issues</a> |
49
+ <a href="https://github.com/codingjoe/threadmill/releases">Changelog</a> |
50
+ <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
51
+ </p>
52
+
53
+ **A queue agnostic worker for Django's task framework.**
54
+
55
+ ## Design Principles
56
+
57
+ - **Durability** – We recover from any failures, even poorly written tasks.
58
+ - **Consistency** – We never lose data, even if someone unplugs the power or network.
59
+ - **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks.
60
+
61
+ > [!WARNING]
62
+ > Threadmill requires a development version of Django and is in a preview stage.
63
+
64
+ [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/)
65
+ [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill)
66
+ [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
67
+
68
+ ## Setup
69
+
70
+ You need to have [Django's Task framework][django-tasks] setup properly.
71
+
72
+ ```console
73
+ uv add threadmill
74
+ ```
75
+
76
+ Add `threadmill` to your `INSTALLED_APPS` in `settings.py`:
77
+
78
+ ```python
79
+ # settings.py
80
+ INSTALLED_APPS = [
81
+ "threadmill",
82
+ # ...
83
+ ]
84
+ ```
85
+
86
+ Finally, you launch the worker pool:
87
+
88
+ ```console
89
+ uv run manage.py threadmill
90
+ ```
91
+
92
+ ## Usage
93
+
94
+ The workers are inspired by Gunicorn, and the CLI is very similar.
95
+
96
+ ### Utilization
97
+
98
+ Depending on your workload, you can tweak the number of processes and threads.
99
+ Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
100
+
101
+ ```console
102
+ uv run manage.py threadmill --processes 4 --threads 2
103
+ ```
104
+
105
+ ### Health
106
+
107
+ If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
108
+
109
+ ```console
110
+ uv run manage.py threadmill --max-tasks 1000 --max-tasks-jitter 100
111
+ ```
112
+
113
+ This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
114
+
115
+ Should a worker crash or be killed, the pool will automatically restart it.
116
+
117
+ ### Shutdown
118
+
119
+ A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
120
+ All workers will finish the tasks they acquired and publish them.
121
+
122
+ You can use `--exit-empty` to exit immediately after all tasks have been processed,
123
+ which might be useful for draining a one-off queue.
124
+
125
+ ### Task Backlog
126
+
127
+ You can prefetch tasks from a queue to avoid IO latency bottlenecks.
128
+ However, this will increase the memory usage of the worker pool.
129
+
130
+ ```console
131
+ uv run manage.py threadmill --prefetch 100
132
+ ```
133
+
134
+ ### Task Timeouts
135
+
136
+ > [!WARNING]
137
+ > Work in progress, this feature is not yet stable.
138
+
139
+ Task timeouts are important to ensure the long-term health of your pool.
140
+ However, they need to be aligned with your queueing system's timeout settings.
141
+ The message queue needs to requeue a task that hasn't been acknowledged within the timeout.
142
+
143
+ ## Integration
144
+
145
+ > [!NOTE]
146
+ > This section is for people who want to integrate Threadmill into their queueing system.
147
+
148
+ Threadmill is designed to be durable and requires a queueing system to support late acknowledgement.
149
+
150
+ To use Threadmill, your backend will need to inherit from `threadmill.backends.AcknowledgeableTaskBackend` and implement the following methods:
151
+
152
+ ```python
153
+ class AcknowledgeableTaskBackend(BaseTaskBackend, ABC):
154
+ """Provide an interface for tasks queues to be processed by the executor."""
155
+
156
+ def acquire(
157
+ self, *queue_names: str, timeout: datetime.timedelta | None = None
158
+ ) -> TaskResult:
159
+ """
160
+ Return and lock the next task to be processed without removing it from the queue.
161
+
162
+ Args:
163
+ queue_names: The names of the queues to acquire tasks from.
164
+ timeout: The maximum time to wait for a task. If None, wait indefinitely.
165
+
166
+ Raises:
167
+ TimeoutError: If no task is available within the specified timeout.
168
+ """
169
+ raise NotImplementedError
170
+
171
+ def acknowledge(self, task_result: TaskResult) -> None:
172
+ """Remove the task from the queue and publish the result."""
173
+ raise NotImplementedError
174
+ ```
175
+
@@ -0,0 +1,11 @@
1
+ threadmill/__init__.py,sha256=d74cITlnqi-fy8kpkAxnuko9LL3HNcJMdCLWF-1wo1w,187
2
+ threadmill/_version.py,sha256=WfyBXn1Ch4X0PrFrN6x2PrPqz8ypEAzFgY_XX00HkLI,528
3
+ threadmill/backends.py,sha256=YVEpPmweKxtfpLqXJXbHx9tOpqvTyetFeJMko4exrIc,1132
4
+ threadmill/executor.py,sha256=iaEs3VCxeRc7JAqTspCZq-n367m8LmegWFOM0c0hP_0,11243
5
+ threadmill/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ threadmill/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
+ threadmill/management/commands/threadmill.py,sha256=XiO4V0RF00tq1rHP__MdtYDcqDPAF-DbeELOG8yTqro,4109
8
+ threadmill-0.1.0.dist-info/licenses/LICENSE,sha256=bUZFuK9dEiNN8YcEyAgR8ULo17q2FHLGr-tcqGTIr8A,1303
9
+ threadmill-0.1.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
10
+ threadmill-0.1.0.dist-info/METADATA,sha256=XPuuFGsrefIXdn8Vgu0HwDGTm90aQ29k99KYzwQv4h0,6483
11
+ threadmill-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: flit 3.12.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,24 @@
1
+ BSD 2-Clause License
2
+
3
+ Copyright (c) 2026, Johannes Maron
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
19
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
22
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.