threadmill 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {threadmill-0.9.0 → threadmill-0.10.0}/PKG-INFO +21 -1
- {threadmill-0.9.0 → threadmill-0.10.0}/README.md +18 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/pyproject.toml +2 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/_version.py +3 -3
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/redis.py +2 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/executor.py +54 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/management/commands/threadmill.py +11 -0
- threadmill-0.10.0/threadmill/signals.py +44 -0
- threadmill-0.9.0/threadmill/signals.py +0 -10
- {threadmill-0.9.0 → threadmill-0.10.0}/LICENSE +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/apps.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/base.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/acknowledge.lua +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/acquire.lua +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/mover.lua +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/reaper.lua +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/exceptions.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/app.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/inspector.scss +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/screens.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/telemetry.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/management/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/management/commands/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/retry.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/urls.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: threadmill
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: The most reliable backend for Django's task framework.
|
|
5
5
|
Keywords: Django,tasks,worker
|
|
6
6
|
Author-email: Johannes Maron <johannes@maron.family>
|
|
@@ -28,6 +28,7 @@ License-File: LICENSE
|
|
|
28
28
|
Requires-Dist: django>=6.1
|
|
29
29
|
Requires-Dist: textual>=8.2.7 ; extra == "inspector"
|
|
30
30
|
Requires-Dist: redis>=5 ; extra == "redis"
|
|
31
|
+
Requires-Dist: sentry-sdk>=2.44 ; extra == "sentry"
|
|
31
32
|
Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
|
|
32
33
|
Project-URL: Documentation, https://github.com/codingjoe/threadmill/
|
|
33
34
|
Project-URL: Funding, https://github.com/sponsors/codingjoe
|
|
@@ -37,6 +38,7 @@ Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/late
|
|
|
37
38
|
Project-URL: Source, https://github.com/codingjoe/threadmill
|
|
38
39
|
Provides-Extra: inspector
|
|
39
40
|
Provides-Extra: redis
|
|
41
|
+
Provides-Extra: sentry
|
|
40
42
|
|
|
41
43
|
<p align="center">
|
|
42
44
|
<picture>
|
|
@@ -171,6 +173,23 @@ uv add "threadmill[inspector]"
|
|
|
171
173
|
uv run manage.py threadmill inspector
|
|
172
174
|
```
|
|
173
175
|
|
|
176
|
+
### Sentry Metrics
|
|
177
|
+
|
|
178
|
+
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
179
|
+
is installed. Install the extra:
|
|
180
|
+
|
|
181
|
+
```console
|
|
182
|
+
uv add "threadmill[sentry]"
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
| Metric | Type | Attributes | Meaning |
|
|
186
|
+
| -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
|
|
187
|
+
| `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
|
|
188
|
+
| `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
|
|
189
|
+
| `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
|
|
190
|
+
| `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
|
|
191
|
+
| `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
|
|
192
|
+
|
|
174
193
|
### Autoscaling
|
|
175
194
|
|
|
176
195
|
Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
|
|
@@ -320,4 +339,5 @@ def fetch_github_api(url: str): ...
|
|
|
320
339
|
[](https://github.com/sponsors/codingjoe)
|
|
321
340
|
|
|
322
341
|
[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
|
|
342
|
+
[sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
|
|
323
343
|
|
|
@@ -131,6 +131,23 @@ uv add "threadmill[inspector]"
|
|
|
131
131
|
uv run manage.py threadmill inspector
|
|
132
132
|
```
|
|
133
133
|
|
|
134
|
+
### Sentry Metrics
|
|
135
|
+
|
|
136
|
+
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
137
|
+
is installed. Install the extra:
|
|
138
|
+
|
|
139
|
+
```console
|
|
140
|
+
uv add "threadmill[sentry]"
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
| Metric | Type | Attributes | Meaning |
|
|
144
|
+
| -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
|
|
145
|
+
| `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
|
|
146
|
+
| `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
|
|
147
|
+
| `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
|
|
148
|
+
| `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
|
|
149
|
+
| `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
|
|
150
|
+
|
|
134
151
|
### Autoscaling
|
|
135
152
|
|
|
136
153
|
Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
|
|
@@ -280,3 +297,4 @@ def fetch_github_api(url: str): ...
|
|
|
280
297
|
[](https://github.com/sponsors/codingjoe)
|
|
281
298
|
|
|
282
299
|
[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
|
|
300
|
+
[sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
|
|
@@ -37,6 +37,7 @@ optional-dependencies.inspector = [
|
|
|
37
37
|
"textual>=8.2.7",
|
|
38
38
|
]
|
|
39
39
|
optional-dependencies.redis = [ "redis>=5" ]
|
|
40
|
+
optional-dependencies.sentry = [ "sentry-sdk>=2.44" ]
|
|
40
41
|
urls.Changelog = "https://github.com/codingjoe/threadmill/releases"
|
|
41
42
|
urls.Documentation = "https://github.com/codingjoe/threadmill/"
|
|
42
43
|
urls.Funding = "https://github.com/sponsors/codingjoe"
|
|
@@ -62,6 +63,7 @@ test = [
|
|
|
62
63
|
"pytest-cov",
|
|
63
64
|
"pytest-django",
|
|
64
65
|
"redis>=5",
|
|
66
|
+
"sentry-sdk>=2.44",
|
|
65
67
|
]
|
|
66
68
|
|
|
67
69
|
[tool.setuptools_scm]
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.10.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 10, 0)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'ga458039fb'
|
|
@@ -538,6 +538,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
538
538
|
pipe.publish(self.telemetry_channel, f"ingress:{task_result.task.queue_name}")
|
|
539
539
|
pipe.execute()
|
|
540
540
|
|
|
541
|
+
task_enqueued.send(self.__class__, task_result=task_result)
|
|
542
|
+
|
|
541
543
|
def dequeue(self, task_result: TaskResult) -> None:
|
|
542
544
|
self.client.zrem(
|
|
543
545
|
self._segment_key(task_result.status, task_result.task.queue_name),
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Task worker executor implementation."""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import contextlib
|
|
4
5
|
import dataclasses
|
|
5
6
|
import datetime
|
|
6
7
|
import json
|
|
@@ -26,6 +27,10 @@ from django.tasks.signals import task_finished, task_started
|
|
|
26
27
|
from django.utils import timezone
|
|
27
28
|
from django.utils.json import normalize_json
|
|
28
29
|
|
|
30
|
+
sentry_metrics = None
|
|
31
|
+
with contextlib.suppress(ImportError):
|
|
32
|
+
from sentry_sdk import metrics as sentry_metrics
|
|
33
|
+
|
|
29
34
|
if typing.TYPE_CHECKING:
|
|
30
35
|
from .backends.base import Broker, ThreadmillTaskBackend, ThreadmillTaskResult
|
|
31
36
|
|
|
@@ -128,6 +133,7 @@ class TaskExecutor:
|
|
|
128
133
|
prefetch_count: int | None = None
|
|
129
134
|
poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
|
|
130
135
|
poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
|
|
136
|
+
sample_interval: datetime.timedelta = datetime.timedelta(seconds=30)
|
|
131
137
|
shutdown_requested: threading.Event = dataclasses.field(
|
|
132
138
|
default_factory=threading.Event, init=False
|
|
133
139
|
)
|
|
@@ -183,7 +189,16 @@ class TaskExecutor:
|
|
|
183
189
|
self.pool_maintainer = threading.Thread(
|
|
184
190
|
target=self.maintain_worker_pool, daemon=True
|
|
185
191
|
)
|
|
192
|
+
self.report_worker_counts()
|
|
186
193
|
threads = [self.pool_maintainer]
|
|
194
|
+
if sentry_metrics is not None:
|
|
195
|
+
threads.append(
|
|
196
|
+
threading.Thread(
|
|
197
|
+
target=self.report_queue_depth,
|
|
198
|
+
name="threadmill-queue-depth",
|
|
199
|
+
daemon=True,
|
|
200
|
+
)
|
|
201
|
+
)
|
|
187
202
|
if self.backend.broker_class:
|
|
188
203
|
self.broker = self.backend.broker_class(self.backend)
|
|
189
204
|
threads.append(self.broker)
|
|
@@ -208,11 +223,14 @@ class TaskExecutor:
|
|
|
208
223
|
self.broker.shutdown()
|
|
209
224
|
with ThreadPoolExecutor(max_workers=self.process_count) as executor:
|
|
210
225
|
executor.map(lambda worker: worker.shutdown(), self.worker_processes)
|
|
226
|
+
if self.pool_maintainer is not None:
|
|
227
|
+
self.report_worker_counts()
|
|
211
228
|
|
|
212
229
|
def maintain_worker_pool(self) -> None:
|
|
213
230
|
"""Restart worker processes that have exited, until shutdown is requested."""
|
|
214
231
|
while not self.shutdown_requested.is_set():
|
|
215
232
|
all_dead = True
|
|
233
|
+
replaced = False
|
|
216
234
|
for index, worker in enumerate(self.worker_processes):
|
|
217
235
|
if worker.is_alive():
|
|
218
236
|
all_dead = False
|
|
@@ -227,11 +245,47 @@ class TaskExecutor:
|
|
|
227
245
|
logger.warning(
|
|
228
246
|
"Unable to start worker-%d. Retrying ...", index, exc_info=error
|
|
229
247
|
)
|
|
248
|
+
replaced = True
|
|
249
|
+
if replaced:
|
|
250
|
+
self.report_worker_counts()
|
|
230
251
|
if all_dead and self.exit_empty:
|
|
231
252
|
self.shutdown()
|
|
232
253
|
return
|
|
233
254
|
self.shutdown_requested.wait(1)
|
|
234
255
|
|
|
256
|
+
def report_worker_counts(self) -> None:
|
|
257
|
+
"""Record the live worker process and thread counts."""
|
|
258
|
+
if sentry_metrics is not None:
|
|
259
|
+
process_count = sum(worker.is_alive() for worker in self.worker_processes)
|
|
260
|
+
sentry_metrics.gauge(
|
|
261
|
+
"threadmill.worker.count",
|
|
262
|
+
process_count,
|
|
263
|
+
attributes={"pool": "process"},
|
|
264
|
+
)
|
|
265
|
+
sentry_metrics.gauge(
|
|
266
|
+
"threadmill.worker.count",
|
|
267
|
+
process_count * self.thread_count,
|
|
268
|
+
attributes={"pool": "thread"},
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
def report_queue_depth(self) -> None:
|
|
272
|
+
"""Sample queue depth into Sentry until shutdown."""
|
|
273
|
+
if self.sample_interval > datetime.timedelta(0):
|
|
274
|
+
while not self.shutdown_requested.is_set():
|
|
275
|
+
telemetry = asyncio.run(self.backend.queue_stats())
|
|
276
|
+
for queue_name, stats in telemetry.queues.items():
|
|
277
|
+
for status, depth in (
|
|
278
|
+
("ready", stats.counts.ready),
|
|
279
|
+
("running", stats.counts.running),
|
|
280
|
+
("deferred", stats.counts.deferred),
|
|
281
|
+
):
|
|
282
|
+
sentry_metrics.gauge(
|
|
283
|
+
"threadmill.queue.depth",
|
|
284
|
+
depth,
|
|
285
|
+
attributes={"queue": queue_name, "status": status},
|
|
286
|
+
)
|
|
287
|
+
self.shutdown_requested.wait(self.sample_interval.total_seconds())
|
|
288
|
+
|
|
235
289
|
|
|
236
290
|
class WorkerProcess(multiprocessing.Process):
|
|
237
291
|
"""One worker process with one prefetcher and `thread_count` consumer threads."""
|
|
@@ -108,6 +108,15 @@ class WorkerCommand(ThreadmillCommand):
|
|
|
108
108
|
default=datetime.timedelta(seconds=1),
|
|
109
109
|
help="Maximum wait between idle acquire attempts in seconds.",
|
|
110
110
|
)
|
|
111
|
+
parser.add_argument(
|
|
112
|
+
"--sample-interval",
|
|
113
|
+
type=self.parse_duration,
|
|
114
|
+
default=datetime.timedelta(seconds=30),
|
|
115
|
+
help=(
|
|
116
|
+
"Interval between queue depth samples in seconds."
|
|
117
|
+
" Defaults to 30. Set to 0 to disable queue depth sampling."
|
|
118
|
+
),
|
|
119
|
+
)
|
|
111
120
|
parser.add_argument(
|
|
112
121
|
"--exit-empty",
|
|
113
122
|
action="store_true",
|
|
@@ -134,6 +143,7 @@ class WorkerCommand(ThreadmillCommand):
|
|
|
134
143
|
prefetch_count,
|
|
135
144
|
poll_interval,
|
|
136
145
|
poll_max_interval,
|
|
146
|
+
sample_interval,
|
|
137
147
|
exit_empty,
|
|
138
148
|
log_format,
|
|
139
149
|
**options,
|
|
@@ -168,6 +178,7 @@ class WorkerCommand(ThreadmillCommand):
|
|
|
168
178
|
prefetch_count=prefetch_count,
|
|
169
179
|
poll_interval=poll_interval,
|
|
170
180
|
poll_max_interval=poll_max_interval,
|
|
181
|
+
sample_interval=sample_interval,
|
|
171
182
|
exit_empty=exit_empty,
|
|
172
183
|
queues=queues,
|
|
173
184
|
log_formatter=log_formatter,
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from django.db import close_old_connections
|
|
2
|
+
from django.dispatch import receiver
|
|
3
|
+
from django.tasks.signals import task_enqueued, task_finished, task_started
|
|
4
|
+
|
|
5
|
+
from .backends.base import ThreadmillTaskBackend
|
|
6
|
+
from .executor import TaskExecutor, sentry_metrics
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@receiver([task_started, task_finished], sender=TaskExecutor)
|
|
10
|
+
def close_task_database_connection(sender, task_result, **kwargs):
|
|
11
|
+
close_old_connections()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@receiver(task_enqueued)
|
|
15
|
+
def record_task_enqueued(sender, task_result, **kwargs):
|
|
16
|
+
"""Record one task entering its queue."""
|
|
17
|
+
if sentry_metrics is not None and issubclass(sender, ThreadmillTaskBackend):
|
|
18
|
+
sentry_metrics.count(
|
|
19
|
+
"threadmill.queue.ingress",
|
|
20
|
+
1,
|
|
21
|
+
attributes={"queue": task_result.task.queue_name},
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@receiver(task_finished, sender=TaskExecutor)
|
|
26
|
+
def record_task_finished(sender, task_result, **kwargs):
|
|
27
|
+
"""Emit one egress count and one duration observation per attempt."""
|
|
28
|
+
if sentry_metrics is not None:
|
|
29
|
+
sentry_metrics.count(
|
|
30
|
+
"threadmill.queue.egress",
|
|
31
|
+
1,
|
|
32
|
+
attributes={"queue": task_result.task.queue_name},
|
|
33
|
+
)
|
|
34
|
+
duration = task_result.finished_at - task_result.last_attempted_at
|
|
35
|
+
sentry_metrics.distribution(
|
|
36
|
+
"threadmill.task.duration",
|
|
37
|
+
duration.total_seconds() * 1e3,
|
|
38
|
+
unit="millisecond",
|
|
39
|
+
attributes={
|
|
40
|
+
"task": task_result.task.module_path,
|
|
41
|
+
"outcome": task_result.status.value.lower(),
|
|
42
|
+
"attempt": task_result.attempts,
|
|
43
|
+
},
|
|
44
|
+
)
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
from django.db import close_old_connections
|
|
2
|
-
from django.dispatch import receiver
|
|
3
|
-
from django.tasks.signals import task_finished, task_started
|
|
4
|
-
|
|
5
|
-
from .executor import TaskExecutor
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
@receiver([task_started, task_finished], sender=TaskExecutor)
|
|
9
|
-
def close_task_database_connection(sender, task_result, **kwargs):
|
|
10
|
-
close_old_connections()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|