threadmill 0.8.4__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {threadmill-0.8.4 → threadmill-0.10.0}/PKG-INFO +62 -1
- {threadmill-0.8.4 → threadmill-0.10.0}/README.md +59 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/pyproject.toml +2 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/_version.py +3 -3
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/redis.py +32 -35
- threadmill-0.10.0/threadmill/exceptions.py +13 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/executor.py +54 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/management/commands/threadmill.py +52 -50
- threadmill-0.10.0/threadmill/signals.py +44 -0
- threadmill-0.10.0/threadmill/urls.py +13 -0
- threadmill-0.10.0/threadmill/views.py +32 -0
- threadmill-0.8.4/threadmill/exceptions.py +0 -5
- threadmill-0.8.4/threadmill/signals.py +0 -10
- {threadmill-0.8.4 → threadmill-0.10.0}/LICENSE +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/__init__.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/apps.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/__init__.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/base.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/acknowledge.lua +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/acquire.lua +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/mover.lua +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/reaper.lua +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/__init__.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/app.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/inspector.scss +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/screens.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/telemetry.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/management/__init__.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/management/commands/__init__.py +0 -0
- {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/retry.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: threadmill
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: The most reliable backend for Django's task framework.
|
|
5
5
|
Keywords: Django,tasks,worker
|
|
6
6
|
Author-email: Johannes Maron <johannes@maron.family>
|
|
@@ -28,6 +28,7 @@ License-File: LICENSE
|
|
|
28
28
|
Requires-Dist: django>=6.1
|
|
29
29
|
Requires-Dist: textual>=8.2.7 ; extra == "inspector"
|
|
30
30
|
Requires-Dist: redis>=5 ; extra == "redis"
|
|
31
|
+
Requires-Dist: sentry-sdk>=2.44 ; extra == "sentry"
|
|
31
32
|
Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
|
|
32
33
|
Project-URL: Documentation, https://github.com/codingjoe/threadmill/
|
|
33
34
|
Project-URL: Funding, https://github.com/sponsors/codingjoe
|
|
@@ -37,6 +38,7 @@ Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/late
|
|
|
37
38
|
Project-URL: Source, https://github.com/codingjoe/threadmill
|
|
38
39
|
Provides-Extra: inspector
|
|
39
40
|
Provides-Extra: redis
|
|
41
|
+
Provides-Extra: sentry
|
|
40
42
|
|
|
41
43
|
<p align="center">
|
|
42
44
|
<picture>
|
|
@@ -171,6 +173,64 @@ uv add "threadmill[inspector]"
|
|
|
171
173
|
uv run manage.py threadmill inspector
|
|
172
174
|
```
|
|
173
175
|
|
|
176
|
+
### Sentry Metrics
|
|
177
|
+
|
|
178
|
+
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
179
|
+
is installed. Install the extra:
|
|
180
|
+
|
|
181
|
+
```console
|
|
182
|
+
uv add "threadmill[sentry]"
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
| Metric | Type | Attributes | Meaning |
|
|
186
|
+
| -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
|
|
187
|
+
| `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
|
|
188
|
+
| `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
|
|
189
|
+
| `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
|
|
190
|
+
| `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
|
|
191
|
+
| `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
|
|
192
|
+
|
|
193
|
+
### Autoscaling
|
|
194
|
+
|
|
195
|
+
Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
|
|
196
|
+
KEDA is a CNCF autoscaler for Kubernetes.
|
|
197
|
+
|
|
198
|
+
The `ready` value is the number of queued tasks that a worker can pick up now.
|
|
199
|
+
|
|
200
|
+
Include the URL in your project:
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
# urls.py
|
|
204
|
+
from django.urls import include, path
|
|
205
|
+
|
|
206
|
+
urlpatterns = [
|
|
207
|
+
path("threadmill/", include("threadmill.urls")),
|
|
208
|
+
]
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Create the ScaledObject with the public URL and both queues:
|
|
212
|
+
|
|
213
|
+
```yaml
|
|
214
|
+
apiVersion: keda.sh/v1alpha1
|
|
215
|
+
kind: ScaledObject
|
|
216
|
+
metadata:
|
|
217
|
+
name: threadmill-workers
|
|
218
|
+
spec:
|
|
219
|
+
scaleTargetRef:
|
|
220
|
+
name: threadmill-workers
|
|
221
|
+
triggers:
|
|
222
|
+
- type: metrics-api
|
|
223
|
+
metadata:
|
|
224
|
+
url: "https://example.com/threadmill/queue-stats/?queue=emails&queue=reports"
|
|
225
|
+
valueLocation: "ready"
|
|
226
|
+
targetValue: "1000"
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Use one endpoint per worker group.
|
|
230
|
+
Set `backend` to use another backend alias. It defaults to `default`.
|
|
231
|
+
|
|
232
|
+
Set `targetValue` to the ready backlog one replica can drain during your scale-up window.
|
|
233
|
+
|
|
174
234
|
### Redis Backend Options
|
|
175
235
|
|
|
176
236
|
> [!IMPORTANT]
|
|
@@ -279,4 +339,5 @@ def fetch_github_api(url: str): ...
|
|
|
279
339
|
[](https://github.com/sponsors/codingjoe)
|
|
280
340
|
|
|
281
341
|
[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
|
|
342
|
+
[sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
|
|
282
343
|
|
|
@@ -131,6 +131,64 @@ uv add "threadmill[inspector]"
|
|
|
131
131
|
uv run manage.py threadmill inspector
|
|
132
132
|
```
|
|
133
133
|
|
|
134
|
+
### Sentry Metrics
|
|
135
|
+
|
|
136
|
+
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
137
|
+
is installed. Install the extra:
|
|
138
|
+
|
|
139
|
+
```console
|
|
140
|
+
uv add "threadmill[sentry]"
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
| Metric | Type | Attributes | Meaning |
|
|
144
|
+
| -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
|
|
145
|
+
| `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
|
|
146
|
+
| `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
|
|
147
|
+
| `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
|
|
148
|
+
| `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
|
|
149
|
+
| `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
|
|
150
|
+
|
|
151
|
+
### Autoscaling
|
|
152
|
+
|
|
153
|
+
Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
|
|
154
|
+
KEDA is a CNCF autoscaler for Kubernetes.
|
|
155
|
+
|
|
156
|
+
The `ready` value is the number of queued tasks that a worker can pick up now.
|
|
157
|
+
|
|
158
|
+
Include the URL in your project:
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
# urls.py
|
|
162
|
+
from django.urls import include, path
|
|
163
|
+
|
|
164
|
+
urlpatterns = [
|
|
165
|
+
path("threadmill/", include("threadmill.urls")),
|
|
166
|
+
]
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Create the ScaledObject with the public URL and both queues:
|
|
170
|
+
|
|
171
|
+
```yaml
|
|
172
|
+
apiVersion: keda.sh/v1alpha1
|
|
173
|
+
kind: ScaledObject
|
|
174
|
+
metadata:
|
|
175
|
+
name: threadmill-workers
|
|
176
|
+
spec:
|
|
177
|
+
scaleTargetRef:
|
|
178
|
+
name: threadmill-workers
|
|
179
|
+
triggers:
|
|
180
|
+
- type: metrics-api
|
|
181
|
+
metadata:
|
|
182
|
+
url: "https://example.com/threadmill/queue-stats/?queue=emails&queue=reports"
|
|
183
|
+
valueLocation: "ready"
|
|
184
|
+
targetValue: "1000"
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Use one endpoint per worker group.
|
|
188
|
+
Set `backend` to use another backend alias. It defaults to `default`.
|
|
189
|
+
|
|
190
|
+
Set `targetValue` to the ready backlog one replica can drain during your scale-up window.
|
|
191
|
+
|
|
134
192
|
### Redis Backend Options
|
|
135
193
|
|
|
136
194
|
> [!IMPORTANT]
|
|
@@ -239,3 +297,4 @@ def fetch_github_api(url: str): ...
|
|
|
239
297
|
[](https://github.com/sponsors/codingjoe)
|
|
240
298
|
|
|
241
299
|
[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
|
|
300
|
+
[sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
|
|
@@ -37,6 +37,7 @@ optional-dependencies.inspector = [
|
|
|
37
37
|
"textual>=8.2.7",
|
|
38
38
|
]
|
|
39
39
|
optional-dependencies.redis = [ "redis>=5" ]
|
|
40
|
+
optional-dependencies.sentry = [ "sentry-sdk>=2.44" ]
|
|
40
41
|
urls.Changelog = "https://github.com/codingjoe/threadmill/releases"
|
|
41
42
|
urls.Documentation = "https://github.com/codingjoe/threadmill/"
|
|
42
43
|
urls.Funding = "https://github.com/sponsors/codingjoe"
|
|
@@ -62,6 +63,7 @@ test = [
|
|
|
62
63
|
"pytest-cov",
|
|
63
64
|
"pytest-django",
|
|
64
65
|
"redis>=5",
|
|
66
|
+
"sentry-sdk>=2.44",
|
|
65
67
|
]
|
|
66
68
|
|
|
67
69
|
[tool.setuptools_scm]
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.10.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 10, 0)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'ga458039fb'
|
|
@@ -242,7 +242,6 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
242
242
|
redis.default_backoff(), self.redis_connection_retries
|
|
243
243
|
),
|
|
244
244
|
)
|
|
245
|
-
self._async_client: redis.asyncio.Redis | None = None
|
|
246
245
|
self.redis_url = redis_url
|
|
247
246
|
self.key_prefix = f"threadmill:{{{alias}}}"
|
|
248
247
|
self.telemetry_channel = self.TELEMETRY_CHANNEL.format(prefix=self.key_prefix)
|
|
@@ -270,18 +269,14 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
270
269
|
self._idle_backoffs.backoff = backoff
|
|
271
270
|
return backoff
|
|
272
271
|
|
|
273
|
-
@property
|
|
274
272
|
def async_client(self) -> redis.asyncio.Redis:
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
),
|
|
283
|
-
)
|
|
284
|
-
return self._async_client
|
|
273
|
+
return redis.asyncio.Redis.from_url(
|
|
274
|
+
self.redis_url,
|
|
275
|
+
decode_responses=True,
|
|
276
|
+
retry=redis.asyncio.retry.Retry(
|
|
277
|
+
redis.default_backoff(), self.redis_connection_retries
|
|
278
|
+
),
|
|
279
|
+
)
|
|
285
280
|
|
|
286
281
|
def _compute_score(self, priority: int, enqueued_at: datetime.datetime) -> float:
|
|
287
282
|
"""
|
|
@@ -543,6 +538,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
543
538
|
pipe.publish(self.telemetry_channel, f"ingress:{task_result.task.queue_name}")
|
|
544
539
|
pipe.execute()
|
|
545
540
|
|
|
541
|
+
task_enqueued.send(self.__class__, task_result=task_result)
|
|
542
|
+
|
|
546
543
|
def dequeue(self, task_result: TaskResult) -> None:
|
|
547
544
|
self.client.zrem(
|
|
548
545
|
self._segment_key(task_result.status, task_result.task.queue_name),
|
|
@@ -623,17 +620,19 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
623
620
|
async def queue_stats(
|
|
624
621
|
self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
|
|
625
622
|
) -> BackendTelemetry:
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
623
|
+
async with self.async_client() as client:
|
|
624
|
+
pipe = client.pipeline()
|
|
625
|
+
for queue_name in self.queues:
|
|
626
|
+
pipe.zcard(self._segment_key(TaskResultStatus.READY, queue_name))
|
|
627
|
+
pipe.zcard(self._segment_key(TaskResultStatus.RUNNING, queue_name))
|
|
628
|
+
pipe.zcard(
|
|
629
|
+
self.DEFERRED_KEY.format(
|
|
630
|
+
prefix=self.key_prefix, queue_name=queue_name
|
|
631
|
+
)
|
|
632
|
+
)
|
|
633
|
+
pipe.zcard(self._segment_key(TaskResultStatus.SUCCESSFUL, queue_name))
|
|
634
|
+
pipe.zcard(self._segment_key(TaskResultStatus.FAILED, queue_name))
|
|
635
|
+
results = await pipe.execute()
|
|
637
636
|
zero_rates = QueueRates(interval=interval, ingress=0, egress=0)
|
|
638
637
|
queues: dict[str, QueueStats] = {}
|
|
639
638
|
for index, queue_name in enumerate(self.queues):
|
|
@@ -653,14 +652,13 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
653
652
|
async def worker_telemetry(
|
|
654
653
|
self,
|
|
655
654
|
) -> collections.abc.AsyncGenerator[TelemetryEvent]:
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
direction, _, queue_name = data.partition(":")
|
|
655
|
+
async with self.async_client() as client:
|
|
656
|
+
pubsub = client.pubsub()
|
|
657
|
+
await pubsub.subscribe(self.telemetry_channel)
|
|
658
|
+
pubsub.ignore_subscribe_messages = True
|
|
659
|
+
try:
|
|
660
|
+
async for message in pubsub.listen():
|
|
661
|
+
direction, _, queue_name = message["data"].partition(":")
|
|
664
662
|
try:
|
|
665
663
|
event = TelemetryEvent(
|
|
666
664
|
direction=TelemetryDirection(direction),
|
|
@@ -669,11 +667,10 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
669
667
|
except ValueError:
|
|
670
668
|
continue
|
|
671
669
|
yield event
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
670
|
+
finally:
|
|
671
|
+
await pubsub.unsubscribe(self.telemetry_channel)
|
|
672
|
+
await pubsub.aclose()
|
|
675
673
|
|
|
676
674
|
def close(self) -> None:
|
|
677
675
|
"""Close the Redis connections."""
|
|
678
676
|
self.client.close()
|
|
679
|
-
self._async_client = None
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Custom exceptions for the threadmill task framework."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class MissingExtraError(ImportError):
|
|
5
|
+
"""Raised when an optional dependency extra is not installed."""
|
|
6
|
+
|
|
7
|
+
def __init__(self, extra: str) -> None:
|
|
8
|
+
self.extra = extra
|
|
9
|
+
super().__init__(f"Optional dependency missing. Install: threadmill[{extra}]")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AcknowledgementTimeoutError(Exception):
|
|
13
|
+
"""Raised when a task's lease has expired before it could be acknowledged."""
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Task worker executor implementation."""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import contextlib
|
|
4
5
|
import dataclasses
|
|
5
6
|
import datetime
|
|
6
7
|
import json
|
|
@@ -26,6 +27,10 @@ from django.tasks.signals import task_finished, task_started
|
|
|
26
27
|
from django.utils import timezone
|
|
27
28
|
from django.utils.json import normalize_json
|
|
28
29
|
|
|
30
|
+
sentry_metrics = None
|
|
31
|
+
with contextlib.suppress(ImportError):
|
|
32
|
+
from sentry_sdk import metrics as sentry_metrics
|
|
33
|
+
|
|
29
34
|
if typing.TYPE_CHECKING:
|
|
30
35
|
from .backends.base import Broker, ThreadmillTaskBackend, ThreadmillTaskResult
|
|
31
36
|
|
|
@@ -128,6 +133,7 @@ class TaskExecutor:
|
|
|
128
133
|
prefetch_count: int | None = None
|
|
129
134
|
poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
|
|
130
135
|
poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
|
|
136
|
+
sample_interval: datetime.timedelta = datetime.timedelta(seconds=30)
|
|
131
137
|
shutdown_requested: threading.Event = dataclasses.field(
|
|
132
138
|
default_factory=threading.Event, init=False
|
|
133
139
|
)
|
|
@@ -183,7 +189,16 @@ class TaskExecutor:
|
|
|
183
189
|
self.pool_maintainer = threading.Thread(
|
|
184
190
|
target=self.maintain_worker_pool, daemon=True
|
|
185
191
|
)
|
|
192
|
+
self.report_worker_counts()
|
|
186
193
|
threads = [self.pool_maintainer]
|
|
194
|
+
if sentry_metrics is not None:
|
|
195
|
+
threads.append(
|
|
196
|
+
threading.Thread(
|
|
197
|
+
target=self.report_queue_depth,
|
|
198
|
+
name="threadmill-queue-depth",
|
|
199
|
+
daemon=True,
|
|
200
|
+
)
|
|
201
|
+
)
|
|
187
202
|
if self.backend.broker_class:
|
|
188
203
|
self.broker = self.backend.broker_class(self.backend)
|
|
189
204
|
threads.append(self.broker)
|
|
@@ -208,11 +223,14 @@ class TaskExecutor:
|
|
|
208
223
|
self.broker.shutdown()
|
|
209
224
|
with ThreadPoolExecutor(max_workers=self.process_count) as executor:
|
|
210
225
|
executor.map(lambda worker: worker.shutdown(), self.worker_processes)
|
|
226
|
+
if self.pool_maintainer is not None:
|
|
227
|
+
self.report_worker_counts()
|
|
211
228
|
|
|
212
229
|
def maintain_worker_pool(self) -> None:
|
|
213
230
|
"""Restart worker processes that have exited, until shutdown is requested."""
|
|
214
231
|
while not self.shutdown_requested.is_set():
|
|
215
232
|
all_dead = True
|
|
233
|
+
replaced = False
|
|
216
234
|
for index, worker in enumerate(self.worker_processes):
|
|
217
235
|
if worker.is_alive():
|
|
218
236
|
all_dead = False
|
|
@@ -227,11 +245,47 @@ class TaskExecutor:
|
|
|
227
245
|
logger.warning(
|
|
228
246
|
"Unable to start worker-%d. Retrying ...", index, exc_info=error
|
|
229
247
|
)
|
|
248
|
+
replaced = True
|
|
249
|
+
if replaced:
|
|
250
|
+
self.report_worker_counts()
|
|
230
251
|
if all_dead and self.exit_empty:
|
|
231
252
|
self.shutdown()
|
|
232
253
|
return
|
|
233
254
|
self.shutdown_requested.wait(1)
|
|
234
255
|
|
|
256
|
+
def report_worker_counts(self) -> None:
|
|
257
|
+
"""Record the live worker process and thread counts."""
|
|
258
|
+
if sentry_metrics is not None:
|
|
259
|
+
process_count = sum(worker.is_alive() for worker in self.worker_processes)
|
|
260
|
+
sentry_metrics.gauge(
|
|
261
|
+
"threadmill.worker.count",
|
|
262
|
+
process_count,
|
|
263
|
+
attributes={"pool": "process"},
|
|
264
|
+
)
|
|
265
|
+
sentry_metrics.gauge(
|
|
266
|
+
"threadmill.worker.count",
|
|
267
|
+
process_count * self.thread_count,
|
|
268
|
+
attributes={"pool": "thread"},
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
def report_queue_depth(self) -> None:
|
|
272
|
+
"""Sample queue depth into Sentry until shutdown."""
|
|
273
|
+
if self.sample_interval > datetime.timedelta(0):
|
|
274
|
+
while not self.shutdown_requested.is_set():
|
|
275
|
+
telemetry = asyncio.run(self.backend.queue_stats())
|
|
276
|
+
for queue_name, stats in telemetry.queues.items():
|
|
277
|
+
for status, depth in (
|
|
278
|
+
("ready", stats.counts.ready),
|
|
279
|
+
("running", stats.counts.running),
|
|
280
|
+
("deferred", stats.counts.deferred),
|
|
281
|
+
):
|
|
282
|
+
sentry_metrics.gauge(
|
|
283
|
+
"threadmill.queue.depth",
|
|
284
|
+
depth,
|
|
285
|
+
attributes={"queue": queue_name, "status": status},
|
|
286
|
+
)
|
|
287
|
+
self.shutdown_requested.wait(self.sample_interval.total_seconds())
|
|
288
|
+
|
|
235
289
|
|
|
236
290
|
class WorkerProcess(multiprocessing.Process):
|
|
237
291
|
"""One worker process with one prefetcher and `thread_count` consumer threads."""
|
|
@@ -8,10 +8,10 @@ from django.core.management.base import BaseCommand as DjangoBaseCommand
|
|
|
8
8
|
from django.tasks import (
|
|
9
9
|
DEFAULT_TASK_BACKEND_ALIAS,
|
|
10
10
|
DEFAULT_TASK_QUEUE_NAME,
|
|
11
|
-
InvalidTaskBackend,
|
|
12
11
|
task_backends,
|
|
13
12
|
)
|
|
14
13
|
|
|
14
|
+
from ...exceptions import MissingExtraError
|
|
15
15
|
from ...executor import JsonFormatter, TaskExecutor
|
|
16
16
|
|
|
17
17
|
|
|
@@ -22,18 +22,36 @@ def kill_softly(signum, frame):
|
|
|
22
22
|
raise KeyboardInterrupt(message)
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
class
|
|
26
|
-
"""Run task workers to process enqueued tasks from the specified backends and queues."""
|
|
27
|
-
|
|
28
|
-
help = __doc__
|
|
29
|
-
|
|
25
|
+
class ThreadmillCommand(DjangoBaseCommand):
|
|
30
26
|
def add_arguments(self, parser):
|
|
31
27
|
parser.add_argument(
|
|
32
28
|
"-b",
|
|
33
29
|
"--backend",
|
|
34
30
|
default=DEFAULT_TASK_BACKEND_ALIAS,
|
|
35
|
-
help="Alias of the tasks backend
|
|
31
|
+
help="Alias of the tasks backend.",
|
|
36
32
|
)
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def parse_duration(value: str) -> datetime.timedelta:
|
|
36
|
+
return datetime.timedelta(seconds=float(value))
|
|
37
|
+
|
|
38
|
+
def install_signal_handlers(self) -> None:
|
|
39
|
+
match sys.platform:
|
|
40
|
+
case "win32":
|
|
41
|
+
signal.signal(signal.SIGBREAK, kill_softly)
|
|
42
|
+
case _:
|
|
43
|
+
signal.signal(signal.SIGHUP, kill_softly)
|
|
44
|
+
signal.signal(signal.SIGTERM, kill_softly)
|
|
45
|
+
signal.signal(signal.SIGINT, kill_softly)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class WorkerCommand(ThreadmillCommand):
|
|
49
|
+
"""Run task workers to process enqueued tasks from the specified backends and queues."""
|
|
50
|
+
|
|
51
|
+
help = __doc__
|
|
52
|
+
|
|
53
|
+
def add_arguments(self, parser):
|
|
54
|
+
super().add_arguments(parser)
|
|
37
55
|
parser.add_argument(
|
|
38
56
|
"-q",
|
|
39
57
|
"--queues",
|
|
@@ -80,16 +98,25 @@ class WorkerCommand(DjangoBaseCommand):
|
|
|
80
98
|
)
|
|
81
99
|
parser.add_argument(
|
|
82
100
|
"--poll-interval",
|
|
83
|
-
type=
|
|
84
|
-
default=0.01,
|
|
101
|
+
type=self.parse_duration,
|
|
102
|
+
default=datetime.timedelta(seconds=0.01),
|
|
85
103
|
help="Base wait between idle acquire attempts in seconds.",
|
|
86
104
|
)
|
|
87
105
|
parser.add_argument(
|
|
88
106
|
"--poll-max-interval",
|
|
89
|
-
type=
|
|
90
|
-
default=1,
|
|
107
|
+
type=self.parse_duration,
|
|
108
|
+
default=datetime.timedelta(seconds=1),
|
|
91
109
|
help="Maximum wait between idle acquire attempts in seconds.",
|
|
92
110
|
)
|
|
111
|
+
parser.add_argument(
|
|
112
|
+
"--sample-interval",
|
|
113
|
+
type=self.parse_duration,
|
|
114
|
+
default=datetime.timedelta(seconds=30),
|
|
115
|
+
help=(
|
|
116
|
+
"Interval between queue depth samples in seconds."
|
|
117
|
+
" Defaults to 30. Set to 0 to disable queue depth sampling."
|
|
118
|
+
),
|
|
119
|
+
)
|
|
93
120
|
parser.add_argument(
|
|
94
121
|
"--exit-empty",
|
|
95
122
|
action="store_true",
|
|
@@ -116,23 +143,14 @@ class WorkerCommand(DjangoBaseCommand):
|
|
|
116
143
|
prefetch_count,
|
|
117
144
|
poll_interval,
|
|
118
145
|
poll_max_interval,
|
|
146
|
+
sample_interval,
|
|
119
147
|
exit_empty,
|
|
120
148
|
log_format,
|
|
121
149
|
**options,
|
|
122
150
|
):
|
|
123
|
-
|
|
124
|
-
case "win32":
|
|
125
|
-
signal.signal(signal.SIGBREAK, kill_softly)
|
|
126
|
-
case _:
|
|
127
|
-
signal.signal(signal.SIGHUP, kill_softly)
|
|
128
|
-
signal.signal(signal.SIGTERM, kill_softly)
|
|
129
|
-
signal.signal(signal.SIGINT, kill_softly)
|
|
151
|
+
self.install_signal_handlers()
|
|
130
152
|
self.stdout.write(self.style.SUCCESS("Starting workers…"))
|
|
131
|
-
|
|
132
|
-
backend = task_backends[backend]
|
|
133
|
-
except InvalidTaskBackend:
|
|
134
|
-
message = f"Invalid backend: {backend!r}"
|
|
135
|
-
raise CommandError(message)
|
|
153
|
+
backend = task_backends[backend]
|
|
136
154
|
if _non_queues := set(queues) - set(backend.queues):
|
|
137
155
|
message = f"Backend does not support all specified queues: {_non_queues!r}"
|
|
138
156
|
raise CommandError(message)
|
|
@@ -158,8 +176,9 @@ class WorkerCommand(DjangoBaseCommand):
|
|
|
158
176
|
max_tasks=max_tasks,
|
|
159
177
|
max_tasks_jitter=max_tasks_jitter,
|
|
160
178
|
prefetch_count=prefetch_count,
|
|
161
|
-
poll_interval=
|
|
162
|
-
poll_max_interval=
|
|
179
|
+
poll_interval=poll_interval,
|
|
180
|
+
poll_max_interval=poll_max_interval,
|
|
181
|
+
sample_interval=sample_interval,
|
|
163
182
|
exit_empty=exit_empty,
|
|
164
183
|
queues=queues,
|
|
165
184
|
log_formatter=log_formatter,
|
|
@@ -172,37 +191,17 @@ class WorkerCommand(DjangoBaseCommand):
|
|
|
172
191
|
exe.shutdown()
|
|
173
192
|
|
|
174
193
|
|
|
175
|
-
class InspectorCommand(
|
|
194
|
+
class InspectorCommand(ThreadmillCommand):
|
|
176
195
|
"""Launch the textual TUI inspector for task backends and queues."""
|
|
177
196
|
|
|
178
197
|
help = __doc__
|
|
179
198
|
|
|
180
|
-
def add_arguments(self, parser):
|
|
181
|
-
parser.add_argument(
|
|
182
|
-
"-b",
|
|
183
|
-
"--backend",
|
|
184
|
-
default=DEFAULT_TASK_BACKEND_ALIAS,
|
|
185
|
-
help="Alias of the tasks backend to inspect.",
|
|
186
|
-
)
|
|
187
|
-
|
|
188
199
|
def handle(self, *, backend, **options):
|
|
189
200
|
try:
|
|
190
|
-
from ...backends.base import ThreadmillTaskBackend
|
|
191
201
|
from ...inspector.app import InspectorApp
|
|
192
|
-
except ImportError:
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
try:
|
|
196
|
-
backend = task_backends[backend]
|
|
197
|
-
except InvalidTaskBackend as e:
|
|
198
|
-
message = f"Invalid backend: {backend!r}"
|
|
199
|
-
raise CommandError(message) from e
|
|
200
|
-
|
|
201
|
-
if not isinstance(backend, ThreadmillTaskBackend):
|
|
202
|
-
message = f"Backend {backend.alias!r} does not support inspection."
|
|
203
|
-
raise CommandError(message)
|
|
204
|
-
|
|
205
|
-
InspectorApp(backend=backend).run()
|
|
202
|
+
except ImportError as e:
|
|
203
|
+
raise MissingExtraError("inspector") from e
|
|
204
|
+
InspectorApp(backend=task_backends[backend]).run()
|
|
206
205
|
|
|
207
206
|
|
|
208
207
|
class Command(BaseCommand):
|
|
@@ -223,4 +222,7 @@ class Command(BaseCommand):
|
|
|
223
222
|
|
|
224
223
|
def handle(self, *args, **options):
|
|
225
224
|
subcommand_name = options.pop("subcommand")
|
|
226
|
-
|
|
225
|
+
try:
|
|
226
|
+
self.subcommands[subcommand_name]().execute(*args, **options)
|
|
227
|
+
except MissingExtraError as e:
|
|
228
|
+
raise CommandError(str(e)) from e
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from django.db import close_old_connections
|
|
2
|
+
from django.dispatch import receiver
|
|
3
|
+
from django.tasks.signals import task_enqueued, task_finished, task_started
|
|
4
|
+
|
|
5
|
+
from .backends.base import ThreadmillTaskBackend
|
|
6
|
+
from .executor import TaskExecutor, sentry_metrics
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@receiver([task_started, task_finished], sender=TaskExecutor)
|
|
10
|
+
def close_task_database_connection(sender, task_result, **kwargs):
|
|
11
|
+
close_old_connections()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@receiver(task_enqueued)
|
|
15
|
+
def record_task_enqueued(sender, task_result, **kwargs):
|
|
16
|
+
"""Record one task entering its queue."""
|
|
17
|
+
if sentry_metrics is not None and issubclass(sender, ThreadmillTaskBackend):
|
|
18
|
+
sentry_metrics.count(
|
|
19
|
+
"threadmill.queue.ingress",
|
|
20
|
+
1,
|
|
21
|
+
attributes={"queue": task_result.task.queue_name},
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@receiver(task_finished, sender=TaskExecutor)
|
|
26
|
+
def record_task_finished(sender, task_result, **kwargs):
|
|
27
|
+
"""Emit one egress count and one duration observation per attempt."""
|
|
28
|
+
if sentry_metrics is not None:
|
|
29
|
+
sentry_metrics.count(
|
|
30
|
+
"threadmill.queue.egress",
|
|
31
|
+
1,
|
|
32
|
+
attributes={"queue": task_result.task.queue_name},
|
|
33
|
+
)
|
|
34
|
+
duration = task_result.finished_at - task_result.last_attempted_at
|
|
35
|
+
sentry_metrics.distribution(
|
|
36
|
+
"threadmill.task.duration",
|
|
37
|
+
duration.total_seconds() * 1e3,
|
|
38
|
+
unit="millisecond",
|
|
39
|
+
attributes={
|
|
40
|
+
"task": task_result.task.module_path,
|
|
41
|
+
"outcome": task_result.status.value.lower(),
|
|
42
|
+
"attempt": task_result.attempts,
|
|
43
|
+
},
|
|
44
|
+
)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from django.http import Http404, HttpRequest, JsonResponse
|
|
2
|
+
from django.tasks import (
|
|
3
|
+
DEFAULT_TASK_BACKEND_ALIAS,
|
|
4
|
+
InvalidTaskBackend,
|
|
5
|
+
task_backends,
|
|
6
|
+
)
|
|
7
|
+
from django.views import View
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class QueueStatsView(View):
|
|
11
|
+
async def get(self, request: HttpRequest) -> JsonResponse:
|
|
12
|
+
try:
|
|
13
|
+
backend = task_backends[
|
|
14
|
+
request.GET.get("backend", DEFAULT_TASK_BACKEND_ALIAS)
|
|
15
|
+
]
|
|
16
|
+
except InvalidTaskBackend as exception:
|
|
17
|
+
raise Http404 from exception
|
|
18
|
+
try:
|
|
19
|
+
telemetry = await backend.queue_stats()
|
|
20
|
+
except (AttributeError, NotImplementedError) as exception:
|
|
21
|
+
raise Http404 from exception
|
|
22
|
+
try:
|
|
23
|
+
return JsonResponse(
|
|
24
|
+
{
|
|
25
|
+
"ready": sum(
|
|
26
|
+
telemetry.queues[queue_name].counts.ready
|
|
27
|
+
for queue_name in request.GET.getlist("queue", backend.queues)
|
|
28
|
+
)
|
|
29
|
+
}
|
|
30
|
+
)
|
|
31
|
+
except (AttributeError, KeyError) as exception:
|
|
32
|
+
raise Http404 from exception
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
from django.db import close_old_connections
|
|
2
|
-
from django.dispatch import receiver
|
|
3
|
-
from django.tasks.signals import task_finished, task_started
|
|
4
|
-
|
|
5
|
-
from .executor import TaskExecutor
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
@receiver([task_started, task_finished], sender=TaskExecutor)
|
|
9
|
-
def close_task_database_connection(sender, task_result, **kwargs):
|
|
10
|
-
close_old_connections()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|