threadmill 0.8.4__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {threadmill-0.8.4 → threadmill-0.10.0}/PKG-INFO +62 -1
  2. {threadmill-0.8.4 → threadmill-0.10.0}/README.md +59 -0
  3. {threadmill-0.8.4 → threadmill-0.10.0}/pyproject.toml +2 -0
  4. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/_version.py +3 -3
  5. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/redis.py +32 -35
  6. threadmill-0.10.0/threadmill/exceptions.py +13 -0
  7. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/executor.py +54 -0
  8. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/management/commands/threadmill.py +52 -50
  9. threadmill-0.10.0/threadmill/signals.py +44 -0
  10. threadmill-0.10.0/threadmill/urls.py +13 -0
  11. threadmill-0.10.0/threadmill/views.py +32 -0
  12. threadmill-0.8.4/threadmill/exceptions.py +0 -5
  13. threadmill-0.8.4/threadmill/signals.py +0 -10
  14. {threadmill-0.8.4 → threadmill-0.10.0}/LICENSE +0 -0
  15. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/__init__.py +0 -0
  16. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/apps.py +0 -0
  17. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/__init__.py +0 -0
  18. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/base.py +0 -0
  19. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/acknowledge.lua +0 -0
  20. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/acquire.lua +0 -0
  21. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/mover.lua +0 -0
  22. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/backends/lua/reaper.lua +0 -0
  23. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/__init__.py +0 -0
  24. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/app.py +0 -0
  25. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/inspector.scss +0 -0
  26. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/screens.py +0 -0
  27. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/inspector/telemetry.py +0 -0
  28. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/management/__init__.py +0 -0
  29. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/management/commands/__init__.py +0 -0
  30. {threadmill-0.8.4 → threadmill-0.10.0}/threadmill/retry.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: threadmill
3
- Version: 0.8.4
3
+ Version: 0.10.0
4
4
  Summary: The most reliable backend for Django's task framework.
5
5
  Keywords: Django,tasks,worker
6
6
  Author-email: Johannes Maron <johannes@maron.family>
@@ -28,6 +28,7 @@ License-File: LICENSE
28
28
  Requires-Dist: django>=6.1
29
29
  Requires-Dist: textual>=8.2.7 ; extra == "inspector"
30
30
  Requires-Dist: redis>=5 ; extra == "redis"
31
+ Requires-Dist: sentry-sdk>=2.44 ; extra == "sentry"
31
32
  Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
32
33
  Project-URL: Documentation, https://github.com/codingjoe/threadmill/
33
34
  Project-URL: Funding, https://github.com/sponsors/codingjoe
@@ -37,6 +38,7 @@ Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/late
37
38
  Project-URL: Source, https://github.com/codingjoe/threadmill
38
39
  Provides-Extra: inspector
39
40
  Provides-Extra: redis
41
+ Provides-Extra: sentry
40
42
 
41
43
  <p align="center">
42
44
  <picture>
@@ -171,6 +173,64 @@ uv add "threadmill[inspector]"
171
173
  uv run manage.py threadmill inspector
172
174
  ```
173
175
 
176
+ ### Sentry Metrics
177
+
178
+ Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
179
+ is installed. Install the extra:
180
+
181
+ ```console
182
+ uv add "threadmill[sentry]"
183
+ ```
184
+
185
+ | Metric | Type | Attributes | Meaning |
186
+ | -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
187
+ | `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
188
+ | `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
189
+ | `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
190
+ | `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
191
+ | `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
192
+
193
+ ### Autoscaling
194
+
195
+ Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
196
+ KEDA is a CNCF autoscaler for Kubernetes.
197
+
198
+ The `ready` value is the number of queued tasks that a worker can pick up now.
199
+
200
+ Include the URL in your project:
201
+
202
+ ```python
203
+ # urls.py
204
+ from django.urls import include, path
205
+
206
+ urlpatterns = [
207
+ path("threadmill/", include("threadmill.urls")),
208
+ ]
209
+ ```
210
+
211
+ Create the ScaledObject with the public URL and both queues:
212
+
213
+ ```yaml
214
+ apiVersion: keda.sh/v1alpha1
215
+ kind: ScaledObject
216
+ metadata:
217
+ name: threadmill-workers
218
+ spec:
219
+ scaleTargetRef:
220
+ name: threadmill-workers
221
+ triggers:
222
+ - type: metrics-api
223
+ metadata:
224
+ url: "https://example.com/threadmill/queue-stats/?queue=emails&queue=reports"
225
+ valueLocation: "ready"
226
+ targetValue: "1000"
227
+ ```
228
+
229
+ Use one endpoint per worker group.
230
+ Set `backend` to use another backend alias. It defaults to `default`.
231
+
232
+ Set `targetValue` to the ready backlog one replica can drain during your scale-up window.
233
+
174
234
  ### Redis Backend Options
175
235
 
176
236
  > [!IMPORTANT]
@@ -279,4 +339,5 @@ def fetch_github_api(url: str): ...
279
339
  [![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
280
340
 
281
341
  [django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
342
+ [sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
282
343
 
@@ -131,6 +131,64 @@ uv add "threadmill[inspector]"
131
131
  uv run manage.py threadmill inspector
132
132
  ```
133
133
 
134
+ ### Sentry Metrics
135
+
136
+ Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
137
+ is installed. Install the extra:
138
+
139
+ ```console
140
+ uv add "threadmill[sentry]"
141
+ ```
142
+
143
+ | Metric | Type | Attributes | Meaning |
144
+ | -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
145
+ | `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
146
+ | `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
147
+ | `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
148
+ | `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
149
+ | `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
150
+
151
+ ### Autoscaling
152
+
153
+ Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
154
+ KEDA is a CNCF autoscaler for Kubernetes.
155
+
156
+ The `ready` value is the number of queued tasks that a worker can pick up now.
157
+
158
+ Include the URL in your project:
159
+
160
+ ```python
161
+ # urls.py
162
+ from django.urls import include, path
163
+
164
+ urlpatterns = [
165
+ path("threadmill/", include("threadmill.urls")),
166
+ ]
167
+ ```
168
+
169
+ Create the ScaledObject with the public URL and both queues:
170
+
171
+ ```yaml
172
+ apiVersion: keda.sh/v1alpha1
173
+ kind: ScaledObject
174
+ metadata:
175
+ name: threadmill-workers
176
+ spec:
177
+ scaleTargetRef:
178
+ name: threadmill-workers
179
+ triggers:
180
+ - type: metrics-api
181
+ metadata:
182
+ url: "https://example.com/threadmill/queue-stats/?queue=emails&queue=reports"
183
+ valueLocation: "ready"
184
+ targetValue: "1000"
185
+ ```
186
+
187
+ Use one endpoint per worker group.
188
+ Set `backend` to use another backend alias. It defaults to `default`.
189
+
190
+ Set `targetValue` to the ready backlog one replica can drain during your scale-up window.
191
+
134
192
  ### Redis Backend Options
135
193
 
136
194
  > [!IMPORTANT]
@@ -239,3 +297,4 @@ def fetch_github_api(url: str): ...
239
297
  [![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
240
298
 
241
299
  [django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
300
+ [sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
@@ -37,6 +37,7 @@ optional-dependencies.inspector = [
37
37
  "textual>=8.2.7",
38
38
  ]
39
39
  optional-dependencies.redis = [ "redis>=5" ]
40
+ optional-dependencies.sentry = [ "sentry-sdk>=2.44" ]
40
41
  urls.Changelog = "https://github.com/codingjoe/threadmill/releases"
41
42
  urls.Documentation = "https://github.com/codingjoe/threadmill/"
42
43
  urls.Funding = "https://github.com/sponsors/codingjoe"
@@ -62,6 +63,7 @@ test = [
62
63
  "pytest-cov",
63
64
  "pytest-django",
64
65
  "redis>=5",
66
+ "sentry-sdk>=2.44",
65
67
  ]
66
68
 
67
69
  [tool.setuptools_scm]
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.8.4'
22
- __version_tuple__ = version_tuple = (0, 8, 4)
21
+ __version__ = version = '0.10.0'
22
+ __version_tuple__ = version_tuple = (0, 10, 0)
23
23
 
24
- __commit_id__ = commit_id = 'g071a4ce76'
24
+ __commit_id__ = commit_id = 'ga458039fb'
@@ -242,7 +242,6 @@ class RedisTaskBackend(ThreadmillTaskBackend):
242
242
  redis.default_backoff(), self.redis_connection_retries
243
243
  ),
244
244
  )
245
- self._async_client: redis.asyncio.Redis | None = None
246
245
  self.redis_url = redis_url
247
246
  self.key_prefix = f"threadmill:{{{alias}}}"
248
247
  self.telemetry_channel = self.TELEMETRY_CHANNEL.format(prefix=self.key_prefix)
@@ -270,18 +269,14 @@ class RedisTaskBackend(ThreadmillTaskBackend):
270
269
  self._idle_backoffs.backoff = backoff
271
270
  return backoff
272
271
 
273
- @property
274
272
  def async_client(self) -> redis.asyncio.Redis:
275
- """Lazily-created async Redis client, reused across calls."""
276
- if self._async_client is None:
277
- self._async_client = redis.asyncio.Redis.from_url(
278
- self.redis_url,
279
- decode_responses=True,
280
- retry=redis.asyncio.retry.Retry(
281
- redis.default_backoff(), self.redis_connection_retries
282
- ),
283
- )
284
- return self._async_client
273
+ return redis.asyncio.Redis.from_url(
274
+ self.redis_url,
275
+ decode_responses=True,
276
+ retry=redis.asyncio.retry.Retry(
277
+ redis.default_backoff(), self.redis_connection_retries
278
+ ),
279
+ )
285
280
 
286
281
  def _compute_score(self, priority: int, enqueued_at: datetime.datetime) -> float:
287
282
  """
@@ -543,6 +538,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
543
538
  pipe.publish(self.telemetry_channel, f"ingress:{task_result.task.queue_name}")
544
539
  pipe.execute()
545
540
 
541
+ task_enqueued.send(self.__class__, task_result=task_result)
542
+
546
543
  def dequeue(self, task_result: TaskResult) -> None:
547
544
  self.client.zrem(
548
545
  self._segment_key(task_result.status, task_result.task.queue_name),
@@ -623,17 +620,19 @@ class RedisTaskBackend(ThreadmillTaskBackend):
623
620
  async def queue_stats(
624
621
  self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
625
622
  ) -> BackendTelemetry:
626
- client = self.async_client
627
- pipe = client.pipeline()
628
- for queue_name in self.queues:
629
- pipe.zcard(self._segment_key(TaskResultStatus.READY, queue_name))
630
- pipe.zcard(self._segment_key(TaskResultStatus.RUNNING, queue_name))
631
- pipe.zcard(
632
- self.DEFERRED_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
633
- )
634
- pipe.zcard(self._segment_key(TaskResultStatus.SUCCESSFUL, queue_name))
635
- pipe.zcard(self._segment_key(TaskResultStatus.FAILED, queue_name))
636
- results = await pipe.execute()
623
+ async with self.async_client() as client:
624
+ pipe = client.pipeline()
625
+ for queue_name in self.queues:
626
+ pipe.zcard(self._segment_key(TaskResultStatus.READY, queue_name))
627
+ pipe.zcard(self._segment_key(TaskResultStatus.RUNNING, queue_name))
628
+ pipe.zcard(
629
+ self.DEFERRED_KEY.format(
630
+ prefix=self.key_prefix, queue_name=queue_name
631
+ )
632
+ )
633
+ pipe.zcard(self._segment_key(TaskResultStatus.SUCCESSFUL, queue_name))
634
+ pipe.zcard(self._segment_key(TaskResultStatus.FAILED, queue_name))
635
+ results = await pipe.execute()
637
636
  zero_rates = QueueRates(interval=interval, ingress=0, egress=0)
638
637
  queues: dict[str, QueueStats] = {}
639
638
  for index, queue_name in enumerate(self.queues):
@@ -653,14 +652,13 @@ class RedisTaskBackend(ThreadmillTaskBackend):
653
652
  async def worker_telemetry(
654
653
  self,
655
654
  ) -> collections.abc.AsyncGenerator[TelemetryEvent]:
656
- client = self.async_client
657
- pubsub = client.pubsub()
658
- await pubsub.subscribe(self.telemetry_channel)
659
- pubsub.ignore_subscribe_messages = True
660
- try:
661
- async for message in pubsub.listen():
662
- if (data := message.get("data")) is not None:
663
- direction, _, queue_name = data.partition(":")
655
+ async with self.async_client() as client:
656
+ pubsub = client.pubsub()
657
+ await pubsub.subscribe(self.telemetry_channel)
658
+ pubsub.ignore_subscribe_messages = True
659
+ try:
660
+ async for message in pubsub.listen():
661
+ direction, _, queue_name = message["data"].partition(":")
664
662
  try:
665
663
  event = TelemetryEvent(
666
664
  direction=TelemetryDirection(direction),
@@ -669,11 +667,10 @@ class RedisTaskBackend(ThreadmillTaskBackend):
669
667
  except ValueError:
670
668
  continue
671
669
  yield event
672
- finally:
673
- await pubsub.unsubscribe(self.telemetry_channel)
674
- await pubsub.aclose()
670
+ finally:
671
+ await pubsub.unsubscribe(self.telemetry_channel)
672
+ await pubsub.aclose()
675
673
 
676
674
  def close(self) -> None:
677
675
  """Close the Redis connections."""
678
676
  self.client.close()
679
- self._async_client = None
@@ -0,0 +1,13 @@
1
+ """Custom exceptions for the threadmill task framework."""
2
+
3
+
4
+ class MissingExtraError(ImportError):
5
+ """Raised when an optional dependency extra is not installed."""
6
+
7
+ def __init__(self, extra: str) -> None:
8
+ self.extra = extra
9
+ super().__init__(f"Optional dependency missing. Install: threadmill[{extra}]")
10
+
11
+
12
+ class AcknowledgementTimeoutError(Exception):
13
+ """Raised when a task's lease has expired before it could be acknowledged."""
@@ -1,6 +1,7 @@
1
1
  """Task worker executor implementation."""
2
2
 
3
3
  import asyncio
4
+ import contextlib
4
5
  import dataclasses
5
6
  import datetime
6
7
  import json
@@ -26,6 +27,10 @@ from django.tasks.signals import task_finished, task_started
26
27
  from django.utils import timezone
27
28
  from django.utils.json import normalize_json
28
29
 
30
+ sentry_metrics = None
31
+ with contextlib.suppress(ImportError):
32
+ from sentry_sdk import metrics as sentry_metrics
33
+
29
34
  if typing.TYPE_CHECKING:
30
35
  from .backends.base import Broker, ThreadmillTaskBackend, ThreadmillTaskResult
31
36
 
@@ -128,6 +133,7 @@ class TaskExecutor:
128
133
  prefetch_count: int | None = None
129
134
  poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
130
135
  poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
136
+ sample_interval: datetime.timedelta = datetime.timedelta(seconds=30)
131
137
  shutdown_requested: threading.Event = dataclasses.field(
132
138
  default_factory=threading.Event, init=False
133
139
  )
@@ -183,7 +189,16 @@ class TaskExecutor:
183
189
  self.pool_maintainer = threading.Thread(
184
190
  target=self.maintain_worker_pool, daemon=True
185
191
  )
192
+ self.report_worker_counts()
186
193
  threads = [self.pool_maintainer]
194
+ if sentry_metrics is not None:
195
+ threads.append(
196
+ threading.Thread(
197
+ target=self.report_queue_depth,
198
+ name="threadmill-queue-depth",
199
+ daemon=True,
200
+ )
201
+ )
187
202
  if self.backend.broker_class:
188
203
  self.broker = self.backend.broker_class(self.backend)
189
204
  threads.append(self.broker)
@@ -208,11 +223,14 @@ class TaskExecutor:
208
223
  self.broker.shutdown()
209
224
  with ThreadPoolExecutor(max_workers=self.process_count) as executor:
210
225
  executor.map(lambda worker: worker.shutdown(), self.worker_processes)
226
+ if self.pool_maintainer is not None:
227
+ self.report_worker_counts()
211
228
 
212
229
  def maintain_worker_pool(self) -> None:
213
230
  """Restart worker processes that have exited, until shutdown is requested."""
214
231
  while not self.shutdown_requested.is_set():
215
232
  all_dead = True
233
+ replaced = False
216
234
  for index, worker in enumerate(self.worker_processes):
217
235
  if worker.is_alive():
218
236
  all_dead = False
@@ -227,11 +245,47 @@ class TaskExecutor:
227
245
  logger.warning(
228
246
  "Unable to start worker-%d. Retrying ...", index, exc_info=error
229
247
  )
248
+ replaced = True
249
+ if replaced:
250
+ self.report_worker_counts()
230
251
  if all_dead and self.exit_empty:
231
252
  self.shutdown()
232
253
  return
233
254
  self.shutdown_requested.wait(1)
234
255
 
256
+ def report_worker_counts(self) -> None:
257
+ """Record the live worker process and thread counts."""
258
+ if sentry_metrics is not None:
259
+ process_count = sum(worker.is_alive() for worker in self.worker_processes)
260
+ sentry_metrics.gauge(
261
+ "threadmill.worker.count",
262
+ process_count,
263
+ attributes={"pool": "process"},
264
+ )
265
+ sentry_metrics.gauge(
266
+ "threadmill.worker.count",
267
+ process_count * self.thread_count,
268
+ attributes={"pool": "thread"},
269
+ )
270
+
271
+ def report_queue_depth(self) -> None:
272
+ """Sample queue depth into Sentry until shutdown."""
273
+ if self.sample_interval > datetime.timedelta(0):
274
+ while not self.shutdown_requested.is_set():
275
+ telemetry = asyncio.run(self.backend.queue_stats())
276
+ for queue_name, stats in telemetry.queues.items():
277
+ for status, depth in (
278
+ ("ready", stats.counts.ready),
279
+ ("running", stats.counts.running),
280
+ ("deferred", stats.counts.deferred),
281
+ ):
282
+ sentry_metrics.gauge(
283
+ "threadmill.queue.depth",
284
+ depth,
285
+ attributes={"queue": queue_name, "status": status},
286
+ )
287
+ self.shutdown_requested.wait(self.sample_interval.total_seconds())
288
+
235
289
 
236
290
  class WorkerProcess(multiprocessing.Process):
237
291
  """One worker process with one prefetcher and `thread_count` consumer threads."""
@@ -8,10 +8,10 @@ from django.core.management.base import BaseCommand as DjangoBaseCommand
8
8
  from django.tasks import (
9
9
  DEFAULT_TASK_BACKEND_ALIAS,
10
10
  DEFAULT_TASK_QUEUE_NAME,
11
- InvalidTaskBackend,
12
11
  task_backends,
13
12
  )
14
13
 
14
+ from ...exceptions import MissingExtraError
15
15
  from ...executor import JsonFormatter, TaskExecutor
16
16
 
17
17
 
@@ -22,18 +22,36 @@ def kill_softly(signum, frame):
22
22
  raise KeyboardInterrupt(message)
23
23
 
24
24
 
25
- class WorkerCommand(DjangoBaseCommand):
26
- """Run task workers to process enqueued tasks from the specified backends and queues."""
27
-
28
- help = __doc__
29
-
25
+ class ThreadmillCommand(DjangoBaseCommand):
30
26
  def add_arguments(self, parser):
31
27
  parser.add_argument(
32
28
  "-b",
33
29
  "--backend",
34
30
  default=DEFAULT_TASK_BACKEND_ALIAS,
35
- help="Alias of the tasks backend to use.",
31
+ help="Alias of the tasks backend.",
36
32
  )
33
+
34
+ @staticmethod
35
+ def parse_duration(value: str) -> datetime.timedelta:
36
+ return datetime.timedelta(seconds=float(value))
37
+
38
+ def install_signal_handlers(self) -> None:
39
+ match sys.platform:
40
+ case "win32":
41
+ signal.signal(signal.SIGBREAK, kill_softly)
42
+ case _:
43
+ signal.signal(signal.SIGHUP, kill_softly)
44
+ signal.signal(signal.SIGTERM, kill_softly)
45
+ signal.signal(signal.SIGINT, kill_softly)
46
+
47
+
48
+ class WorkerCommand(ThreadmillCommand):
49
+ """Run task workers to process enqueued tasks from the specified backends and queues."""
50
+
51
+ help = __doc__
52
+
53
+ def add_arguments(self, parser):
54
+ super().add_arguments(parser)
37
55
  parser.add_argument(
38
56
  "-q",
39
57
  "--queues",
@@ -80,16 +98,25 @@ class WorkerCommand(DjangoBaseCommand):
80
98
  )
81
99
  parser.add_argument(
82
100
  "--poll-interval",
83
- type=float,
84
- default=0.01,
101
+ type=self.parse_duration,
102
+ default=datetime.timedelta(seconds=0.01),
85
103
  help="Base wait between idle acquire attempts in seconds.",
86
104
  )
87
105
  parser.add_argument(
88
106
  "--poll-max-interval",
89
- type=float,
90
- default=1,
107
+ type=self.parse_duration,
108
+ default=datetime.timedelta(seconds=1),
91
109
  help="Maximum wait between idle acquire attempts in seconds.",
92
110
  )
111
+ parser.add_argument(
112
+ "--sample-interval",
113
+ type=self.parse_duration,
114
+ default=datetime.timedelta(seconds=30),
115
+ help=(
116
+ "Interval between queue depth samples in seconds."
117
+ " Defaults to 30. Set to 0 to disable queue depth sampling."
118
+ ),
119
+ )
93
120
  parser.add_argument(
94
121
  "--exit-empty",
95
122
  action="store_true",
@@ -116,23 +143,14 @@ class WorkerCommand(DjangoBaseCommand):
116
143
  prefetch_count,
117
144
  poll_interval,
118
145
  poll_max_interval,
146
+ sample_interval,
119
147
  exit_empty,
120
148
  log_format,
121
149
  **options,
122
150
  ):
123
- match sys.platform:
124
- case "win32":
125
- signal.signal(signal.SIGBREAK, kill_softly)
126
- case _:
127
- signal.signal(signal.SIGHUP, kill_softly)
128
- signal.signal(signal.SIGTERM, kill_softly)
129
- signal.signal(signal.SIGINT, kill_softly)
151
+ self.install_signal_handlers()
130
152
  self.stdout.write(self.style.SUCCESS("Starting workers…"))
131
- try:
132
- backend = task_backends[backend]
133
- except InvalidTaskBackend:
134
- message = f"Invalid backend: {backend!r}"
135
- raise CommandError(message)
153
+ backend = task_backends[backend]
136
154
  if _non_queues := set(queues) - set(backend.queues):
137
155
  message = f"Backend does not support all specified queues: {_non_queues!r}"
138
156
  raise CommandError(message)
@@ -158,8 +176,9 @@ class WorkerCommand(DjangoBaseCommand):
158
176
  max_tasks=max_tasks,
159
177
  max_tasks_jitter=max_tasks_jitter,
160
178
  prefetch_count=prefetch_count,
161
- poll_interval=datetime.timedelta(seconds=poll_interval),
162
- poll_max_interval=datetime.timedelta(seconds=poll_max_interval),
179
+ poll_interval=poll_interval,
180
+ poll_max_interval=poll_max_interval,
181
+ sample_interval=sample_interval,
163
182
  exit_empty=exit_empty,
164
183
  queues=queues,
165
184
  log_formatter=log_formatter,
@@ -172,37 +191,17 @@ class WorkerCommand(DjangoBaseCommand):
172
191
  exe.shutdown()
173
192
 
174
193
 
175
- class InspectorCommand(DjangoBaseCommand):
194
+ class InspectorCommand(ThreadmillCommand):
176
195
  """Launch the textual TUI inspector for task backends and queues."""
177
196
 
178
197
  help = __doc__
179
198
 
180
- def add_arguments(self, parser):
181
- parser.add_argument(
182
- "-b",
183
- "--backend",
184
- default=DEFAULT_TASK_BACKEND_ALIAS,
185
- help="Alias of the tasks backend to inspect.",
186
- )
187
-
188
199
  def handle(self, *, backend, **options):
189
200
  try:
190
- from ...backends.base import ThreadmillTaskBackend
191
201
  from ...inspector.app import InspectorApp
192
- except ImportError:
193
- message = "Optional dependency missing. Install: threadmill[inspector]"
194
- raise CommandError(message)
195
- try:
196
- backend = task_backends[backend]
197
- except InvalidTaskBackend as e:
198
- message = f"Invalid backend: {backend!r}"
199
- raise CommandError(message) from e
200
-
201
- if not isinstance(backend, ThreadmillTaskBackend):
202
- message = f"Backend {backend.alias!r} does not support inspection."
203
- raise CommandError(message)
204
-
205
- InspectorApp(backend=backend).run()
202
+ except ImportError as e:
203
+ raise MissingExtraError("inspector") from e
204
+ InspectorApp(backend=task_backends[backend]).run()
206
205
 
207
206
 
208
207
  class Command(BaseCommand):
@@ -223,4 +222,7 @@ class Command(BaseCommand):
223
222
 
224
223
  def handle(self, *args, **options):
225
224
  subcommand_name = options.pop("subcommand")
226
- self.subcommands[subcommand_name]().execute(*args, **options)
225
+ try:
226
+ self.subcommands[subcommand_name]().execute(*args, **options)
227
+ except MissingExtraError as e:
228
+ raise CommandError(str(e)) from e
@@ -0,0 +1,44 @@
1
+ from django.db import close_old_connections
2
+ from django.dispatch import receiver
3
+ from django.tasks.signals import task_enqueued, task_finished, task_started
4
+
5
+ from .backends.base import ThreadmillTaskBackend
6
+ from .executor import TaskExecutor, sentry_metrics
7
+
8
+
9
+ @receiver([task_started, task_finished], sender=TaskExecutor)
10
+ def close_task_database_connection(sender, task_result, **kwargs):
11
+ close_old_connections()
12
+
13
+
14
+ @receiver(task_enqueued)
15
+ def record_task_enqueued(sender, task_result, **kwargs):
16
+ """Record one task entering its queue."""
17
+ if sentry_metrics is not None and issubclass(sender, ThreadmillTaskBackend):
18
+ sentry_metrics.count(
19
+ "threadmill.queue.ingress",
20
+ 1,
21
+ attributes={"queue": task_result.task.queue_name},
22
+ )
23
+
24
+
25
+ @receiver(task_finished, sender=TaskExecutor)
26
+ def record_task_finished(sender, task_result, **kwargs):
27
+ """Emit one egress count and one duration observation per attempt."""
28
+ if sentry_metrics is not None:
29
+ sentry_metrics.count(
30
+ "threadmill.queue.egress",
31
+ 1,
32
+ attributes={"queue": task_result.task.queue_name},
33
+ )
34
+ duration = task_result.finished_at - task_result.last_attempted_at
35
+ sentry_metrics.distribution(
36
+ "threadmill.task.duration",
37
+ duration.total_seconds() * 1e3,
38
+ unit="millisecond",
39
+ attributes={
40
+ "task": task_result.task.module_path,
41
+ "outcome": task_result.status.value.lower(),
42
+ "attempt": task_result.attempts,
43
+ },
44
+ )
@@ -0,0 +1,13 @@
1
+ from django.urls import path
2
+
3
+ from .views import QueueStatsView
4
+
5
+ app_name = "threadmill"
6
+
7
+ urlpatterns = [
8
+ path(
9
+ "queue-stats/",
10
+ QueueStatsView.as_view(),
11
+ name="queue-stats",
12
+ ),
13
+ ]
@@ -0,0 +1,32 @@
1
+ from django.http import Http404, HttpRequest, JsonResponse
2
+ from django.tasks import (
3
+ DEFAULT_TASK_BACKEND_ALIAS,
4
+ InvalidTaskBackend,
5
+ task_backends,
6
+ )
7
+ from django.views import View
8
+
9
+
10
+ class QueueStatsView(View):
11
+ async def get(self, request: HttpRequest) -> JsonResponse:
12
+ try:
13
+ backend = task_backends[
14
+ request.GET.get("backend", DEFAULT_TASK_BACKEND_ALIAS)
15
+ ]
16
+ except InvalidTaskBackend as exception:
17
+ raise Http404 from exception
18
+ try:
19
+ telemetry = await backend.queue_stats()
20
+ except (AttributeError, NotImplementedError) as exception:
21
+ raise Http404 from exception
22
+ try:
23
+ return JsonResponse(
24
+ {
25
+ "ready": sum(
26
+ telemetry.queues[queue_name].counts.ready
27
+ for queue_name in request.GET.getlist("queue", backend.queues)
28
+ )
29
+ }
30
+ )
31
+ except (AttributeError, KeyError) as exception:
32
+ raise Http404 from exception
@@ -1,5 +0,0 @@
1
- """Custom exceptions for the threadmill task framework."""
2
-
3
-
4
- class AcknowledgementTimeoutError(Exception):
5
- """Raised when a task's lease has expired before it could be acknowledged."""
@@ -1,10 +0,0 @@
1
- from django.db import close_old_connections
2
- from django.dispatch import receiver
3
- from django.tasks.signals import task_finished, task_started
4
-
5
- from .executor import TaskExecutor
6
-
7
-
8
- @receiver([task_started, task_finished], sender=TaskExecutor)
9
- def close_task_database_connection(sender, task_result, **kwargs):
10
- close_old_connections()
File without changes