threadmill 0.9.0__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {threadmill-0.9.0 → threadmill-0.10.0}/PKG-INFO +21 -1
  2. {threadmill-0.9.0 → threadmill-0.10.0}/README.md +18 -0
  3. {threadmill-0.9.0 → threadmill-0.10.0}/pyproject.toml +2 -0
  4. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/_version.py +3 -3
  5. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/redis.py +2 -0
  6. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/executor.py +54 -0
  7. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/management/commands/threadmill.py +11 -0
  8. threadmill-0.10.0/threadmill/signals.py +44 -0
  9. threadmill-0.9.0/threadmill/signals.py +0 -10
  10. {threadmill-0.9.0 → threadmill-0.10.0}/LICENSE +0 -0
  11. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/__init__.py +0 -0
  12. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/apps.py +0 -0
  13. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/__init__.py +0 -0
  14. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/base.py +0 -0
  15. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/acknowledge.lua +0 -0
  16. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/acquire.lua +0 -0
  17. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/mover.lua +0 -0
  18. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/backends/lua/reaper.lua +0 -0
  19. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/exceptions.py +0 -0
  20. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/__init__.py +0 -0
  21. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/app.py +0 -0
  22. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/inspector.scss +0 -0
  23. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/screens.py +0 -0
  24. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/inspector/telemetry.py +0 -0
  25. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/management/__init__.py +0 -0
  26. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/management/commands/__init__.py +0 -0
  27. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/retry.py +0 -0
  28. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/urls.py +0 -0
  29. {threadmill-0.9.0 → threadmill-0.10.0}/threadmill/views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: threadmill
3
- Version: 0.9.0
3
+ Version: 0.10.0
4
4
  Summary: The most reliable backend for Django's task framework.
5
5
  Keywords: Django,tasks,worker
6
6
  Author-email: Johannes Maron <johannes@maron.family>
@@ -28,6 +28,7 @@ License-File: LICENSE
28
28
  Requires-Dist: django>=6.1
29
29
  Requires-Dist: textual>=8.2.7 ; extra == "inspector"
30
30
  Requires-Dist: redis>=5 ; extra == "redis"
31
+ Requires-Dist: sentry-sdk>=2.44 ; extra == "sentry"
31
32
  Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
32
33
  Project-URL: Documentation, https://github.com/codingjoe/threadmill/
33
34
  Project-URL: Funding, https://github.com/sponsors/codingjoe
@@ -37,6 +38,7 @@ Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/late
37
38
  Project-URL: Source, https://github.com/codingjoe/threadmill
38
39
  Provides-Extra: inspector
39
40
  Provides-Extra: redis
41
+ Provides-Extra: sentry
40
42
 
41
43
  <p align="center">
42
44
  <picture>
@@ -171,6 +173,23 @@ uv add "threadmill[inspector]"
171
173
  uv run manage.py threadmill inspector
172
174
  ```
173
175
 
176
+ ### Sentry Metrics
177
+
178
+ Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
179
+ is installed. Install the extra:
180
+
181
+ ```console
182
+ uv add "threadmill[sentry]"
183
+ ```
184
+
185
+ | Metric | Type | Attributes | Meaning |
186
+ | -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
187
+ | `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
188
+ | `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
189
+ | `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
190
+ | `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
191
+ | `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
192
+
174
193
  ### Autoscaling
175
194
 
176
195
  Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
@@ -320,4 +339,5 @@ def fetch_github_api(url: str): ...
320
339
  [![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
321
340
 
322
341
  [django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
342
+ [sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
323
343
 
@@ -131,6 +131,23 @@ uv add "threadmill[inspector]"
131
131
  uv run manage.py threadmill inspector
132
132
  ```
133
133
 
134
+ ### Sentry Metrics
135
+
136
+ Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
137
+ is installed. Install the extra:
138
+
139
+ ```console
140
+ uv add "threadmill[sentry]"
141
+ ```
142
+
143
+ | Metric | Type | Attributes | Meaning |
144
+ | -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
145
+ | `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
146
+ | `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
147
+ | `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
148
+ | `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
149
+ | `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
150
+
134
151
  ### Autoscaling
135
152
 
136
153
  Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
@@ -280,3 +297,4 @@ def fetch_github_api(url: str): ...
280
297
  [![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
281
298
 
282
299
  [django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
300
+ [sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
@@ -37,6 +37,7 @@ optional-dependencies.inspector = [
37
37
  "textual>=8.2.7",
38
38
  ]
39
39
  optional-dependencies.redis = [ "redis>=5" ]
40
+ optional-dependencies.sentry = [ "sentry-sdk>=2.44" ]
40
41
  urls.Changelog = "https://github.com/codingjoe/threadmill/releases"
41
42
  urls.Documentation = "https://github.com/codingjoe/threadmill/"
42
43
  urls.Funding = "https://github.com/sponsors/codingjoe"
@@ -62,6 +63,7 @@ test = [
62
63
  "pytest-cov",
63
64
  "pytest-django",
64
65
  "redis>=5",
66
+ "sentry-sdk>=2.44",
65
67
  ]
66
68
 
67
69
  [tool.setuptools_scm]
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.9.0'
22
- __version_tuple__ = version_tuple = (0, 9, 0)
21
+ __version__ = version = '0.10.0'
22
+ __version_tuple__ = version_tuple = (0, 10, 0)
23
23
 
24
- __commit_id__ = commit_id = 'gf1d9fc31e'
24
+ __commit_id__ = commit_id = 'ga458039fb'
@@ -538,6 +538,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
538
538
  pipe.publish(self.telemetry_channel, f"ingress:{task_result.task.queue_name}")
539
539
  pipe.execute()
540
540
 
541
+ task_enqueued.send(self.__class__, task_result=task_result)
542
+
541
543
  def dequeue(self, task_result: TaskResult) -> None:
542
544
  self.client.zrem(
543
545
  self._segment_key(task_result.status, task_result.task.queue_name),
@@ -1,6 +1,7 @@
1
1
  """Task worker executor implementation."""
2
2
 
3
3
  import asyncio
4
+ import contextlib
4
5
  import dataclasses
5
6
  import datetime
6
7
  import json
@@ -26,6 +27,10 @@ from django.tasks.signals import task_finished, task_started
26
27
  from django.utils import timezone
27
28
  from django.utils.json import normalize_json
28
29
 
30
+ sentry_metrics = None
31
+ with contextlib.suppress(ImportError):
32
+ from sentry_sdk import metrics as sentry_metrics
33
+
29
34
  if typing.TYPE_CHECKING:
30
35
  from .backends.base import Broker, ThreadmillTaskBackend, ThreadmillTaskResult
31
36
 
@@ -128,6 +133,7 @@ class TaskExecutor:
128
133
  prefetch_count: int | None = None
129
134
  poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
130
135
  poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
136
+ sample_interval: datetime.timedelta = datetime.timedelta(seconds=30)
131
137
  shutdown_requested: threading.Event = dataclasses.field(
132
138
  default_factory=threading.Event, init=False
133
139
  )
@@ -183,7 +189,16 @@ class TaskExecutor:
183
189
  self.pool_maintainer = threading.Thread(
184
190
  target=self.maintain_worker_pool, daemon=True
185
191
  )
192
+ self.report_worker_counts()
186
193
  threads = [self.pool_maintainer]
194
+ if sentry_metrics is not None:
195
+ threads.append(
196
+ threading.Thread(
197
+ target=self.report_queue_depth,
198
+ name="threadmill-queue-depth",
199
+ daemon=True,
200
+ )
201
+ )
187
202
  if self.backend.broker_class:
188
203
  self.broker = self.backend.broker_class(self.backend)
189
204
  threads.append(self.broker)
@@ -208,11 +223,14 @@ class TaskExecutor:
208
223
  self.broker.shutdown()
209
224
  with ThreadPoolExecutor(max_workers=self.process_count) as executor:
210
225
  executor.map(lambda worker: worker.shutdown(), self.worker_processes)
226
+ if self.pool_maintainer is not None:
227
+ self.report_worker_counts()
211
228
 
212
229
  def maintain_worker_pool(self) -> None:
213
230
  """Restart worker processes that have exited, until shutdown is requested."""
214
231
  while not self.shutdown_requested.is_set():
215
232
  all_dead = True
233
+ replaced = False
216
234
  for index, worker in enumerate(self.worker_processes):
217
235
  if worker.is_alive():
218
236
  all_dead = False
@@ -227,11 +245,47 @@ class TaskExecutor:
227
245
  logger.warning(
228
246
  "Unable to start worker-%d. Retrying ...", index, exc_info=error
229
247
  )
248
+ replaced = True
249
+ if replaced:
250
+ self.report_worker_counts()
230
251
  if all_dead and self.exit_empty:
231
252
  self.shutdown()
232
253
  return
233
254
  self.shutdown_requested.wait(1)
234
255
 
256
+ def report_worker_counts(self) -> None:
257
+ """Record the live worker process and thread counts."""
258
+ if sentry_metrics is not None:
259
+ process_count = sum(worker.is_alive() for worker in self.worker_processes)
260
+ sentry_metrics.gauge(
261
+ "threadmill.worker.count",
262
+ process_count,
263
+ attributes={"pool": "process"},
264
+ )
265
+ sentry_metrics.gauge(
266
+ "threadmill.worker.count",
267
+ process_count * self.thread_count,
268
+ attributes={"pool": "thread"},
269
+ )
270
+
271
+ def report_queue_depth(self) -> None:
272
+ """Sample queue depth into Sentry until shutdown."""
273
+ if self.sample_interval > datetime.timedelta(0):
274
+ while not self.shutdown_requested.is_set():
275
+ telemetry = asyncio.run(self.backend.queue_stats())
276
+ for queue_name, stats in telemetry.queues.items():
277
+ for status, depth in (
278
+ ("ready", stats.counts.ready),
279
+ ("running", stats.counts.running),
280
+ ("deferred", stats.counts.deferred),
281
+ ):
282
+ sentry_metrics.gauge(
283
+ "threadmill.queue.depth",
284
+ depth,
285
+ attributes={"queue": queue_name, "status": status},
286
+ )
287
+ self.shutdown_requested.wait(self.sample_interval.total_seconds())
288
+
235
289
 
236
290
  class WorkerProcess(multiprocessing.Process):
237
291
  """One worker process with one prefetcher and `thread_count` consumer threads."""
@@ -108,6 +108,15 @@ class WorkerCommand(ThreadmillCommand):
108
108
  default=datetime.timedelta(seconds=1),
109
109
  help="Maximum wait between idle acquire attempts in seconds.",
110
110
  )
111
+ parser.add_argument(
112
+ "--sample-interval",
113
+ type=self.parse_duration,
114
+ default=datetime.timedelta(seconds=30),
115
+ help=(
116
+ "Interval between queue depth samples in seconds."
117
+ " Defaults to 30. Set to 0 to disable queue depth sampling."
118
+ ),
119
+ )
111
120
  parser.add_argument(
112
121
  "--exit-empty",
113
122
  action="store_true",
@@ -134,6 +143,7 @@ class WorkerCommand(ThreadmillCommand):
134
143
  prefetch_count,
135
144
  poll_interval,
136
145
  poll_max_interval,
146
+ sample_interval,
137
147
  exit_empty,
138
148
  log_format,
139
149
  **options,
@@ -168,6 +178,7 @@ class WorkerCommand(ThreadmillCommand):
168
178
  prefetch_count=prefetch_count,
169
179
  poll_interval=poll_interval,
170
180
  poll_max_interval=poll_max_interval,
181
+ sample_interval=sample_interval,
171
182
  exit_empty=exit_empty,
172
183
  queues=queues,
173
184
  log_formatter=log_formatter,
@@ -0,0 +1,44 @@
1
+ from django.db import close_old_connections
2
+ from django.dispatch import receiver
3
+ from django.tasks.signals import task_enqueued, task_finished, task_started
4
+
5
+ from .backends.base import ThreadmillTaskBackend
6
+ from .executor import TaskExecutor, sentry_metrics
7
+
8
+
9
+ @receiver([task_started, task_finished], sender=TaskExecutor)
10
+ def close_task_database_connection(sender, task_result, **kwargs):
11
+ close_old_connections()
12
+
13
+
14
+ @receiver(task_enqueued)
15
+ def record_task_enqueued(sender, task_result, **kwargs):
16
+ """Record one task entering its queue."""
17
+ if sentry_metrics is not None and issubclass(sender, ThreadmillTaskBackend):
18
+ sentry_metrics.count(
19
+ "threadmill.queue.ingress",
20
+ 1,
21
+ attributes={"queue": task_result.task.queue_name},
22
+ )
23
+
24
+
25
+ @receiver(task_finished, sender=TaskExecutor)
26
+ def record_task_finished(sender, task_result, **kwargs):
27
+ """Emit one egress count and one duration observation per attempt."""
28
+ if sentry_metrics is not None:
29
+ sentry_metrics.count(
30
+ "threadmill.queue.egress",
31
+ 1,
32
+ attributes={"queue": task_result.task.queue_name},
33
+ )
34
+ duration = task_result.finished_at - task_result.last_attempted_at
35
+ sentry_metrics.distribution(
36
+ "threadmill.task.duration",
37
+ duration.total_seconds() * 1e3,
38
+ unit="millisecond",
39
+ attributes={
40
+ "task": task_result.task.module_path,
41
+ "outcome": task_result.status.value.lower(),
42
+ "attempt": task_result.attempts,
43
+ },
44
+ )
@@ -1,10 +0,0 @@
1
- from django.db import close_old_connections
2
- from django.dispatch import receiver
3
- from django.tasks.signals import task_finished, task_started
4
-
5
- from .executor import TaskExecutor
6
-
7
-
8
- @receiver([task_started, task_finished], sender=TaskExecutor)
9
- def close_task_database_connection(sender, task_result, **kwargs):
10
- close_old_connections()
File without changes