threadmill 0.8.0__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {threadmill-0.8.0 → threadmill-0.8.2}/PKG-INFO +13 -12
- {threadmill-0.8.0 → threadmill-0.8.2}/README.md +12 -11
- {threadmill-0.8.0 → threadmill-0.8.2}/pyproject.toml +44 -11
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/_version.py +3 -3
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/base.py +17 -7
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/redis.py +41 -23
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/exceptions.py +1 -1
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/executor.py +12 -3
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/inspector/app.py +8 -12
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/management/commands/threadmill.py +16 -14
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/retry.py +3 -1
- {threadmill-0.8.0 → threadmill-0.8.2}/LICENSE +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/__init__.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/apps.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/__init__.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/lua/acknowledge.lua +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/lua/acquire.lua +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/lua/mover.lua +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/backends/lua/reaper.lua +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/inspector/__init__.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/inspector/inspector.scss +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/inspector/screens.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/inspector/telemetry.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/management/__init__.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/management/commands/__init__.py +0 -0
- {threadmill-0.8.0 → threadmill-0.8.2}/threadmill/signals.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: threadmill
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: The most reliable backend for Django's task framework.
|
|
5
5
|
Keywords: Django,tasks,worker
|
|
6
6
|
Author-email: Johannes Maron <johannes@maron.family>
|
|
@@ -179,17 +179,18 @@ uv run manage.py threadmill inspector
|
|
|
179
179
|
The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
|
|
180
180
|
`TASKS` configuration:
|
|
181
181
|
|
|
182
|
-
| Option
|
|
183
|
-
|
|
|
184
|
-
| `lease_ttl`
|
|
185
|
-
| `result_ttl`
|
|
186
|
-
| `broker_interval`
|
|
187
|
-
| `batch_size`
|
|
188
|
-
| `poll_interval`
|
|
189
|
-
| `poll_max_interval`
|
|
182
|
+
| Option | Default | Description |
|
|
183
|
+
| -------------------------- | ------------------------- | ----------------------------------------------------------------------------------------- |
|
|
184
|
+
| `lease_ttl` | `timedelta(hours=1)` | Max time from acquisition to acknowledgement before the task is retried or marked FAILED. |
|
|
185
|
+
| `result_ttl` | `timedelta(days=1)` | How long task results are retained before automatic removal. |
|
|
186
|
+
| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes. |
|
|
187
|
+
| `batch_size` | `100` | Max tasks to move or reap per broker pass. |
|
|
188
|
+
| `poll_interval` | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll. |
|
|
189
|
+
| `poll_max_interval` | `timedelta(seconds=1)` | Max wait between idle acquire attempts. |
|
|
190
|
+
| `redis_connection_retries` | `3` | Retries per command when its Redis connection fails before the error surfaces. |
|
|
190
191
|
|
|
191
192
|
A task whose lease expired reaches the `retry` callback as an
|
|
192
|
-
`
|
|
193
|
+
`AcknowledgementTimeoutError` error, or is marked FAILED when nothing retries it.
|
|
193
194
|
A claimed task whose stored payload cannot be read any more is dropped with the
|
|
194
195
|
read error logged. A dropped task records no result, so it leaves the inspector
|
|
195
196
|
and cannot be requeued.
|
|
@@ -226,7 +227,7 @@ import datetime
|
|
|
226
227
|
from django.tasks import task
|
|
227
228
|
from requests import HTTPError
|
|
228
229
|
|
|
229
|
-
from threadmill.exceptions import
|
|
230
|
+
from threadmill.exceptions import AcknowledgementTimeoutError
|
|
230
231
|
from threadmill.retry import ExponentialBackoff
|
|
231
232
|
|
|
232
233
|
|
|
@@ -236,7 +237,7 @@ from threadmill.retry import ExponentialBackoff
|
|
|
236
237
|
max_delay=datetime.timedelta(minutes=5),
|
|
237
238
|
factor=2.0,
|
|
238
239
|
max_retries=5,
|
|
239
|
-
expected_exceptions=(HTTPError,
|
|
240
|
+
expected_exceptions=(HTTPError, AcknowledgementTimeoutError),
|
|
240
241
|
)
|
|
241
242
|
)
|
|
242
243
|
def fetch_github_api(url: str): ...
|
|
@@ -139,17 +139,18 @@ uv run manage.py threadmill inspector
|
|
|
139
139
|
The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
|
|
140
140
|
`TASKS` configuration:
|
|
141
141
|
|
|
142
|
-
| Option
|
|
143
|
-
|
|
|
144
|
-
| `lease_ttl`
|
|
145
|
-
| `result_ttl`
|
|
146
|
-
| `broker_interval`
|
|
147
|
-
| `batch_size`
|
|
148
|
-
| `poll_interval`
|
|
149
|
-
| `poll_max_interval`
|
|
142
|
+
| Option | Default | Description |
|
|
143
|
+
| -------------------------- | ------------------------- | ----------------------------------------------------------------------------------------- |
|
|
144
|
+
| `lease_ttl` | `timedelta(hours=1)` | Max time from acquisition to acknowledgement before the task is retried or marked FAILED. |
|
|
145
|
+
| `result_ttl` | `timedelta(days=1)` | How long task results are retained before automatic removal. |
|
|
146
|
+
| `broker_interval` | `timedelta(seconds=1)` | Interval between background broker maintenance passes. |
|
|
147
|
+
| `batch_size` | `100` | Max tasks to move or reap per broker pass. |
|
|
148
|
+
| `poll_interval` | `timedelta(seconds=0.01)` | Base wait between idle acquire attempts, doubled after each empty poll. |
|
|
149
|
+
| `poll_max_interval` | `timedelta(seconds=1)` | Max wait between idle acquire attempts. |
|
|
150
|
+
| `redis_connection_retries` | `3` | Retries per command when its Redis connection fails before the error surfaces. |
|
|
150
151
|
|
|
151
152
|
A task whose lease expired reaches the `retry` callback as an
|
|
152
|
-
`
|
|
153
|
+
`AcknowledgementTimeoutError` error, or is marked FAILED when nothing retries it.
|
|
153
154
|
A claimed task whose stored payload cannot be read any more is dropped with the
|
|
154
155
|
read error logged. A dropped task records no result, so it leaves the inspector
|
|
155
156
|
and cannot be requeued.
|
|
@@ -186,7 +187,7 @@ import datetime
|
|
|
186
187
|
from django.tasks import task
|
|
187
188
|
from requests import HTTPError
|
|
188
189
|
|
|
189
|
-
from threadmill.exceptions import
|
|
190
|
+
from threadmill.exceptions import AcknowledgementTimeoutError
|
|
190
191
|
from threadmill.retry import ExponentialBackoff
|
|
191
192
|
|
|
192
193
|
|
|
@@ -196,7 +197,7 @@ from threadmill.retry import ExponentialBackoff
|
|
|
196
197
|
max_delay=datetime.timedelta(minutes=5),
|
|
197
198
|
factor=2.0,
|
|
198
199
|
max_retries=5,
|
|
199
|
-
expected_exceptions=(HTTPError,
|
|
200
|
+
expected_exceptions=(HTTPError, AcknowledgementTimeoutError),
|
|
200
201
|
)
|
|
201
202
|
)
|
|
202
203
|
def fetch_github_api(url: str): ...
|
|
@@ -72,18 +72,51 @@ module.name = "threadmill"
|
|
|
72
72
|
|
|
73
73
|
[tool.ruff]
|
|
74
74
|
src = [ "benchmarks", "tests", "threadmill" ]
|
|
75
|
-
lint.select = [
|
|
76
|
-
"
|
|
77
|
-
"
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"
|
|
75
|
+
lint.extend-select = [
|
|
76
|
+
"C90",
|
|
77
|
+
"D",
|
|
78
|
+
"D213",
|
|
79
|
+
"DJ",
|
|
80
|
+
"E",
|
|
81
|
+
"G",
|
|
82
|
+
"N",
|
|
83
|
+
"PERF",
|
|
84
|
+
"PT",
|
|
85
|
+
"PTH",
|
|
86
|
+
"RET",
|
|
87
|
+
"S",
|
|
88
|
+
"SIM",
|
|
89
|
+
"SLOT",
|
|
90
|
+
"T20",
|
|
91
|
+
"TRY",
|
|
92
|
+
"UP",
|
|
93
|
+
"W",
|
|
94
|
+
]
|
|
95
|
+
lint.ignore = [ "D1", "D203", "D212", "E501", "RUF01" ]
|
|
96
|
+
lint.per-file-ignores."benchmarks/**" = [
|
|
97
|
+
"DJ006",
|
|
98
|
+
"DJ007",
|
|
99
|
+
"DJ008",
|
|
100
|
+
"PLR2004",
|
|
101
|
+
"S101",
|
|
102
|
+
"S105",
|
|
103
|
+
"S106",
|
|
104
|
+
"S311",
|
|
105
|
+
"S603",
|
|
106
|
+
"S607"
|
|
107
|
+
]
|
|
108
|
+
lint.per-file-ignores."tests/**" = [
|
|
109
|
+
"DJ006",
|
|
110
|
+
"DJ007",
|
|
111
|
+
"DJ008",
|
|
112
|
+
"PLR2004",
|
|
113
|
+
"S101",
|
|
114
|
+
"S105",
|
|
115
|
+
"S106",
|
|
116
|
+
"S311",
|
|
117
|
+
"S603",
|
|
118
|
+
"S607"
|
|
85
119
|
]
|
|
86
|
-
lint.ignore = [ "B904", "D1", "E501", "S101" ]
|
|
87
120
|
lint.isort.combine-as-imports = true
|
|
88
121
|
lint.isort.force-wrap-aliases = true
|
|
89
122
|
lint.isort.known-first-party = [ "threadmill", "tests" ]
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.8.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 8,
|
|
21
|
+
__version__ = version = '0.8.2'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 8, 2)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'gdff213884'
|
|
@@ -22,7 +22,8 @@ logger = logging.getLogger(__name__)
|
|
|
22
22
|
|
|
23
23
|
@dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
|
|
24
24
|
class RetryTask(Task):
|
|
25
|
-
"""
|
|
25
|
+
"""
|
|
26
|
+
Task with an optional retry callback for backoff scheduling.
|
|
26
27
|
|
|
27
28
|
Usage:
|
|
28
29
|
|
|
@@ -47,6 +48,7 @@ class RetryTask(Task):
|
|
|
47
48
|
if hasattr(self.retry, "deconstruct"):
|
|
48
49
|
return self.retry.deconstruct()
|
|
49
50
|
return f"{self.retry.__module__}.{self.retry.__qualname__}"
|
|
51
|
+
return None
|
|
50
52
|
|
|
51
53
|
@classmethod
|
|
52
54
|
def _reconstruct(cls, kwargs):
|
|
@@ -184,12 +186,11 @@ class TaskResultEncoder(DjangoJSONEncoder):
|
|
|
184
186
|
for field in dataclasses.fields(TaskError)
|
|
185
187
|
}
|
|
186
188
|
if isinstance(o, RetryTask):
|
|
187
|
-
|
|
189
|
+
return {
|
|
188
190
|
field.name: getattr(o, field.name)
|
|
189
191
|
for field in dataclasses.fields(RetryTask)
|
|
190
192
|
if field.name not in {"func", "retry"}
|
|
191
193
|
} | {"func": o.module_path, "retry": o.retry_path}
|
|
192
|
-
return data
|
|
193
194
|
return super().default(o)
|
|
194
195
|
|
|
195
196
|
|
|
@@ -231,9 +232,11 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
|
|
|
231
232
|
if task.retry is not None and not (
|
|
232
233
|
is_module_level_function(task.retry) or hasattr(task.retry, "deconstruct")
|
|
233
234
|
):
|
|
234
|
-
|
|
235
|
-
"Task's retry function must be defined at a module level or be a
|
|
235
|
+
message = (
|
|
236
|
+
"Task's retry function must be defined at a module level or be a "
|
|
237
|
+
"deconstructible callable."
|
|
236
238
|
)
|
|
239
|
+
raise InvalidTask(message)
|
|
237
240
|
|
|
238
241
|
@staticmethod
|
|
239
242
|
def create_task_error(exception: BaseException) -> TaskError:
|
|
@@ -254,6 +257,7 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
|
|
|
254
257
|
task_result.id,
|
|
255
258
|
task_result.task.module_path,
|
|
256
259
|
)
|
|
260
|
+
return None
|
|
257
261
|
|
|
258
262
|
def acquire(
|
|
259
263
|
self,
|
|
@@ -290,7 +294,8 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
|
|
|
290
294
|
raise NotImplementedError
|
|
291
295
|
|
|
292
296
|
def requeue(self, task_result: TaskResult, run_after: datetime.datetime) -> None:
|
|
293
|
-
"""
|
|
297
|
+
"""
|
|
298
|
+
Re-queue a failed task result for a retry attempt after `run_after`.
|
|
294
299
|
|
|
295
300
|
Cleans up any persisted failed result so the method works both for
|
|
296
301
|
in-flight retries (task still running) and inspector-driven requeues
|
|
@@ -308,6 +313,10 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
|
|
|
308
313
|
"""
|
|
309
314
|
Yield up to `count` tasks from a queue in the given status segment.
|
|
310
315
|
|
|
316
|
+
Pending work comes back in queue order, so the first READY task is the
|
|
317
|
+
next one a worker picks up. Finished results come back most recently
|
|
318
|
+
finished first.
|
|
319
|
+
|
|
311
320
|
Args:
|
|
312
321
|
queue_name: The name of the queue to peek into.
|
|
313
322
|
status: The status of the tasks to yield.
|
|
@@ -332,7 +341,8 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
|
|
|
332
341
|
raise NotImplementedError
|
|
333
342
|
|
|
334
343
|
def purge(self, queue_name: str) -> None:
|
|
335
|
-
"""
|
|
344
|
+
"""
|
|
345
|
+
Delete every task across all segments of a queue.
|
|
336
346
|
|
|
337
347
|
Args:
|
|
338
348
|
queue_name: The queue to purge.
|
|
@@ -14,6 +14,8 @@ from pathlib import Path
|
|
|
14
14
|
|
|
15
15
|
import redis
|
|
16
16
|
import redis.asyncio
|
|
17
|
+
import redis.asyncio.retry
|
|
18
|
+
import redis.retry
|
|
17
19
|
from django.tasks import DEFAULT_TASK_QUEUE_NAME, TaskResult, TaskResultStatus
|
|
18
20
|
from django.tasks.exceptions import TaskResultDoesNotExist
|
|
19
21
|
from django.tasks.signals import task_enqueued
|
|
@@ -30,7 +32,7 @@ from threadmill.backends.base import (
|
|
|
30
32
|
ThreadmillTaskBackend,
|
|
31
33
|
ThreadmillTaskResult,
|
|
32
34
|
)
|
|
33
|
-
from threadmill.exceptions import
|
|
35
|
+
from threadmill.exceptions import AcknowledgementTimeoutError
|
|
34
36
|
|
|
35
37
|
logger = logging.getLogger(__name__)
|
|
36
38
|
|
|
@@ -52,7 +54,8 @@ def _parse_lease_started_at(value: str | None) -> datetime.datetime | None:
|
|
|
52
54
|
|
|
53
55
|
@dataclasses.dataclass(kw_only=True, slots=True)
|
|
54
56
|
class _IdleBackoff:
|
|
55
|
-
"""
|
|
57
|
+
"""
|
|
58
|
+
Idle polling state for a single worker thread.
|
|
56
59
|
|
|
57
60
|
Every worker thread backs off and rotates independently, so one thread's
|
|
58
61
|
misses never reset or double another thread's delay. Private, so it is not
|
|
@@ -139,7 +142,7 @@ class RedisBroker(Broker):
|
|
|
139
142
|
errors=[
|
|
140
143
|
*task_result.errors,
|
|
141
144
|
self.backend.create_task_error(
|
|
142
|
-
|
|
145
|
+
AcknowledgementTimeoutError("Task processing lease expired.")
|
|
143
146
|
),
|
|
144
147
|
],
|
|
145
148
|
)
|
|
@@ -159,17 +162,18 @@ class RedisBroker(Broker):
|
|
|
159
162
|
for queue_name in self.backend.queues:
|
|
160
163
|
try:
|
|
161
164
|
self._move_queue(queue_name)
|
|
162
|
-
except Exception:
|
|
165
|
+
except Exception:
|
|
163
166
|
logger.exception("Mover error for queue %r", queue_name)
|
|
164
167
|
|
|
165
168
|
try:
|
|
166
169
|
self._reap_running_queue(queue_name)
|
|
167
|
-
except Exception:
|
|
170
|
+
except Exception:
|
|
168
171
|
logger.exception("Running reaper error for queue %r", queue_name)
|
|
169
172
|
|
|
170
173
|
|
|
171
174
|
class RedisTaskBackend(ThreadmillTaskBackend):
|
|
172
|
-
"""
|
|
175
|
+
"""
|
|
176
|
+
Redis-backed durable priority queue backend.
|
|
173
177
|
|
|
174
178
|
Uses sorted sets for priority ordering, a running set for in-flight
|
|
175
179
|
tracking, and a deferred set for scheduled tasks. All multi-step operations
|
|
@@ -225,10 +229,19 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
225
229
|
try:
|
|
226
230
|
redis_url = params["REDIS_URL"]
|
|
227
231
|
except KeyError as e:
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
+
message = (
|
|
233
|
+
"REDIS_URL must be specified in your settings for the "
|
|
234
|
+
f"{type(self).__name__}."
|
|
235
|
+
)
|
|
236
|
+
raise ValueError(message) from e
|
|
237
|
+
self.redis_connection_retries = self.options.get("redis_connection_retries", 3)
|
|
238
|
+
self.client = redis.from_url(
|
|
239
|
+
redis_url,
|
|
240
|
+
decode_responses=True,
|
|
241
|
+
retry=redis.retry.Retry(
|
|
242
|
+
redis.default_backoff(), self.redis_connection_retries
|
|
243
|
+
),
|
|
244
|
+
)
|
|
232
245
|
self._async_client: redis.asyncio.Redis | None = None
|
|
233
246
|
self.redis_url = redis_url
|
|
234
247
|
self.key_prefix = f"threadmill:{{{alias}}}"
|
|
@@ -262,12 +275,17 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
262
275
|
"""Lazily-created async Redis client, reused across calls."""
|
|
263
276
|
if self._async_client is None:
|
|
264
277
|
self._async_client = redis.asyncio.Redis.from_url(
|
|
265
|
-
self.redis_url,
|
|
278
|
+
self.redis_url,
|
|
279
|
+
decode_responses=True,
|
|
280
|
+
retry=redis.asyncio.retry.Retry(
|
|
281
|
+
redis.default_backoff(), self.redis_connection_retries
|
|
282
|
+
),
|
|
266
283
|
)
|
|
267
284
|
return self._async_client
|
|
268
285
|
|
|
269
286
|
def _compute_score(self, priority: int, enqueued_at: datetime.datetime) -> float:
|
|
270
|
-
"""
|
|
287
|
+
"""
|
|
288
|
+
Compute a ZSET score for priority-ordered FIFO queueing.
|
|
271
289
|
|
|
272
290
|
Higher priority (more positive) tasks are popped first. Within the same
|
|
273
291
|
priority, earlier enqueued tasks are popped first.
|
|
@@ -281,7 +299,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
281
299
|
args: Sequence | None = None,
|
|
282
300
|
kwargs: dict | None = None,
|
|
283
301
|
) -> TaskResult:
|
|
284
|
-
"""
|
|
302
|
+
"""
|
|
303
|
+
Enqueue a task for execution.
|
|
285
304
|
|
|
286
305
|
If the task has a run_after datetime, it is stored in the deferred set
|
|
287
306
|
instead of the active priority queue.
|
|
@@ -391,9 +410,11 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
391
410
|
try:
|
|
392
411
|
remaining = deadline - time.monotonic()
|
|
393
412
|
except TypeError:
|
|
394
|
-
|
|
413
|
+
message = "No task available."
|
|
414
|
+
raise queue.Empty(message)
|
|
395
415
|
if remaining <= 0:
|
|
396
|
-
|
|
416
|
+
message = "No task available within the specified timeout."
|
|
417
|
+
raise TimeoutError(message)
|
|
397
418
|
# Stop doubling once the interval reaches poll_max_interval; larger
|
|
398
419
|
# exponents would only overflow the float math.
|
|
399
420
|
cap = int(self.poll_max_interval / self.poll_interval).bit_length()
|
|
@@ -414,7 +435,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
414
435
|
lease_started_at: datetime.datetime | None,
|
|
415
436
|
lease_token: str | None,
|
|
416
437
|
) -> ThreadmillTaskResult:
|
|
417
|
-
"""
|
|
438
|
+
"""
|
|
439
|
+
Return a stored task result as a running attempt.
|
|
418
440
|
|
|
419
441
|
`worker=None` means the lease records no worker; an empty string still
|
|
420
442
|
counts as an attempt, and a task that already records a start keeps it.
|
|
@@ -556,10 +578,6 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
556
578
|
def _peek_tasks(
|
|
557
579
|
self, zset_key: str, count: int, *, leased: bool
|
|
558
580
|
) -> Generator[TaskResult]:
|
|
559
|
-
"""Yield up to `count` stored tasks in queue order.
|
|
560
|
-
|
|
561
|
-
Leased tasks are yielded as their lease reports them.
|
|
562
|
-
"""
|
|
563
581
|
pipe = self.client.pipeline()
|
|
564
582
|
for task_id in self.client.zrange(zset_key, 0, count - 1):
|
|
565
583
|
task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_id)
|
|
@@ -584,9 +602,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
584
602
|
yield self.deserialize_task_result(stored)
|
|
585
603
|
|
|
586
604
|
def _peek_results(self, zset_key: str, count: int) -> Generator[TaskResult]:
|
|
587
|
-
"""Yield up to `count` finished results in finish order."""
|
|
588
605
|
pipe = self.client.pipeline()
|
|
589
|
-
for result_id in self.client.zrange(zset_key, 0, count - 1):
|
|
606
|
+
for result_id in self.client.zrange(zset_key, 0, count - 1, desc=True):
|
|
590
607
|
result_key = self.RESULT_KEY.format(
|
|
591
608
|
prefix=self.key_prefix, result_id=result_id
|
|
592
609
|
)
|
|
@@ -600,7 +617,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
600
617
|
self.RESULT_KEY.format(prefix=self.key_prefix, result_id=result_id)
|
|
601
618
|
):
|
|
602
619
|
return self.deserialize_task_result(data)
|
|
603
|
-
|
|
620
|
+
message = f"Task result {result_id!r} does not exist."
|
|
621
|
+
raise TaskResultDoesNotExist(message)
|
|
604
622
|
|
|
605
623
|
async def queue_stats(
|
|
606
624
|
self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
|
|
@@ -101,7 +101,8 @@ def configure_logging(formatter: logging.Formatter) -> None:
|
|
|
101
101
|
|
|
102
102
|
|
|
103
103
|
def warn_when_free_threading_is_unavailable() -> None:
|
|
104
|
-
"""
|
|
104
|
+
"""
|
|
105
|
+
Warn when a free-threaded build runs with the GIL enabled.
|
|
105
106
|
|
|
106
107
|
A C extension that has not declared free-threading support re-enables the GIL
|
|
107
108
|
for the whole process, which silently costs the parallelism the build exists
|
|
@@ -150,6 +151,7 @@ class TaskExecutor:
|
|
|
150
151
|
if self.max_tasks:
|
|
151
152
|
jitter = random.randint(0, self.max_tasks_jitter) # noqa: S311
|
|
152
153
|
return max((self.max_tasks + jitter) // self.thread_count, 1)
|
|
154
|
+
return None
|
|
153
155
|
|
|
154
156
|
def create_worker_process(self) -> WorkerProcess:
|
|
155
157
|
"""Create and start a new worker process."""
|
|
@@ -206,7 +208,13 @@ class TaskExecutor:
|
|
|
206
208
|
worker.join(timeout=0)
|
|
207
209
|
if self.exit_empty:
|
|
208
210
|
continue
|
|
209
|
-
|
|
211
|
+
try:
|
|
212
|
+
self.worker_processes[index] = self.create_worker_process()
|
|
213
|
+
except OSError as error:
|
|
214
|
+
# Keep the dead worker and retry on the next tick.
|
|
215
|
+
logger.warning(
|
|
216
|
+
"Unable to start worker-%d. Retrying ...", index, exc_info=error
|
|
217
|
+
)
|
|
210
218
|
if all_dead and self.exit_empty:
|
|
211
219
|
self.shutdown()
|
|
212
220
|
return
|
|
@@ -325,7 +333,8 @@ class TaskPrefetcher(threading.Thread):
|
|
|
325
333
|
def run(self) -> None:
|
|
326
334
|
try:
|
|
327
335
|
self.fill_buffer()
|
|
328
|
-
|
|
336
|
+
# Prefetcher failures must reach the consumer, and a thread cannot raise.
|
|
337
|
+
except Exception as exception: # noqa: BLE001
|
|
329
338
|
self.completion.set_exception(exception)
|
|
330
339
|
finally:
|
|
331
340
|
if not self.completion.done():
|
|
@@ -163,8 +163,7 @@ class TaskDetail(Static):
|
|
|
163
163
|
lines.append("[b]Errors:[/b]")
|
|
164
164
|
for error in result.errors:
|
|
165
165
|
lines.append(f" {error.exception_class_path}")
|
|
166
|
-
for line in error.traceback.splitlines()
|
|
167
|
-
lines.append(f" {line}")
|
|
166
|
+
lines.extend(f" {line}" for line in error.traceback.splitlines())
|
|
168
167
|
return "\n".join(lines)
|
|
169
168
|
|
|
170
169
|
|
|
@@ -291,7 +290,7 @@ class TaskList(Vertical):
|
|
|
291
290
|
if confirmed:
|
|
292
291
|
try:
|
|
293
292
|
self.backend.requeue(task, datetime.datetime.now(datetime.UTC))
|
|
294
|
-
except Exception:
|
|
293
|
+
except Exception:
|
|
295
294
|
logger.exception("Requeue failed for task %s", task.id)
|
|
296
295
|
self.app.notify("Requeue failed.", severity="error")
|
|
297
296
|
else:
|
|
@@ -313,7 +312,7 @@ class TaskList(Vertical):
|
|
|
313
312
|
if confirmed:
|
|
314
313
|
try:
|
|
315
314
|
self.backend.dequeue(task)
|
|
316
|
-
except Exception:
|
|
315
|
+
except Exception:
|
|
317
316
|
logger.exception("Delete failed for task %s", task.id)
|
|
318
317
|
self.app.notify("Delete failed.", severity="error")
|
|
319
318
|
else:
|
|
@@ -341,7 +340,7 @@ class TaskList(Vertical):
|
|
|
341
340
|
self._current_results = list(
|
|
342
341
|
backend.peek(queue_name=queue_name, status=status, count=100)
|
|
343
342
|
)
|
|
344
|
-
except Exception:
|
|
343
|
+
except Exception:
|
|
345
344
|
logger.exception("Failed to peek tasks for %r", queue_name)
|
|
346
345
|
self._current_results = []
|
|
347
346
|
columns = TAB_COLUMNS[tab_id]
|
|
@@ -435,7 +434,7 @@ class QueueList(ListView):
|
|
|
435
434
|
if confirmed:
|
|
436
435
|
try:
|
|
437
436
|
self.app.backend.purge(queue_name)
|
|
438
|
-
except Exception:
|
|
437
|
+
except Exception:
|
|
439
438
|
logger.exception("Purge failed for %r", queue_name)
|
|
440
439
|
self.app.notify("Purge failed.", severity="error")
|
|
441
440
|
else:
|
|
@@ -577,7 +576,7 @@ class InspectorApp(App):
|
|
|
577
576
|
"""Fetch queue counts and merge live pub/sub rates into the telemetry."""
|
|
578
577
|
try:
|
|
579
578
|
telemetry = await self.backend.queue_stats()
|
|
580
|
-
except Exception:
|
|
579
|
+
except Exception:
|
|
581
580
|
logger.exception("Failed to refresh queue stats")
|
|
582
581
|
else:
|
|
583
582
|
for queue_name, stats in telemetry.queues.items():
|
|
@@ -617,8 +616,5 @@ class InspectorApp(App):
|
|
|
617
616
|
pass
|
|
618
617
|
else:
|
|
619
618
|
if stream is not None:
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
self.telemetry_buffer.record(event, now=time.monotonic())
|
|
623
|
-
except asyncio.CancelledError:
|
|
624
|
-
raise
|
|
619
|
+
async for event in stream:
|
|
620
|
+
self.telemetry_buffer.record(event, now=time.monotonic())
|
|
@@ -18,7 +18,8 @@ from ...executor import JsonFormatter, TaskExecutor
|
|
|
18
18
|
def kill_softly(signum, frame):
|
|
19
19
|
"""Raise a KeyboardInterrupt to stop the worker gracefully."""
|
|
20
20
|
signame = signal.Signals(signum).name
|
|
21
|
-
|
|
21
|
+
message = f"Received {signame} ({signum}), shutting down…"
|
|
22
|
+
raise KeyboardInterrupt(message)
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
class WorkerCommand(DjangoBaseCommand):
|
|
@@ -130,11 +131,11 @@ class WorkerCommand(DjangoBaseCommand):
|
|
|
130
131
|
try:
|
|
131
132
|
backend = task_backends[backend]
|
|
132
133
|
except InvalidTaskBackend:
|
|
133
|
-
|
|
134
|
+
message = f"Invalid backend: {backend!r}"
|
|
135
|
+
raise CommandError(message)
|
|
134
136
|
if _non_queues := set(queues) - set(backend.queues):
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
)
|
|
137
|
+
message = f"Backend does not support all specified queues: {_non_queues!r}"
|
|
138
|
+
raise CommandError(message)
|
|
138
139
|
try:
|
|
139
140
|
log_formatter = (
|
|
140
141
|
JsonFormatter() if log_format is None else logging.Formatter(log_format)
|
|
@@ -145,9 +146,11 @@ class WorkerCommand(DjangoBaseCommand):
|
|
|
145
146
|
)
|
|
146
147
|
)
|
|
147
148
|
except (TypeError, ValueError) as e:
|
|
148
|
-
|
|
149
|
+
message = f"Invalid log format: {log_format!r}"
|
|
150
|
+
raise CommandError(message) from e
|
|
149
151
|
if prefetch_count is not None and prefetch_count < 0:
|
|
150
|
-
|
|
152
|
+
message = f"Invalid prefetch count: {prefetch_count!r}"
|
|
153
|
+
raise CommandError(message)
|
|
151
154
|
exe = TaskExecutor(
|
|
152
155
|
backend=backend,
|
|
153
156
|
workers=workers,
|
|
@@ -187,18 +190,17 @@ class InspectorCommand(DjangoBaseCommand):
|
|
|
187
190
|
from ...backends.base import ThreadmillTaskBackend
|
|
188
191
|
from ...inspector.app import InspectorApp
|
|
189
192
|
except ImportError:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
)
|
|
193
|
+
message = "Optional dependency missing. Install: threadmill[inspector]"
|
|
194
|
+
raise CommandError(message)
|
|
193
195
|
try:
|
|
194
196
|
backend = task_backends[backend]
|
|
195
197
|
except InvalidTaskBackend as e:
|
|
196
|
-
|
|
198
|
+
message = f"Invalid backend: {backend!r}"
|
|
199
|
+
raise CommandError(message) from e
|
|
197
200
|
|
|
198
201
|
if not isinstance(backend, ThreadmillTaskBackend):
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
)
|
|
202
|
+
message = f"Backend {backend.alias!r} does not support inspection."
|
|
203
|
+
raise CommandError(message)
|
|
202
204
|
|
|
203
205
|
InspectorApp(backend=backend).run()
|
|
204
206
|
|
|
@@ -10,7 +10,8 @@ if typing.TYPE_CHECKING:
|
|
|
10
10
|
|
|
11
11
|
@dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
|
|
12
12
|
class ExponentialBackoff:
|
|
13
|
-
"""
|
|
13
|
+
"""
|
|
14
|
+
Delay retries exponentially, capped at ``max_delay`` and limited to ``max_retries`` attempts.
|
|
14
15
|
|
|
15
16
|
Only retry exceptions listed in ``expected_exceptions``.
|
|
16
17
|
|
|
@@ -57,6 +58,7 @@ class ExponentialBackoff:
|
|
|
57
58
|
self.max_delay.total_seconds(),
|
|
58
59
|
)
|
|
59
60
|
return datetime.timedelta(seconds=delay_secs)
|
|
61
|
+
return None
|
|
60
62
|
|
|
61
63
|
def deconstruct(self):
|
|
62
64
|
return (
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|