threadmill 0.9.0__tar.gz → 0.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {threadmill-0.9.0 → threadmill-0.10.1}/PKG-INFO +25 -2
- {threadmill-0.9.0 → threadmill-0.10.1}/README.md +22 -1
- {threadmill-0.9.0 → threadmill-0.10.1}/pyproject.toml +2 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/_version.py +3 -3
- threadmill-0.10.1/threadmill/backends/lua/threadmill.lua +170 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/backends/redis.py +408 -118
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/executor.py +54 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/management/commands/threadmill.py +11 -0
- threadmill-0.10.1/threadmill/signals.py +44 -0
- threadmill-0.9.0/threadmill/backends/lua/acknowledge.lua +0 -43
- threadmill-0.9.0/threadmill/backends/lua/acquire.lua +0 -57
- threadmill-0.9.0/threadmill/backends/lua/mover.lua +0 -19
- threadmill-0.9.0/threadmill/backends/lua/reaper.lua +0 -23
- threadmill-0.9.0/threadmill/signals.py +0 -10
- {threadmill-0.9.0 → threadmill-0.10.1}/LICENSE +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/apps.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/backends/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/backends/base.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/exceptions.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/inspector/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/inspector/app.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/inspector/inspector.scss +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/inspector/screens.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/inspector/telemetry.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/management/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/management/commands/__init__.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/retry.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/urls.py +0 -0
- {threadmill-0.9.0 → threadmill-0.10.1}/threadmill/views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: threadmill
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.1
|
|
4
4
|
Summary: The most reliable backend for Django's task framework.
|
|
5
5
|
Keywords: Django,tasks,worker
|
|
6
6
|
Author-email: Johannes Maron <johannes@maron.family>
|
|
@@ -28,6 +28,7 @@ License-File: LICENSE
|
|
|
28
28
|
Requires-Dist: django>=6.1
|
|
29
29
|
Requires-Dist: textual>=8.2.7 ; extra == "inspector"
|
|
30
30
|
Requires-Dist: redis>=5 ; extra == "redis"
|
|
31
|
+
Requires-Dist: sentry-sdk>=2.44 ; extra == "sentry"
|
|
31
32
|
Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
|
|
32
33
|
Project-URL: Documentation, https://github.com/codingjoe/threadmill/
|
|
33
34
|
Project-URL: Funding, https://github.com/sponsors/codingjoe
|
|
@@ -37,6 +38,7 @@ Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/late
|
|
|
37
38
|
Project-URL: Source, https://github.com/codingjoe/threadmill
|
|
38
39
|
Provides-Extra: inspector
|
|
39
40
|
Provides-Extra: redis
|
|
41
|
+
Provides-Extra: sentry
|
|
40
42
|
|
|
41
43
|
<p align="center">
|
|
42
44
|
<picture>
|
|
@@ -171,6 +173,25 @@ uv add "threadmill[inspector]"
|
|
|
171
173
|
uv run manage.py threadmill inspector
|
|
172
174
|
```
|
|
173
175
|
|
|
176
|
+
### Sentry Metrics
|
|
177
|
+
|
|
178
|
+

|
|
179
|
+
|
|
180
|
+
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
181
|
+
is installed. Install the extra:
|
|
182
|
+
|
|
183
|
+
```console
|
|
184
|
+
uv add "threadmill[sentry]"
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
| Metric | Type | Attributes | Meaning |
|
|
188
|
+
| -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
|
|
189
|
+
| `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
|
|
190
|
+
| `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
|
|
191
|
+
| `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
|
|
192
|
+
| `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
|
|
193
|
+
| `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
|
|
194
|
+
|
|
174
195
|
### Autoscaling
|
|
175
196
|
|
|
176
197
|
Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
|
|
@@ -215,7 +236,8 @@ Set `targetValue` to the ready backlog one replica can drain during your scale-u
|
|
|
215
236
|
### Redis Backend Options
|
|
216
237
|
|
|
217
238
|
> [!IMPORTANT]
|
|
218
|
-
> Threadmill requires a persistent
|
|
239
|
+
> Threadmill requires Redis 7 or newer and a persistent configuration without eviction.
|
|
240
|
+
> The backend loads its atomic operations as a Redis Functions library on first use.
|
|
219
241
|
|
|
220
242
|
The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
|
|
221
243
|
`TASKS` configuration:
|
|
@@ -320,4 +342,5 @@ def fetch_github_api(url: str): ...
|
|
|
320
342
|
[](https://github.com/sponsors/codingjoe)
|
|
321
343
|
|
|
322
344
|
[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
|
|
345
|
+
[sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
|
|
323
346
|
|
|
@@ -131,6 +131,25 @@ uv add "threadmill[inspector]"
|
|
|
131
131
|
uv run manage.py threadmill inspector
|
|
132
132
|
```
|
|
133
133
|
|
|
134
|
+
### Sentry Metrics
|
|
135
|
+
|
|
136
|
+

|
|
137
|
+
|
|
138
|
+
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
139
|
+
is installed. Install the extra:
|
|
140
|
+
|
|
141
|
+
```console
|
|
142
|
+
uv add "threadmill[sentry]"
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
| Metric | Type | Attributes | Meaning |
|
|
146
|
+
| -------------------------- | ------------ | ------------------------------------------------- | -------------------------------------------------------- |
|
|
147
|
+
| `threadmill.queue.depth` | gauge | `queue`, `status` (ready, running, deferred) | Tasks in a queue with status ready, running, or deferred |
|
|
148
|
+
| `threadmill.queue.ingress` | counter | `queue` | Tasks that enter a queue |
|
|
149
|
+
| `threadmill.queue.egress` | counter | `queue` | Tasks that finish an attempt |
|
|
150
|
+
| `threadmill.worker.count` | gauge | `pool` (process, thread) | Live worker processes and threads in the pool |
|
|
151
|
+
| `threadmill.task.duration` | distribution | `task`, `outcome` (successful, failed), `attempt` | Duration of one task attempt in milliseconds |
|
|
152
|
+
|
|
134
153
|
### Autoscaling
|
|
135
154
|
|
|
136
155
|
Threadmill exposes queue stats as JSON for KEDA's `metrics-api` scaler.
|
|
@@ -175,7 +194,8 @@ Set `targetValue` to the ready backlog one replica can drain during your scale-u
|
|
|
175
194
|
### Redis Backend Options
|
|
176
195
|
|
|
177
196
|
> [!IMPORTANT]
|
|
178
|
-
> Threadmill requires a persistent
|
|
197
|
+
> Threadmill requires Redis 7 or newer and a persistent configuration without eviction.
|
|
198
|
+
> The backend loads its atomic operations as a Redis Functions library on first use.
|
|
179
199
|
|
|
180
200
|
The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
|
|
181
201
|
`TASKS` configuration:
|
|
@@ -280,3 +300,4 @@ def fetch_github_api(url: str): ...
|
|
|
280
300
|
[](https://github.com/sponsors/codingjoe)
|
|
281
301
|
|
|
282
302
|
[django-tasks]: https://docs.djangoproject.com/en/stable/topics/tasks/
|
|
303
|
+
[sentry-metrics]: https://docs.sentry.io/platforms/python/metrics/
|
|
@@ -37,6 +37,7 @@ optional-dependencies.inspector = [
|
|
|
37
37
|
"textual>=8.2.7",
|
|
38
38
|
]
|
|
39
39
|
optional-dependencies.redis = [ "redis>=5" ]
|
|
40
|
+
optional-dependencies.sentry = [ "sentry-sdk>=2.44" ]
|
|
40
41
|
urls.Changelog = "https://github.com/codingjoe/threadmill/releases"
|
|
41
42
|
urls.Documentation = "https://github.com/codingjoe/threadmill/"
|
|
42
43
|
urls.Funding = "https://github.com/sponsors/codingjoe"
|
|
@@ -62,6 +63,7 @@ test = [
|
|
|
62
63
|
"pytest-cov",
|
|
63
64
|
"pytest-django",
|
|
64
65
|
"redis>=5",
|
|
66
|
+
"sentry-sdk>=2.44",
|
|
65
67
|
]
|
|
66
68
|
|
|
67
69
|
[tool.setuptools_scm]
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.10.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 10, 1)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g418b5f043'
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!lua name=threadmill
|
|
2
|
+
|
|
3
|
+
local TASK_DATA_FIELD = 'data'
|
|
4
|
+
local TASK_SCORE_FIELD = 'score'
|
|
5
|
+
local LEASE_WORKER_FIELD = 'lease_worker'
|
|
6
|
+
local LEASE_STARTED_AT_FIELD = 'lease_started_at'
|
|
7
|
+
local LEASE_TOKEN_FIELD = 'lease_token'
|
|
8
|
+
local STATUS_SUCCESSFUL = 'SUCCESSFUL'
|
|
9
|
+
|
|
10
|
+
local function task_key(prefix, task_id)
|
|
11
|
+
return prefix .. task_id
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
local function acquire_tasks(queue_keys, now_ms, lease_started_at_iso,
|
|
15
|
+
task_key_prefix, queue_count, worker, lease_ttl_ms, start_index, max_count)
|
|
16
|
+
local tasks = {}
|
|
17
|
+
local misses = 0
|
|
18
|
+
local queue_index = start_index % queue_count + 1
|
|
19
|
+
-- joe: one full round without a task ends the scan, so a queue whose head
|
|
20
|
+
-- entry is a ghost yields a short batch; retry the queue if ghosts ever dominate
|
|
21
|
+
while #tasks < max_count and misses < queue_count do
|
|
22
|
+
local popped = redis.call('ZPOPMIN', queue_keys[queue_index * 2])
|
|
23
|
+
local data
|
|
24
|
+
if #popped > 0 then
|
|
25
|
+
local key = task_key(task_key_prefix, popped[1])
|
|
26
|
+
data = redis.call('HGET', key, TASK_DATA_FIELD)
|
|
27
|
+
if data then
|
|
28
|
+
local lease_token = string.format(
|
|
29
|
+
'%06x%06x%06x%06x',
|
|
30
|
+
math.random(0, 0xffffff), math.random(0, 0xffffff),
|
|
31
|
+
math.random(0, 0xffffff), math.random(0, 0xffffff))
|
|
32
|
+
redis.call('ZADD', queue_keys[queue_index * 2 - 1],
|
|
33
|
+
now_ms + lease_ttl_ms, popped[1])
|
|
34
|
+
redis.call('HSET', key,
|
|
35
|
+
LEASE_WORKER_FIELD, worker,
|
|
36
|
+
LEASE_STARTED_AT_FIELD, lease_started_at_iso,
|
|
37
|
+
LEASE_TOKEN_FIELD, lease_token)
|
|
38
|
+
table.insert(tasks, { popped[1], data, lease_token, queue_index - 1 })
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
misses = data and 0 or (misses + 1)
|
|
42
|
+
queue_index = queue_index % queue_count + 1
|
|
43
|
+
end
|
|
44
|
+
return tasks
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
local function acknowledge_task(running_key, result_key, task_data_key,
|
|
48
|
+
successful_results_key, failed_results_key, task_id, serialized_result,
|
|
49
|
+
result_ttl_secs, finished_at_ms, status, telemetry_channel, queue_name,
|
|
50
|
+
lease_token)
|
|
51
|
+
local stored_lease_token = redis.call(
|
|
52
|
+
'HGET', task_data_key, LEASE_TOKEN_FIELD)
|
|
53
|
+
if stored_lease_token and stored_lease_token ~= lease_token then
|
|
54
|
+
return 0
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
if redis.call('ZREM', running_key, task_id) == 0 then
|
|
58
|
+
return 0 -- Task already reaped, skip
|
|
59
|
+
end
|
|
60
|
+
redis.call('SET', result_key, serialized_result, 'EX', result_ttl_secs)
|
|
61
|
+
redis.call('DEL', task_data_key)
|
|
62
|
+
local cutoff_ms = finished_at_ms - result_ttl_secs * 1000
|
|
63
|
+
local results_key = successful_results_key
|
|
64
|
+
if status ~= STATUS_SUCCESSFUL then
|
|
65
|
+
results_key = failed_results_key
|
|
66
|
+
end
|
|
67
|
+
redis.call('ZADD', results_key, finished_at_ms, task_id)
|
|
68
|
+
redis.call('ZREMRANGEBYSCORE', results_key, 0, cutoff_ms)
|
|
69
|
+
redis.call('PUBLISH', telemetry_channel, 'egress:' .. queue_name)
|
|
70
|
+
return 1
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
local function delete_task(segment_key, task_data_key, result_key, task_id)
|
|
74
|
+
local removed = redis.call('ZREM', segment_key, task_id)
|
|
75
|
+
if removed == 1 then
|
|
76
|
+
redis.call('DEL', task_data_key, result_key)
|
|
77
|
+
end
|
|
78
|
+
return removed
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
local function move_due_tasks(deferred_key, ready_key, now_ms,
|
|
82
|
+
task_key_prefix, max_count)
|
|
83
|
+
local due_ids = redis.call('ZRANGEBYSCORE', deferred_key,
|
|
84
|
+
0, now_ms, 'LIMIT', 0, max_count)
|
|
85
|
+
local dropped_ids = {}
|
|
86
|
+
for _, task_id in ipairs(due_ids) do
|
|
87
|
+
redis.call('ZREM', deferred_key, task_id)
|
|
88
|
+
local score = redis.call('HGET',
|
|
89
|
+
task_key(task_key_prefix, task_id), TASK_SCORE_FIELD)
|
|
90
|
+
if score then
|
|
91
|
+
redis.call('ZADD', ready_key, score, task_id)
|
|
92
|
+
else
|
|
93
|
+
table.insert(dropped_ids, task_id)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
return dropped_ids
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
local function purge_queue(ready_key, running_key, deferred_key,
|
|
100
|
+
successful_results_key, failed_results_key, task_key_prefix,
|
|
101
|
+
result_key_prefix)
|
|
102
|
+
local function delete_payloads(set_key, key_prefix)
|
|
103
|
+
local deleted = 0
|
|
104
|
+
for _, id in ipairs(redis.call('ZRANGE', set_key, 0, -1)) do
|
|
105
|
+
deleted = deleted + redis.call('DEL', task_key(key_prefix, id))
|
|
106
|
+
end
|
|
107
|
+
return deleted
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
local deleted = 0
|
|
111
|
+
for _, set_key in ipairs({ready_key, running_key, deferred_key}) do
|
|
112
|
+
deleted = deleted + delete_payloads(set_key, task_key_prefix)
|
|
113
|
+
end
|
|
114
|
+
for _, set_key in ipairs({successful_results_key, failed_results_key}) do
|
|
115
|
+
deleted = deleted + delete_payloads(set_key, result_key_prefix)
|
|
116
|
+
end
|
|
117
|
+
redis.call('DEL', ready_key, running_key, deferred_key,
|
|
118
|
+
successful_results_key, failed_results_key)
|
|
119
|
+
return deleted
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
local function reap_expired_tasks(running_key, claim_ttl_ms, max_count,
|
|
123
|
+
task_key_prefix)
|
|
124
|
+
local clock = redis.call('TIME')
|
|
125
|
+
local now_ms = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
|
|
126
|
+
local claim_deadline_ms = now_ms + claim_ttl_ms
|
|
127
|
+
local expired_ids = redis.call('ZRANGEBYSCORE', running_key,
|
|
128
|
+
0, now_ms, 'LIMIT', 0, max_count)
|
|
129
|
+
local claimed_ids = {}
|
|
130
|
+
for _, task_id in ipairs(expired_ids) do
|
|
131
|
+
if redis.call('EXISTS', task_key(task_key_prefix, task_id)) == 1 then
|
|
132
|
+
redis.call('ZADD', running_key, claim_deadline_ms, task_id)
|
|
133
|
+
table.insert(claimed_ids, task_id)
|
|
134
|
+
else
|
|
135
|
+
redis.call('ZREM', running_key, task_id)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
return claimed_ids
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
redis.register_function('threadmill_acquire', function(keys, args)
|
|
142
|
+
return acquire_tasks(keys, tonumber(args[1]), args[2], args[3],
|
|
143
|
+
tonumber(args[4]), args[5], tonumber(args[6]), tonumber(args[7]),
|
|
144
|
+
tonumber(args[8]))
|
|
145
|
+
end)
|
|
146
|
+
|
|
147
|
+
redis.register_function('threadmill_acknowledge', function(keys, args)
|
|
148
|
+
return acknowledge_task(keys[1], keys[2], keys[3], keys[4], keys[5],
|
|
149
|
+
args[1], args[2], tonumber(args[3]), tonumber(args[4]), args[5], args[6],
|
|
150
|
+
args[7], args[8])
|
|
151
|
+
end)
|
|
152
|
+
|
|
153
|
+
redis.register_function('threadmill_dequeue', function(keys, args)
|
|
154
|
+
return delete_task(keys[1], keys[2], keys[3], args[1])
|
|
155
|
+
end)
|
|
156
|
+
|
|
157
|
+
redis.register_function('threadmill_mover', function(keys, args)
|
|
158
|
+
return move_due_tasks(keys[1], keys[2], tonumber(args[1]), args[2],
|
|
159
|
+
tonumber(args[3]))
|
|
160
|
+
end)
|
|
161
|
+
|
|
162
|
+
redis.register_function('threadmill_purge', function(keys, args)
|
|
163
|
+
return purge_queue(keys[1], keys[2], keys[3], keys[4], keys[5], args[1],
|
|
164
|
+
args[2])
|
|
165
|
+
end)
|
|
166
|
+
|
|
167
|
+
redis.register_function('threadmill_reaper', function(keys, args)
|
|
168
|
+
return reap_expired_tasks(keys[1], tonumber(args[1]), tonumber(args[2]),
|
|
169
|
+
args[3])
|
|
170
|
+
end)
|