threadmill 0.10.0__tar.gz → 0.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {threadmill-0.10.0 → threadmill-0.10.1}/PKG-INFO +5 -2
- {threadmill-0.10.0 → threadmill-0.10.1}/README.md +4 -1
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/_version.py +3 -3
- threadmill-0.10.1/threadmill/backends/lua/threadmill.lua +170 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/backends/redis.py +406 -118
- threadmill-0.10.0/threadmill/backends/lua/acknowledge.lua +0 -43
- threadmill-0.10.0/threadmill/backends/lua/acquire.lua +0 -57
- threadmill-0.10.0/threadmill/backends/lua/mover.lua +0 -19
- threadmill-0.10.0/threadmill/backends/lua/reaper.lua +0 -23
- {threadmill-0.10.0 → threadmill-0.10.1}/LICENSE +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/pyproject.toml +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/__init__.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/apps.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/backends/__init__.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/backends/base.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/exceptions.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/executor.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/inspector/__init__.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/inspector/app.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/inspector/inspector.scss +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/inspector/screens.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/inspector/telemetry.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/management/__init__.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/management/commands/__init__.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/management/commands/threadmill.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/retry.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/signals.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/urls.py +0 -0
- {threadmill-0.10.0 → threadmill-0.10.1}/threadmill/views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: threadmill
|
|
3
|
-
Version: 0.10.
|
|
3
|
+
Version: 0.10.1
|
|
4
4
|
Summary: The most reliable backend for Django's task framework.
|
|
5
5
|
Keywords: Django,tasks,worker
|
|
6
6
|
Author-email: Johannes Maron <johannes@maron.family>
|
|
@@ -175,6 +175,8 @@ uv run manage.py threadmill inspector
|
|
|
175
175
|
|
|
176
176
|
### Sentry Metrics
|
|
177
177
|
|
|
178
|
+

|
|
179
|
+
|
|
178
180
|
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
179
181
|
is installed. Install the extra:
|
|
180
182
|
|
|
@@ -234,7 +236,8 @@ Set `targetValue` to the ready backlog one replica can drain during your scale-u
|
|
|
234
236
|
### Redis Backend Options
|
|
235
237
|
|
|
236
238
|
> [!IMPORTANT]
|
|
237
|
-
> Threadmill requires a persistent
|
|
239
|
+
> Threadmill requires Redis 7 or newer and a persistent configuration without eviction.
|
|
240
|
+
> The backend loads its atomic operations as a Redis Functions library on first use.
|
|
238
241
|
|
|
239
242
|
The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
|
|
240
243
|
`TASKS` configuration:
|
|
@@ -133,6 +133,8 @@ uv run manage.py threadmill inspector
|
|
|
133
133
|
|
|
134
134
|
### Sentry Metrics
|
|
135
135
|
|
|
136
|
+

|
|
137
|
+
|
|
136
138
|
Threadmill emits [Sentry application metrics][sentry-metrics] when `sentry-sdk`
|
|
137
139
|
is installed. Install the extra:
|
|
138
140
|
|
|
@@ -192,7 +194,8 @@ Set `targetValue` to the ready backlog one replica can drain during your scale-u
|
|
|
192
194
|
### Redis Backend Options
|
|
193
195
|
|
|
194
196
|
> [!IMPORTANT]
|
|
195
|
-
> Threadmill requires a persistent
|
|
197
|
+
> Threadmill requires Redis 7 or newer and a persistent configuration without eviction.
|
|
198
|
+
> The backend loads its atomic operations as a Redis Functions library on first use.
|
|
196
199
|
|
|
197
200
|
The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
|
|
198
201
|
`TASKS` configuration:
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.10.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 10,
|
|
21
|
+
__version__ = version = '0.10.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 10, 1)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g418b5f043'
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!lua name=threadmill
|
|
2
|
+
|
|
3
|
+
local TASK_DATA_FIELD = 'data'
|
|
4
|
+
local TASK_SCORE_FIELD = 'score'
|
|
5
|
+
local LEASE_WORKER_FIELD = 'lease_worker'
|
|
6
|
+
local LEASE_STARTED_AT_FIELD = 'lease_started_at'
|
|
7
|
+
local LEASE_TOKEN_FIELD = 'lease_token'
|
|
8
|
+
local STATUS_SUCCESSFUL = 'SUCCESSFUL'
|
|
9
|
+
|
|
10
|
+
local function task_key(prefix, task_id)
|
|
11
|
+
return prefix .. task_id
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
local function acquire_tasks(queue_keys, now_ms, lease_started_at_iso,
|
|
15
|
+
task_key_prefix, queue_count, worker, lease_ttl_ms, start_index, max_count)
|
|
16
|
+
local tasks = {}
|
|
17
|
+
local misses = 0
|
|
18
|
+
local queue_index = start_index % queue_count + 1
|
|
19
|
+
-- joe: one full round without a task ends the scan, so a queue whose head
|
|
20
|
+
-- entry is a ghost yields a short batch; retry the queue if ghosts ever dominate
|
|
21
|
+
while #tasks < max_count and misses < queue_count do
|
|
22
|
+
local popped = redis.call('ZPOPMIN', queue_keys[queue_index * 2])
|
|
23
|
+
local data
|
|
24
|
+
if #popped > 0 then
|
|
25
|
+
local key = task_key(task_key_prefix, popped[1])
|
|
26
|
+
data = redis.call('HGET', key, TASK_DATA_FIELD)
|
|
27
|
+
if data then
|
|
28
|
+
local lease_token = string.format(
|
|
29
|
+
'%06x%06x%06x%06x',
|
|
30
|
+
math.random(0, 0xffffff), math.random(0, 0xffffff),
|
|
31
|
+
math.random(0, 0xffffff), math.random(0, 0xffffff))
|
|
32
|
+
redis.call('ZADD', queue_keys[queue_index * 2 - 1],
|
|
33
|
+
now_ms + lease_ttl_ms, popped[1])
|
|
34
|
+
redis.call('HSET', key,
|
|
35
|
+
LEASE_WORKER_FIELD, worker,
|
|
36
|
+
LEASE_STARTED_AT_FIELD, lease_started_at_iso,
|
|
37
|
+
LEASE_TOKEN_FIELD, lease_token)
|
|
38
|
+
table.insert(tasks, { popped[1], data, lease_token, queue_index - 1 })
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
misses = data and 0 or (misses + 1)
|
|
42
|
+
queue_index = queue_index % queue_count + 1
|
|
43
|
+
end
|
|
44
|
+
return tasks
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
local function acknowledge_task(running_key, result_key, task_data_key,
|
|
48
|
+
successful_results_key, failed_results_key, task_id, serialized_result,
|
|
49
|
+
result_ttl_secs, finished_at_ms, status, telemetry_channel, queue_name,
|
|
50
|
+
lease_token)
|
|
51
|
+
local stored_lease_token = redis.call(
|
|
52
|
+
'HGET', task_data_key, LEASE_TOKEN_FIELD)
|
|
53
|
+
if stored_lease_token and stored_lease_token ~= lease_token then
|
|
54
|
+
return 0
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
if redis.call('ZREM', running_key, task_id) == 0 then
|
|
58
|
+
return 0 -- Task already reaped, skip
|
|
59
|
+
end
|
|
60
|
+
redis.call('SET', result_key, serialized_result, 'EX', result_ttl_secs)
|
|
61
|
+
redis.call('DEL', task_data_key)
|
|
62
|
+
local cutoff_ms = finished_at_ms - result_ttl_secs * 1000
|
|
63
|
+
local results_key = successful_results_key
|
|
64
|
+
if status ~= STATUS_SUCCESSFUL then
|
|
65
|
+
results_key = failed_results_key
|
|
66
|
+
end
|
|
67
|
+
redis.call('ZADD', results_key, finished_at_ms, task_id)
|
|
68
|
+
redis.call('ZREMRANGEBYSCORE', results_key, 0, cutoff_ms)
|
|
69
|
+
redis.call('PUBLISH', telemetry_channel, 'egress:' .. queue_name)
|
|
70
|
+
return 1
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
local function delete_task(segment_key, task_data_key, result_key, task_id)
|
|
74
|
+
local removed = redis.call('ZREM', segment_key, task_id)
|
|
75
|
+
if removed == 1 then
|
|
76
|
+
redis.call('DEL', task_data_key, result_key)
|
|
77
|
+
end
|
|
78
|
+
return removed
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
local function move_due_tasks(deferred_key, ready_key, now_ms,
|
|
82
|
+
task_key_prefix, max_count)
|
|
83
|
+
local due_ids = redis.call('ZRANGEBYSCORE', deferred_key,
|
|
84
|
+
0, now_ms, 'LIMIT', 0, max_count)
|
|
85
|
+
local dropped_ids = {}
|
|
86
|
+
for _, task_id in ipairs(due_ids) do
|
|
87
|
+
redis.call('ZREM', deferred_key, task_id)
|
|
88
|
+
local score = redis.call('HGET',
|
|
89
|
+
task_key(task_key_prefix, task_id), TASK_SCORE_FIELD)
|
|
90
|
+
if score then
|
|
91
|
+
redis.call('ZADD', ready_key, score, task_id)
|
|
92
|
+
else
|
|
93
|
+
table.insert(dropped_ids, task_id)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
return dropped_ids
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
local function purge_queue(ready_key, running_key, deferred_key,
|
|
100
|
+
successful_results_key, failed_results_key, task_key_prefix,
|
|
101
|
+
result_key_prefix)
|
|
102
|
+
local function delete_payloads(set_key, key_prefix)
|
|
103
|
+
local deleted = 0
|
|
104
|
+
for _, id in ipairs(redis.call('ZRANGE', set_key, 0, -1)) do
|
|
105
|
+
deleted = deleted + redis.call('DEL', task_key(key_prefix, id))
|
|
106
|
+
end
|
|
107
|
+
return deleted
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
local deleted = 0
|
|
111
|
+
for _, set_key in ipairs({ready_key, running_key, deferred_key}) do
|
|
112
|
+
deleted = deleted + delete_payloads(set_key, task_key_prefix)
|
|
113
|
+
end
|
|
114
|
+
for _, set_key in ipairs({successful_results_key, failed_results_key}) do
|
|
115
|
+
deleted = deleted + delete_payloads(set_key, result_key_prefix)
|
|
116
|
+
end
|
|
117
|
+
redis.call('DEL', ready_key, running_key, deferred_key,
|
|
118
|
+
successful_results_key, failed_results_key)
|
|
119
|
+
return deleted
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
local function reap_expired_tasks(running_key, claim_ttl_ms, max_count,
|
|
123
|
+
task_key_prefix)
|
|
124
|
+
local clock = redis.call('TIME')
|
|
125
|
+
local now_ms = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
|
|
126
|
+
local claim_deadline_ms = now_ms + claim_ttl_ms
|
|
127
|
+
local expired_ids = redis.call('ZRANGEBYSCORE', running_key,
|
|
128
|
+
0, now_ms, 'LIMIT', 0, max_count)
|
|
129
|
+
local claimed_ids = {}
|
|
130
|
+
for _, task_id in ipairs(expired_ids) do
|
|
131
|
+
if redis.call('EXISTS', task_key(task_key_prefix, task_id)) == 1 then
|
|
132
|
+
redis.call('ZADD', running_key, claim_deadline_ms, task_id)
|
|
133
|
+
table.insert(claimed_ids, task_id)
|
|
134
|
+
else
|
|
135
|
+
redis.call('ZREM', running_key, task_id)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
return claimed_ids
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
redis.register_function('threadmill_acquire', function(keys, args)
|
|
142
|
+
return acquire_tasks(keys, tonumber(args[1]), args[2], args[3],
|
|
143
|
+
tonumber(args[4]), args[5], tonumber(args[6]), tonumber(args[7]),
|
|
144
|
+
tonumber(args[8]))
|
|
145
|
+
end)
|
|
146
|
+
|
|
147
|
+
redis.register_function('threadmill_acknowledge', function(keys, args)
|
|
148
|
+
return acknowledge_task(keys[1], keys[2], keys[3], keys[4], keys[5],
|
|
149
|
+
args[1], args[2], tonumber(args[3]), tonumber(args[4]), args[5], args[6],
|
|
150
|
+
args[7], args[8])
|
|
151
|
+
end)
|
|
152
|
+
|
|
153
|
+
redis.register_function('threadmill_dequeue', function(keys, args)
|
|
154
|
+
return delete_task(keys[1], keys[2], keys[3], args[1])
|
|
155
|
+
end)
|
|
156
|
+
|
|
157
|
+
redis.register_function('threadmill_mover', function(keys, args)
|
|
158
|
+
return move_due_tasks(keys[1], keys[2], tonumber(args[1]), args[2],
|
|
159
|
+
tonumber(args[3]))
|
|
160
|
+
end)
|
|
161
|
+
|
|
162
|
+
redis.register_function('threadmill_purge', function(keys, args)
|
|
163
|
+
return purge_queue(keys[1], keys[2], keys[3], keys[4], keys[5], args[1],
|
|
164
|
+
args[2])
|
|
165
|
+
end)
|
|
166
|
+
|
|
167
|
+
redis.register_function('threadmill_reaper', function(keys, args)
|
|
168
|
+
return reap_expired_tasks(keys[1], tonumber(args[1]), tonumber(args[2]),
|
|
169
|
+
args[3])
|
|
170
|
+
end)
|
|
@@ -9,8 +9,9 @@ import random
|
|
|
9
9
|
import threading
|
|
10
10
|
import time
|
|
11
11
|
import uuid
|
|
12
|
-
from collections.abc import Generator, Sequence
|
|
12
|
+
from collections.abc import Awaitable, Generator, Sequence
|
|
13
13
|
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
14
15
|
|
|
15
16
|
import redis
|
|
16
17
|
import redis.asyncio
|
|
@@ -36,12 +37,270 @@ from threadmill.exceptions import AcknowledgementTimeoutError
|
|
|
36
37
|
|
|
37
38
|
logger = logging.getLogger(__name__)
|
|
38
39
|
|
|
39
|
-
_LUA_DIR = Path(__file__).resolve().parent / "lua"
|
|
40
40
|
|
|
41
|
+
class ThreadmillFunctionCommands:
|
|
42
|
+
"""
|
|
43
|
+
Functions of the threadmill Redis Functions library.
|
|
44
|
+
|
|
45
|
+
The constants mirror the locals declared at the top of the Lua library.
|
|
46
|
+
The methods are shared by the sync and async clients, so on an async client
|
|
47
|
+
they return an awaitable instead of a result.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
LIBRARY = "threadmill"
|
|
51
|
+
DATA = "data"
|
|
52
|
+
SCORE = "score"
|
|
53
|
+
LEASE_WORKER = "lease_worker"
|
|
54
|
+
LEASE_STARTED_AT = "lease_started_at"
|
|
55
|
+
LEASE_TOKEN = "lease_token" # noqa: S105
|
|
56
|
+
LEASE_FIELDS = (LEASE_WORKER, LEASE_STARTED_AT, LEASE_TOKEN)
|
|
57
|
+
SOURCE = (Path(__file__).resolve().parent / "lua" / "threadmill.lua").read_text()
|
|
58
|
+
|
|
59
|
+
def __init__(self, *args, **kwargs) -> None:
|
|
60
|
+
super().__init__(*args, **kwargs)
|
|
61
|
+
self._functions_loaded = False
|
|
62
|
+
|
|
63
|
+
def acquire(
|
|
64
|
+
self,
|
|
65
|
+
*,
|
|
66
|
+
queue_keys: Sequence[str],
|
|
67
|
+
now_ms: float,
|
|
68
|
+
lease_started_at_iso: str,
|
|
69
|
+
task_key_prefix: str,
|
|
70
|
+
worker: str,
|
|
71
|
+
lease_ttl_ms: int,
|
|
72
|
+
start_index: int,
|
|
73
|
+
max_count: int,
|
|
74
|
+
) -> list[list[str | int]] | Awaitable[list[list[str | int]]]:
|
|
75
|
+
"""
|
|
76
|
+
Lease up to `max_count` ready tasks, one round-robin pop per queue.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
queue_keys: Interleaved running and ready set keys, one pair per queue.
|
|
80
|
+
now_ms: Current time in milliseconds since the UNIX epoch.
|
|
81
|
+
lease_started_at_iso: Current time as an ISO 8601 string, stamped
|
|
82
|
+
as the lease start.
|
|
83
|
+
task_key_prefix: Prefix of the stored task hashes.
|
|
84
|
+
worker: Worker name stamped as the lease holder.
|
|
85
|
+
lease_ttl_ms: Lease duration in milliseconds.
|
|
86
|
+
start_index: Index of the queue pair to scan first.
|
|
87
|
+
max_count: Maximum number of tasks to lease.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
`[task_id, payload, lease_token, queue_index]` tuples, empty when
|
|
91
|
+
no queue yields a task.
|
|
92
|
+
"""
|
|
93
|
+
return self._call(
|
|
94
|
+
"acquire",
|
|
95
|
+
queue_keys,
|
|
96
|
+
[
|
|
97
|
+
str(now_ms),
|
|
98
|
+
lease_started_at_iso,
|
|
99
|
+
task_key_prefix,
|
|
100
|
+
str(len(queue_keys) // 2),
|
|
101
|
+
worker,
|
|
102
|
+
str(lease_ttl_ms),
|
|
103
|
+
str(start_index),
|
|
104
|
+
str(max_count),
|
|
105
|
+
],
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def acknowledge(
|
|
109
|
+
self,
|
|
110
|
+
*,
|
|
111
|
+
running_key: str,
|
|
112
|
+
result_key: str,
|
|
113
|
+
task_data_key: str,
|
|
114
|
+
successful_results_key: str,
|
|
115
|
+
failed_results_key: str,
|
|
116
|
+
task_id: str,
|
|
117
|
+
serialized_result: str,
|
|
118
|
+
result_ttl_secs: int,
|
|
119
|
+
finished_at_ms: float,
|
|
120
|
+
status: str,
|
|
121
|
+
telemetry_channel: str,
|
|
122
|
+
queue_name: str,
|
|
123
|
+
lease_token: str,
|
|
124
|
+
) -> int | Awaitable[int]:
|
|
125
|
+
"""
|
|
126
|
+
Finalize a running task.
|
|
127
|
+
|
|
128
|
+
Removes the running entry, stores the result for `result_ttl_secs`,
|
|
129
|
+
deletes the task hash, records the result in its status history, and
|
|
130
|
+
publishes an egress event.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
running_key: Running set of the task's queue.
|
|
134
|
+
result_key: Result key that receives the serialized result.
|
|
135
|
+
task_data_key: Task hash that holds the payload and lease.
|
|
136
|
+
successful_results_key: Result history for successful tasks.
|
|
137
|
+
failed_results_key: Result history for failed tasks.
|
|
138
|
+
task_id: Task identifier.
|
|
139
|
+
serialized_result: Serialized task result to store.
|
|
140
|
+
result_ttl_secs: Result retention in seconds.
|
|
141
|
+
finished_at_ms: Finish time in milliseconds since the UNIX epoch.
|
|
142
|
+
status: Status name of the finished task.
|
|
143
|
+
telemetry_channel: Pub/sub channel for the egress event.
|
|
144
|
+
queue_name: Queue that the task belongs to.
|
|
145
|
+
lease_token: Token of the lease holder, or an empty string.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
1 when the task was finalized, 0 when it was no longer running.
|
|
149
|
+
"""
|
|
150
|
+
return self._call(
|
|
151
|
+
"acknowledge",
|
|
152
|
+
[
|
|
153
|
+
running_key,
|
|
154
|
+
result_key,
|
|
155
|
+
task_data_key,
|
|
156
|
+
successful_results_key,
|
|
157
|
+
failed_results_key,
|
|
158
|
+
],
|
|
159
|
+
[
|
|
160
|
+
task_id,
|
|
161
|
+
serialized_result,
|
|
162
|
+
str(result_ttl_secs),
|
|
163
|
+
str(finished_at_ms),
|
|
164
|
+
status,
|
|
165
|
+
telemetry_channel,
|
|
166
|
+
queue_name,
|
|
167
|
+
lease_token,
|
|
168
|
+
],
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
def dequeue(
|
|
172
|
+
self, *, segment_key: str, task_data_key: str, result_key: str, task_id: str
|
|
173
|
+
) -> int | Awaitable[int]:
|
|
174
|
+
"""
|
|
175
|
+
Delete a task from its segment together with its payload.
|
|
176
|
+
|
|
177
|
+
The payload is kept when the segment entry is already gone, so a late
|
|
178
|
+
result or a requeued task survives.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
segment_key: Status segment that holds the task ID.
|
|
182
|
+
task_data_key: Task hash to delete.
|
|
183
|
+
result_key: Result key to delete.
|
|
184
|
+
task_id: Task identifier.
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
1 when the segment entry existed, 0 otherwise.
|
|
188
|
+
"""
|
|
189
|
+
return self._call(
|
|
190
|
+
"dequeue", [segment_key, task_data_key, result_key], [task_id]
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
def mover(
|
|
194
|
+
self,
|
|
195
|
+
*,
|
|
196
|
+
deferred_key: str,
|
|
197
|
+
ready_key: str,
|
|
198
|
+
now_ms: float,
|
|
199
|
+
task_key_prefix: str,
|
|
200
|
+
max_count: int,
|
|
201
|
+
) -> list[str] | Awaitable[list[str]]:
|
|
202
|
+
"""
|
|
203
|
+
Move due deferred tasks to the ready queue.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
deferred_key: Deferred set scored by scheduled time.
|
|
207
|
+
ready_key: Ready queue that receives the due tasks.
|
|
208
|
+
now_ms: Current time in milliseconds since the UNIX epoch.
|
|
209
|
+
task_key_prefix: Prefix of the stored task hashes.
|
|
210
|
+
max_count: Maximum number of tasks to move.
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
IDs dropped because their stored payload is missing.
|
|
214
|
+
"""
|
|
215
|
+
return self._call(
|
|
216
|
+
"mover",
|
|
217
|
+
[deferred_key, ready_key],
|
|
218
|
+
[str(now_ms), task_key_prefix, str(max_count)],
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
def purge(
|
|
222
|
+
self,
|
|
223
|
+
*,
|
|
224
|
+
ready_key: str,
|
|
225
|
+
running_key: str,
|
|
226
|
+
deferred_key: str,
|
|
227
|
+
successful_results_key: str,
|
|
228
|
+
failed_results_key: str,
|
|
229
|
+
task_key_prefix: str,
|
|
230
|
+
result_key_prefix: str,
|
|
231
|
+
) -> int | Awaitable[int]:
|
|
232
|
+
"""
|
|
233
|
+
Delete every task and payload of one queue.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
ready_key: Ready set of the queue.
|
|
237
|
+
running_key: Running set of the queue.
|
|
238
|
+
deferred_key: Deferred set of the queue.
|
|
239
|
+
successful_results_key: Successful result history of the queue.
|
|
240
|
+
failed_results_key: Failed result history of the queue.
|
|
241
|
+
task_key_prefix: Prefix of the stored task hashes.
|
|
242
|
+
result_key_prefix: Prefix of the stored results.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
Number of deleted payloads.
|
|
246
|
+
"""
|
|
247
|
+
return self._call(
|
|
248
|
+
"purge",
|
|
249
|
+
[
|
|
250
|
+
ready_key,
|
|
251
|
+
running_key,
|
|
252
|
+
deferred_key,
|
|
253
|
+
successful_results_key,
|
|
254
|
+
failed_results_key,
|
|
255
|
+
],
|
|
256
|
+
[task_key_prefix, result_key_prefix],
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
def reaper(
|
|
260
|
+
self,
|
|
261
|
+
*,
|
|
262
|
+
running_key: str,
|
|
263
|
+
claim_ttl_ms: int,
|
|
264
|
+
max_count: int,
|
|
265
|
+
task_key_prefix: str,
|
|
266
|
+
) -> list[str] | Awaitable[list[str]]:
|
|
267
|
+
"""
|
|
268
|
+
Claim tasks whose lease expired.
|
|
269
|
+
|
|
270
|
+
Args:
|
|
271
|
+
running_key: Running set scored by lease deadline.
|
|
272
|
+
claim_ttl_ms: How long a claim is held, in milliseconds.
|
|
273
|
+
max_count: Maximum number of tasks to claim.
|
|
274
|
+
task_key_prefix: Prefix of the stored task hashes.
|
|
275
|
+
|
|
276
|
+
Returns:
|
|
277
|
+
IDs of the claimed tasks.
|
|
278
|
+
"""
|
|
279
|
+
return self._call(
|
|
280
|
+
"reaper",
|
|
281
|
+
[running_key],
|
|
282
|
+
[str(claim_ttl_ms), str(max_count), task_key_prefix],
|
|
283
|
+
)
|
|
41
284
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
285
|
+
|
|
286
|
+
class ThreadmillRedis(ThreadmillFunctionCommands, redis.Redis):
|
|
287
|
+
"""Synchronous Redis client with the threadmill function commands."""
|
|
288
|
+
|
|
289
|
+
def _call(self, name: str, keys: Sequence[str], args: Sequence[str]) -> Any:
|
|
290
|
+
if not self._functions_loaded:
|
|
291
|
+
self.function_load(self.SOURCE, replace=True)
|
|
292
|
+
self._functions_loaded = True
|
|
293
|
+
return self.fcall(f"{self.LIBRARY}_{name}", len(keys), *keys, *args)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class ThreadmillAsyncRedis(ThreadmillFunctionCommands, redis.asyncio.Redis):
|
|
297
|
+
"""Asynchronous Redis client with the threadmill function commands."""
|
|
298
|
+
|
|
299
|
+
async def _call(self, name: str, keys: Sequence[str], args: Sequence[str]) -> Any:
|
|
300
|
+
if not self._functions_loaded:
|
|
301
|
+
await self.function_load(self.SOURCE, replace=True)
|
|
302
|
+
self._functions_loaded = True
|
|
303
|
+
return await self.fcall(f"{self.LIBRARY}_{name}", len(keys), *keys, *args)
|
|
45
304
|
|
|
46
305
|
|
|
47
306
|
def _parse_lease_started_at(value: str | None) -> datetime.datetime | None:
|
|
@@ -72,19 +331,12 @@ class RedisBroker(Broker):
|
|
|
72
331
|
|
|
73
332
|
backend: RedisTaskBackend
|
|
74
333
|
|
|
75
|
-
MOVER_SCRIPT = _load_lua("mover")
|
|
76
|
-
"""Move tasks whose scheduled time has passed from the deferred to the active queue."""
|
|
77
|
-
REAPER_SCRIPT = _load_lua("reaper")
|
|
78
|
-
"""Claim tasks whose processing lease has expired for the broker to decide their fate."""
|
|
79
|
-
|
|
80
334
|
CLAIM_TTL = datetime.timedelta(seconds=60)
|
|
81
335
|
"""How long a reaper claim is held before another broker pass may take it over."""
|
|
82
336
|
|
|
83
337
|
def __init__(self, backend: RedisTaskBackend) -> None:
|
|
84
338
|
interval = backend.options.get("broker_interval", datetime.timedelta(seconds=1))
|
|
85
339
|
super().__init__(backend, interval=interval)
|
|
86
|
-
self._mover_script = self.backend.client.register_script(self.MOVER_SCRIPT)
|
|
87
|
-
self._reaper_script = self.backend.client.register_script(self.REAPER_SCRIPT)
|
|
88
340
|
|
|
89
341
|
def _move_queue(self, queue_name: str) -> None:
|
|
90
342
|
"""Move due deferred tasks from a single deferred set."""
|
|
@@ -92,43 +344,39 @@ class RedisBroker(Broker):
|
|
|
92
344
|
prefix=self.backend.key_prefix, queue_name=queue_name
|
|
93
345
|
)
|
|
94
346
|
queue_key = self.backend._segment_key(TaskResultStatus.READY, queue_name)
|
|
95
|
-
self.
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
347
|
+
if dropped := self.backend.client.mover(
|
|
348
|
+
deferred_key=deferred_key,
|
|
349
|
+
ready_key=queue_key,
|
|
350
|
+
now_ms=time.time() * 1000,
|
|
351
|
+
task_key_prefix=self.backend.task_key_prefix,
|
|
352
|
+
max_count=self.backend.batch_size,
|
|
353
|
+
):
|
|
354
|
+
logger.warning(
|
|
355
|
+
"Dropped %d deferred task(s) of queue %r without a stored payload: %s",
|
|
356
|
+
len(dropped),
|
|
357
|
+
queue_name,
|
|
358
|
+
", ".join(dropped),
|
|
359
|
+
)
|
|
103
360
|
|
|
104
361
|
def _reap_running_queue(self, queue_name: str) -> None:
|
|
105
362
|
"""Claim tasks whose processing lease has expired from the running set."""
|
|
106
363
|
running_key = self.backend._segment_key(TaskResultStatus.RUNNING, queue_name)
|
|
107
|
-
claimed_ids = self.
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
f"{self.backend.key_prefix}:task:",
|
|
113
|
-
],
|
|
364
|
+
claimed_ids = self.backend.client.reaper(
|
|
365
|
+
running_key=running_key,
|
|
366
|
+
claim_ttl_ms=int(self.CLAIM_TTL.total_seconds() * 1000),
|
|
367
|
+
max_count=self.backend.batch_size,
|
|
368
|
+
task_key_prefix=self.backend.task_key_prefix,
|
|
114
369
|
)
|
|
115
370
|
for task_id in claimed_ids:
|
|
116
|
-
self._reap_task(task_id)
|
|
371
|
+
self._reap_task(task_id, queue_name)
|
|
117
372
|
|
|
118
|
-
def _reap_task(self, task_id: str) -> None:
|
|
119
|
-
"""Requeue or fail it, and
|
|
373
|
+
def _reap_task(self, task_id: str, queue_name: str) -> None:
|
|
374
|
+
"""Requeue or fail it, and remove it when the payload cannot be read."""
|
|
120
375
|
try:
|
|
121
376
|
task_result = self.backend.get_leased_task(task_id)
|
|
122
|
-
except ImportError, ValueError
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
)
|
|
126
|
-
# Drop only the payload: the running entry stays for a lease holder's
|
|
127
|
-
# late acknowledge, and the reaper script sweeps it once the hash is gone.
|
|
128
|
-
self.backend.client.delete(
|
|
129
|
-
self.backend.TASK_KEY.format(
|
|
130
|
-
prefix=self.backend.key_prefix, task_id=task_id
|
|
131
|
-
)
|
|
377
|
+
except (ImportError, ValueError) as exception:
|
|
378
|
+
self.backend.discard_unreadable_task(
|
|
379
|
+
task_id, queue_name=queue_name, error=exception
|
|
132
380
|
)
|
|
133
381
|
return
|
|
134
382
|
if task_result is None:
|
|
@@ -177,7 +425,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
177
425
|
|
|
178
426
|
Uses sorted sets for priority ordering, a running set for in-flight
|
|
179
427
|
tracking, and a deferred set for scheduled tasks. All multi-step operations
|
|
180
|
-
are atomic via
|
|
428
|
+
are atomic via Redis Functions, loaded on first use.
|
|
181
429
|
"""
|
|
182
430
|
|
|
183
431
|
supports_async_task = True
|
|
@@ -187,20 +435,15 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
187
435
|
|
|
188
436
|
broker_class = RedisBroker
|
|
189
437
|
|
|
190
|
-
|
|
191
|
-
|
|
438
|
+
TASK_KEY_PREFIX = "{prefix}:task:"
|
|
439
|
+
TASK_KEY = TASK_KEY_PREFIX + "{task_id}"
|
|
440
|
+
RESULT_KEY_PREFIX = "{prefix}:result:"
|
|
441
|
+
RESULT_KEY = RESULT_KEY_PREFIX + "{result_id}"
|
|
192
442
|
SEGMENT_KEY = "{prefix}:{queue_name}:{status}"
|
|
193
443
|
DEFERRED_KEY = "{prefix}:{queue_name}:deferred"
|
|
194
444
|
|
|
195
|
-
LEASE_FIELDS = ("lease_worker", "lease_started_at", "lease_token")
|
|
196
|
-
|
|
197
445
|
TELEMETRY_CHANNEL = "{prefix}:telemetry"
|
|
198
446
|
|
|
199
|
-
ACQUIRE_SCRIPT = _load_lua("acquire")
|
|
200
|
-
"""Lease up to a given number of tasks from the priority queues in one broker call."""
|
|
201
|
-
ACKNOWLEDGE_SCRIPT = _load_lua("acknowledge")
|
|
202
|
-
"""Remove from running, persist the result, and clean up."""
|
|
203
|
-
|
|
204
447
|
def _segment_key(self, status: TaskResultStatus, queue_name: str) -> str:
|
|
205
448
|
return self.SEGMENT_KEY.format(
|
|
206
449
|
prefix=self.key_prefix,
|
|
@@ -212,7 +455,9 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
212
455
|
"""Return a running task as its lease holds it, or None when its hash is gone."""
|
|
213
456
|
task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_id)
|
|
214
457
|
data, lease_worker, lease_started_at, lease_token = self.client.hmget(
|
|
215
|
-
task_key,
|
|
458
|
+
task_key,
|
|
459
|
+
ThreadmillFunctionCommands.DATA,
|
|
460
|
+
*ThreadmillFunctionCommands.LEASE_FIELDS,
|
|
216
461
|
)
|
|
217
462
|
if data is None:
|
|
218
463
|
return None
|
|
@@ -235,7 +480,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
235
480
|
)
|
|
236
481
|
raise ValueError(message) from e
|
|
237
482
|
self.redis_connection_retries = self.options.get("redis_connection_retries", 3)
|
|
238
|
-
self.client =
|
|
483
|
+
self.client = ThreadmillRedis.from_url(
|
|
239
484
|
redis_url,
|
|
240
485
|
decode_responses=True,
|
|
241
486
|
retry=redis.retry.Retry(
|
|
@@ -244,6 +489,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
244
489
|
)
|
|
245
490
|
self.redis_url = redis_url
|
|
246
491
|
self.key_prefix = f"threadmill:{{{alias}}}"
|
|
492
|
+
self.task_key_prefix = self.TASK_KEY_PREFIX.format(prefix=self.key_prefix)
|
|
493
|
+
self.result_key_prefix = self.RESULT_KEY_PREFIX.format(prefix=self.key_prefix)
|
|
247
494
|
self.telemetry_channel = self.TELEMETRY_CHANNEL.format(prefix=self.key_prefix)
|
|
248
495
|
self.lease_ttl = self.options.get("lease_ttl", datetime.timedelta(hours=1))
|
|
249
496
|
self.result_ttl = self.options.get("result_ttl", datetime.timedelta(days=1))
|
|
@@ -255,8 +502,6 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
255
502
|
"poll_max_interval", datetime.timedelta(seconds=1)
|
|
256
503
|
)
|
|
257
504
|
self._idle_backoffs = threading.local()
|
|
258
|
-
self._acquire_script = self.client.register_script(self.ACQUIRE_SCRIPT)
|
|
259
|
-
self._acknowledge_script = self.client.register_script(self.ACKNOWLEDGE_SCRIPT)
|
|
260
505
|
|
|
261
506
|
@property
|
|
262
507
|
def _idle_backoff(self) -> _IdleBackoff:
|
|
@@ -269,8 +514,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
269
514
|
self._idle_backoffs.backoff = backoff
|
|
270
515
|
return backoff
|
|
271
516
|
|
|
272
|
-
def async_client(self) ->
|
|
273
|
-
return
|
|
517
|
+
def async_client(self) -> ThreadmillAsyncRedis:
|
|
518
|
+
return ThreadmillAsyncRedis.from_url(
|
|
274
519
|
self.redis_url,
|
|
275
520
|
decode_responses=True,
|
|
276
521
|
retry=redis.asyncio.retry.Retry(
|
|
@@ -321,19 +566,15 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
321
566
|
score = self._compute_score(task.priority, enqueued_at)
|
|
322
567
|
serialized = self.serialize_task_result(task_result)
|
|
323
568
|
task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_result.id)
|
|
324
|
-
task_data_ttl = int(
|
|
325
|
-
self.lease_ttl.total_seconds() * 3 + self.result_ttl.total_seconds()
|
|
326
|
-
)
|
|
327
569
|
|
|
328
570
|
pipe = self.client.pipeline()
|
|
329
571
|
pipe.hset(
|
|
330
572
|
task_key,
|
|
331
573
|
mapping={
|
|
332
|
-
|
|
333
|
-
|
|
574
|
+
ThreadmillFunctionCommands.DATA: serialized,
|
|
575
|
+
ThreadmillFunctionCommands.SCORE: str(score),
|
|
334
576
|
},
|
|
335
577
|
)
|
|
336
|
-
pipe.expire(task_key, task_data_ttl)
|
|
337
578
|
pipe.publish(self.telemetry_channel, f"ingress:{task.queue_name}")
|
|
338
579
|
|
|
339
580
|
if task.run_after is not None:
|
|
@@ -375,32 +616,41 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
375
616
|
now_ms = now.timestamp() * 1000
|
|
376
617
|
now_iso = now.isoformat()
|
|
377
618
|
|
|
378
|
-
if result := self.
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
str(idle_backoff.rotation_offset),
|
|
388
|
-
str(count),
|
|
389
|
-
],
|
|
619
|
+
if result := self.client.acquire(
|
|
620
|
+
queue_keys=keys,
|
|
621
|
+
now_ms=now_ms,
|
|
622
|
+
lease_started_at_iso=now_iso,
|
|
623
|
+
task_key_prefix=self.task_key_prefix,
|
|
624
|
+
worker=worker,
|
|
625
|
+
lease_ttl_ms=int(self.lease_ttl.total_seconds() * 1000),
|
|
626
|
+
start_index=idle_backoff.rotation_offset,
|
|
627
|
+
max_count=count,
|
|
390
628
|
):
|
|
391
629
|
idle_backoff.miss_count = 0
|
|
392
630
|
idle_backoff.rotation_offset = (idle_backoff.rotation_offset + 1) % len(
|
|
393
631
|
queue_names
|
|
394
632
|
)
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
633
|
+
task_results: list[ThreadmillTaskResult] = []
|
|
634
|
+
for task_id, data, lease_token, queue_index in result:
|
|
635
|
+
try:
|
|
636
|
+
task_result = self.deserialize_task_result(data)
|
|
637
|
+
except (ImportError, ValueError) as exception:
|
|
638
|
+
self.discard_unreadable_task(
|
|
639
|
+
task_id,
|
|
640
|
+
queue_name=queue_names[int(queue_index)],
|
|
641
|
+
error=exception,
|
|
642
|
+
)
|
|
643
|
+
else:
|
|
644
|
+
task_results.append(
|
|
645
|
+
self._apply_lease(
|
|
646
|
+
task_result,
|
|
647
|
+
worker=worker,
|
|
648
|
+
lease_started_at=now,
|
|
649
|
+
lease_token=lease_token,
|
|
650
|
+
)
|
|
651
|
+
)
|
|
652
|
+
if task_results:
|
|
653
|
+
return task_results
|
|
404
654
|
|
|
405
655
|
try:
|
|
406
656
|
remaining = deadline - time.monotonic()
|
|
@@ -451,6 +701,31 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
451
701
|
),
|
|
452
702
|
)
|
|
453
703
|
|
|
704
|
+
def discard_unreadable_task(
|
|
705
|
+
self,
|
|
706
|
+
task_id: str,
|
|
707
|
+
*,
|
|
708
|
+
queue_name: str,
|
|
709
|
+
error: BaseException,
|
|
710
|
+
) -> None:
|
|
711
|
+
"""Remove a task whose stored payload cannot be read."""
|
|
712
|
+
logger.error(
|
|
713
|
+
"Task %r payload cannot be read: %s; removing it from queue %r",
|
|
714
|
+
task_id,
|
|
715
|
+
error,
|
|
716
|
+
queue_name,
|
|
717
|
+
exc_info=error,
|
|
718
|
+
)
|
|
719
|
+
self.client.dequeue(
|
|
720
|
+
segment_key=self._segment_key(TaskResultStatus.RUNNING, queue_name),
|
|
721
|
+
task_data_key=self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_id),
|
|
722
|
+
result_key=self.RESULT_KEY.format(
|
|
723
|
+
prefix=self.key_prefix, result_id=task_id
|
|
724
|
+
),
|
|
725
|
+
task_id=task_id,
|
|
726
|
+
)
|
|
727
|
+
self.client.publish(self.telemetry_channel, f"egress:{queue_name}")
|
|
728
|
+
|
|
454
729
|
def acknowledge(self, task_result: TaskResult) -> None:
|
|
455
730
|
serialized = self.serialize_task_result(task_result)
|
|
456
731
|
running_key = self._segment_key(
|
|
@@ -474,24 +749,20 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
474
749
|
else None
|
|
475
750
|
)
|
|
476
751
|
|
|
477
|
-
self.
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
self.telemetry_channel,
|
|
492
|
-
task_result.task.queue_name,
|
|
493
|
-
lease_token or "",
|
|
494
|
-
],
|
|
752
|
+
self.client.acknowledge(
|
|
753
|
+
running_key=running_key,
|
|
754
|
+
result_key=result_key,
|
|
755
|
+
task_data_key=task_key,
|
|
756
|
+
successful_results_key=successful_results_key,
|
|
757
|
+
failed_results_key=failed_results_key,
|
|
758
|
+
task_id=task_result.id,
|
|
759
|
+
serialized_result=serialized,
|
|
760
|
+
result_ttl_secs=int(self.result_ttl.total_seconds()),
|
|
761
|
+
finished_at_ms=finish_score,
|
|
762
|
+
status=task_result.status.name,
|
|
763
|
+
telemetry_channel=self.telemetry_channel,
|
|
764
|
+
queue_name=task_result.task.queue_name,
|
|
765
|
+
lease_token=lease_token or "",
|
|
495
766
|
)
|
|
496
767
|
|
|
497
768
|
def requeue(self, task_result: TaskResult, run_after: datetime.datetime) -> None:
|
|
@@ -517,23 +788,20 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
517
788
|
)
|
|
518
789
|
score = self._compute_score(task_result.task.priority, task_result.enqueued_at)
|
|
519
790
|
run_after_ms = run_after.timestamp() * 1000
|
|
520
|
-
task_data_ttl = int(
|
|
521
|
-
self.lease_ttl.total_seconds() * 3 + self.result_ttl.total_seconds()
|
|
522
|
-
)
|
|
523
791
|
|
|
524
792
|
pipe = self.client.pipeline()
|
|
525
793
|
pipe.zrem(running_key, task_result.id)
|
|
526
794
|
pipe.zrem(failed_key, task_result.id)
|
|
527
795
|
pipe.delete(result_key)
|
|
796
|
+
pipe.delete(task_key)
|
|
528
797
|
pipe.hset(
|
|
529
798
|
task_key,
|
|
530
799
|
mapping={
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
**dict.fromkeys(
|
|
800
|
+
ThreadmillFunctionCommands.DATA: serialized,
|
|
801
|
+
ThreadmillFunctionCommands.SCORE: str(score),
|
|
802
|
+
**dict.fromkeys(ThreadmillFunctionCommands.LEASE_FIELDS, ""),
|
|
534
803
|
},
|
|
535
804
|
)
|
|
536
|
-
pipe.expire(task_key, task_data_ttl)
|
|
537
805
|
pipe.zadd(deferred_key, {task_result.id: run_after_ms})
|
|
538
806
|
pipe.publish(self.telemetry_channel, f"ingress:{task_result.task.queue_name}")
|
|
539
807
|
pipe.execute()
|
|
@@ -541,17 +809,33 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
541
809
|
task_enqueued.send(self.__class__, task_result=task_result)
|
|
542
810
|
|
|
543
811
|
def dequeue(self, task_result: TaskResult) -> None:
|
|
544
|
-
self.client.
|
|
545
|
-
self._segment_key(
|
|
546
|
-
|
|
812
|
+
self.client.dequeue(
|
|
813
|
+
segment_key=self._segment_key(
|
|
814
|
+
task_result.status, task_result.task.queue_name
|
|
815
|
+
),
|
|
816
|
+
task_data_key=self.TASK_KEY.format(
|
|
817
|
+
prefix=self.key_prefix, task_id=task_result.id
|
|
818
|
+
),
|
|
819
|
+
result_key=self.RESULT_KEY.format(
|
|
820
|
+
prefix=self.key_prefix, result_id=task_result.id
|
|
821
|
+
),
|
|
822
|
+
task_id=task_result.id,
|
|
547
823
|
)
|
|
548
824
|
|
|
549
825
|
def purge(self, queue_name: str) -> None:
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
826
|
+
self.client.purge(
|
|
827
|
+
ready_key=self._segment_key(TaskResultStatus.READY, queue_name),
|
|
828
|
+
running_key=self._segment_key(TaskResultStatus.RUNNING, queue_name),
|
|
829
|
+
deferred_key=self.DEFERRED_KEY.format(
|
|
830
|
+
prefix=self.key_prefix, queue_name=queue_name
|
|
831
|
+
),
|
|
832
|
+
successful_results_key=self._segment_key(
|
|
833
|
+
TaskResultStatus.SUCCESSFUL, queue_name
|
|
834
|
+
),
|
|
835
|
+
failed_results_key=self._segment_key(TaskResultStatus.FAILED, queue_name),
|
|
836
|
+
task_key_prefix=self.task_key_prefix,
|
|
837
|
+
result_key_prefix=self.result_key_prefix,
|
|
838
|
+
)
|
|
555
839
|
|
|
556
840
|
def peek(
|
|
557
841
|
self,
|
|
@@ -579,9 +863,13 @@ class RedisTaskBackend(ThreadmillTaskBackend):
|
|
|
579
863
|
for task_id in self.client.zrange(zset_key, 0, count - 1):
|
|
580
864
|
task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_id)
|
|
581
865
|
if leased:
|
|
582
|
-
pipe.hmget(
|
|
866
|
+
pipe.hmget(
|
|
867
|
+
task_key,
|
|
868
|
+
ThreadmillFunctionCommands.DATA,
|
|
869
|
+
*ThreadmillFunctionCommands.LEASE_FIELDS,
|
|
870
|
+
)
|
|
583
871
|
else:
|
|
584
|
-
pipe.hget(task_key,
|
|
872
|
+
pipe.hget(task_key, ThreadmillFunctionCommands.DATA)
|
|
585
873
|
for stored in pipe.execute():
|
|
586
874
|
if not stored:
|
|
587
875
|
continue
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
-- Finalize a completed task: remove it from the running set, persist the
|
|
2
|
-
-- result with a TTL, delete the task data hash, and add the result to the
|
|
3
|
-
-- per-status results history set. Evicts results whose finish score falls
|
|
4
|
-
-- outside the retention window (result_ttl). The per-status history sets are
|
|
5
|
-
-- the time series the inspector counts over for telemetry, so no separate
|
|
6
|
-
-- egress window or status counters are needed here. A pub/sub event is
|
|
7
|
-
-- published so the inspector can track live egress throughput.
|
|
8
|
-
--
|
|
9
|
-
-- KEYS[1] -- running set (ZSET)
|
|
10
|
-
-- KEYS[2] -- result key (STRING, stores serialized TaskResult)
|
|
11
|
-
-- KEYS[3] -- task data key (HASH, deleted after acknowledge)
|
|
12
|
-
-- KEYS[4] -- successful results history (ZSET, scored by finish time)
|
|
13
|
-
-- KEYS[5] -- failed results history (ZSET, scored by finish time)
|
|
14
|
-
-- ARGV[1] -- task ID
|
|
15
|
-
-- ARGV[2] -- serialized TaskResult JSON
|
|
16
|
-
-- ARGV[3] -- result TTL in seconds
|
|
17
|
-
-- ARGV[4] -- finish timestamp in milliseconds (score for the history set)
|
|
18
|
-
-- ARGV[5] -- status (SUCCESSFUL or FAILED)
|
|
19
|
-
-- ARGV[6] -- telemetry pub/sub channel
|
|
20
|
-
-- ARGV[7] -- queue name
|
|
21
|
-
-- Returns: 1 on success, 0 if task was not in the running set
|
|
22
|
-
|
|
23
|
-
local lease_token = redis.call('HGET', KEYS[3], 'lease_token')
|
|
24
|
-
if lease_token and lease_token ~= ARGV[8] then
|
|
25
|
-
return 0
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
local removed = redis.call('ZREM', KEYS[1], ARGV[1])
|
|
29
|
-
if removed == 0 then
|
|
30
|
-
return 0 -- Task already reaped, skip
|
|
31
|
-
end
|
|
32
|
-
redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
|
|
33
|
-
redis.call('DEL', KEYS[3])
|
|
34
|
-
local finish = tonumber(ARGV[4])
|
|
35
|
-
local cutoff = finish - tonumber(ARGV[3]) * 1000
|
|
36
|
-
local results_key = KEYS[4]
|
|
37
|
-
if ARGV[5] ~= 'SUCCESSFUL' then
|
|
38
|
-
results_key = KEYS[5]
|
|
39
|
-
end
|
|
40
|
-
redis.call('ZADD', results_key, finish, ARGV[1])
|
|
41
|
-
redis.call('ZREMRANGEBYSCORE', results_key, 0, cutoff)
|
|
42
|
-
redis.call('PUBLISH', ARGV[6], 'egress:' .. ARGV[7])
|
|
43
|
-
return 1
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
-- Lease up to ARGV[8] ready tasks for the calling worker in one round-trip:
|
|
2
|
-
-- each task's stored payload comes back beside the lease token stamped for it.
|
|
3
|
-
-- Queues are tried round-robin from ARGV[7], one queue per pop, so a batch
|
|
4
|
-
-- spreads across queues instead of draining one and a backlogged queue cannot
|
|
5
|
-
-- starve its neighbours.
|
|
6
|
-
--
|
|
7
|
-
-- The payload comes back as it was enqueued. Apply the lease stamped beside it
|
|
8
|
-
-- (lease_worker, lease_started_at, lease_token) to report the task as RUNNING.
|
|
9
|
-
--
|
|
10
|
-
-- KEYS[1..N] -- interleaved running keys and queue keys, one pair per queue:
|
|
11
|
-
-- KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
|
|
12
|
-
-- KEYS[4] = queue, etc.
|
|
13
|
-
-- ARGV[1] -- current time in milliseconds
|
|
14
|
-
-- ARGV[2] -- current time as ISO-8601 string, stamped as lease_started_at
|
|
15
|
-
-- ARGV[3] -- task key prefix (e.g. "threadmill:task:")
|
|
16
|
-
-- ARGV[4] -- number of queue pairs (N/2)
|
|
17
|
-
-- ARGV[5] -- worker name, stamped as lease_worker
|
|
18
|
-
-- ARGV[6] -- lease TTL in milliseconds
|
|
19
|
-
-- ARGV[7] -- start_index; 0-based index of the queue pair to scan first, so
|
|
20
|
-
-- start_index 0 is the pair at KEYS[1] and KEYS[2]
|
|
21
|
-
-- ARGV[8] -- maximum number of tasks to lease
|
|
22
|
-
-- Returns: array of {payload, lease_token} pairs, empty when no queue yields a
|
|
23
|
-
-- task. A queue whose head entry holds no payload yields nothing for that pop
|
|
24
|
-
-- and the scan moves on.
|
|
25
|
-
|
|
26
|
-
local num_queues = tonumber(ARGV[4])
|
|
27
|
-
local lease_ttl_ms = tonumber(ARGV[6])
|
|
28
|
-
local start_index = tonumber(ARGV[7])
|
|
29
|
-
local max_count = tonumber(ARGV[8])
|
|
30
|
-
local task_key_prefix = ARGV[3]
|
|
31
|
-
local now_ms = tonumber(ARGV[1])
|
|
32
|
-
local tasks = {}
|
|
33
|
-
local misses = 0
|
|
34
|
-
local queue_index = start_index % num_queues + 1
|
|
35
|
-
-- joe: one full round without a task ends the scan, so a queue whose head
|
|
36
|
-
-- entry is a ghost yields a short batch; retry the queue if ghosts ever dominate
|
|
37
|
-
while #tasks < max_count and misses < num_queues do
|
|
38
|
-
local popped = redis.call('ZPOPMIN', KEYS[queue_index * 2])
|
|
39
|
-
local data = #popped > 0
|
|
40
|
-
and redis.call('HGET', task_key_prefix .. popped[1], 'data')
|
|
41
|
-
if data then
|
|
42
|
-
local lease_token = string.format(
|
|
43
|
-
'%06x%06x%06x%06x',
|
|
44
|
-
math.random(0, 0xffffff), math.random(0, 0xffffff),
|
|
45
|
-
math.random(0, 0xffffff), math.random(0, 0xffffff))
|
|
46
|
-
local deadline = now_ms + lease_ttl_ms
|
|
47
|
-
redis.call('ZADD', KEYS[queue_index * 2 - 1], deadline, popped[1])
|
|
48
|
-
redis.call('HSET', task_key_prefix .. popped[1],
|
|
49
|
-
'lease_worker', ARGV[5],
|
|
50
|
-
'lease_started_at', ARGV[2],
|
|
51
|
-
'lease_token', lease_token)
|
|
52
|
-
table.insert(tasks, { data, lease_token })
|
|
53
|
-
end
|
|
54
|
-
misses = data and 0 or (misses + 1)
|
|
55
|
-
queue_index = queue_index % num_queues + 1
|
|
56
|
-
end
|
|
57
|
-
return tasks
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
-- Move tasks whose scheduled time has passed from the deferred set to the active
|
|
2
|
-
-- priority queue. Only processes up to a batch limit per call.
|
|
3
|
-
--
|
|
4
|
-
-- KEYS[1] -- deferred set (ZSET, scored by run_after timestamp)
|
|
5
|
-
-- KEYS[2] -- active priority queue (ZSET, scored by priority+time)
|
|
6
|
-
-- ARGV[1] -- current time in milliseconds (all scores <= this are due)
|
|
7
|
-
-- ARGV[2] -- task key prefix (e.g. "threadmill:default:task:")
|
|
8
|
-
-- ARGV[3] -- maximum number of tasks to move per call (batch size)
|
|
9
|
-
-- Returns: number of tasks moved
|
|
10
|
-
|
|
11
|
-
local due = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1], 'LIMIT', 0, tonumber(ARGV[3]))
|
|
12
|
-
for _, task_id in ipairs(due) do
|
|
13
|
-
redis.call('ZREM', KEYS[1], task_id)
|
|
14
|
-
local score = redis.call('HGET', ARGV[2] .. task_id, 'score')
|
|
15
|
-
if score then
|
|
16
|
-
redis.call('ZADD', KEYS[2], score, task_id)
|
|
17
|
-
end
|
|
18
|
-
end
|
|
19
|
-
return #due
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
-- Claim expired tasks for the broker to decide. Undecided claims are picked up
|
|
2
|
-
-- again, and running entries without task data are dropped.
|
|
3
|
-
--
|
|
4
|
-
-- KEYS[1] -- running set (ZSET, scored by lease deadline in milliseconds)
|
|
5
|
-
-- ARGV[1] -- claim TTL in milliseconds
|
|
6
|
-
-- ARGV[2] -- maximum number of tasks to claim per call (batch size)
|
|
7
|
-
-- ARGV[3] -- task key prefix (e.g. "threadmill:default:task:")
|
|
8
|
-
-- Returns: list of claimed task IDs
|
|
9
|
-
|
|
10
|
-
local clock = redis.call('TIME')
|
|
11
|
-
local now_ms = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
|
|
12
|
-
local claim_deadline_ms = now_ms + tonumber(ARGV[1])
|
|
13
|
-
local expired = redis.call('ZRANGEBYSCORE', KEYS[1], 0, now_ms, 'LIMIT', 0, tonumber(ARGV[2]))
|
|
14
|
-
local claimed = {}
|
|
15
|
-
for _, task_id in ipairs(expired) do
|
|
16
|
-
if redis.call('EXISTS', ARGV[3] .. task_id) == 1 then
|
|
17
|
-
redis.call('ZADD', KEYS[1], claim_deadline_ms, task_id)
|
|
18
|
-
table.insert(claimed, task_id)
|
|
19
|
-
else
|
|
20
|
-
redis.call('ZREM', KEYS[1], task_id)
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
return claimed
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|