llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,696 @@
|
|
|
1
|
+
"""Lua of the admission scripts: PICK_TAKE, FINISH, UNDO, RENEW and SWEEP (plan §3.4, §3.18).
|
|
2
|
+
|
|
3
|
+
Each script takes the key prefix and one JSON argument and builds its key names itself, which
|
|
4
|
+
needs the single-primary store 0.3 supports. Times are the store's clock in ms. Budgets are GCRA
|
|
5
|
+
clocks (`smooth`) or minute counters (`fixed`); class clocks enforce the priority shares.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# ruff: noqa: E501 # Lua source in strings; wrapping its lines hurts reading it
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
_COMMON = r"""
|
|
13
|
+
local t = redis.call('TIME')
|
|
14
|
+
local now = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
|
|
15
|
+
local minute = math.floor(now / 60000)
|
|
16
|
+
local P = ARGV[1]
|
|
17
|
+
local req = cjson.decode(ARGV[2])
|
|
18
|
+
-- Timings and limits: `EngineTimings` (sent as `t` by every call) and the registry's breaker
|
|
19
|
+
-- settings (on each row); the numbers after them are the defaults.
|
|
20
|
+
local T = req.t or {}
|
|
21
|
+
local KEY_MS -- budget key lifetime: set below, once num() exists
|
|
22
|
+
local function mk(alias, name) return P .. ':{m:' .. alias .. '}:' .. name end
|
|
23
|
+
local function num(v, d) if v == nil or v == false or v == cjson.null then return d end return tonumber(v) end
|
|
24
|
+
local function hnum(key, field, d) return num(redis.call('HGET', key, field), d) end
|
|
25
|
+
local function present(v) return v ~= nil and v ~= false and v ~= cjson.null end
|
|
26
|
+
KEY_MS = num(T.budget_key_ms, 180000) -- budget and limiter keys live this long (> 2 minutes)
|
|
27
|
+
|
|
28
|
+
-- One budget: field prefix f, limit L per minute, burst fraction bf, window mode.
|
|
29
|
+
local function avail(key, f, L, bf, mode)
|
|
30
|
+
local burst = math.max(1, math.floor(L * bf))
|
|
31
|
+
if mode == 'fixed' then
|
|
32
|
+
local n = 0
|
|
33
|
+
if hnum(key, f .. '_win', -1) == minute then n = hnum(key, f .. '_n', 0) end
|
|
34
|
+
return L - n, n
|
|
35
|
+
end
|
|
36
|
+
local per = 60000 / L
|
|
37
|
+
local tat = math.max(hnum(key, f .. '_tat', now), now)
|
|
38
|
+
return (now + burst * per - tat) / per, tat
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
-- Room for cost c behind R reserved units: returns wait ms (0 = fits) and the new state.
|
|
42
|
+
local reset_minute = num(redis.call('GET', P .. ':reset_minute'), -1)
|
|
43
|
+
local function room(key, f, L, c, R, bf, mode)
|
|
44
|
+
local burst = math.max(1, math.floor(L * bf))
|
|
45
|
+
if mode == 'fixed' then
|
|
46
|
+
-- After a store reset, a fixed window holds until the next minute (plan T10).
|
|
47
|
+
if reset_minute == minute and hnum(key, f .. '_win', -1) ~= minute then
|
|
48
|
+
return (minute + 1) * 60000 - now, nil
|
|
49
|
+
end
|
|
50
|
+
local _, n = avail(key, f, L, bf, mode)
|
|
51
|
+
if n + R + c <= L or (n == 0 and R == 0 and c > L) then return 0, n + c end
|
|
52
|
+
return (minute + 1) * 60000 - now, nil
|
|
53
|
+
end
|
|
54
|
+
local per = 60000 / L
|
|
55
|
+
local tat = math.max(hnum(key, f .. '_tat', now), now)
|
|
56
|
+
if c > burst and R == 0 then
|
|
57
|
+
if tat <= now then return 0, now + c * per end
|
|
58
|
+
return tat - now, nil
|
|
59
|
+
end
|
|
60
|
+
local over = tat + (R + c) * per - now - burst * per
|
|
61
|
+
if over > 0 then return over, nil end
|
|
62
|
+
return 0, tat + c * per
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
local function write(key, f, mode, value)
|
|
66
|
+
if mode == 'fixed' then
|
|
67
|
+
redis.call('HSET', key, f .. '_win', minute, f .. '_n', value)
|
|
68
|
+
else
|
|
69
|
+
redis.call('HSET', key, f .. '_tat', tostring(value))
|
|
70
|
+
end
|
|
71
|
+
redis.call('PEXPIRE', key, KEY_MS)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
-- Give back g units (negative g charges more). Fixed counters only inside the grant's minute.
|
|
75
|
+
local function give(key, f, L, g, mode, granted_minute)
|
|
76
|
+
if not L or L <= 0 or g == 0 then return end
|
|
77
|
+
if mode == 'fixed' then
|
|
78
|
+
if granted_minute ~= minute and g > 0 then return end
|
|
79
|
+
if hnum(key, f .. '_win', -1) ~= minute then
|
|
80
|
+
-- The minute turned: an overrun (g < 0) is charged to the new minute, not dropped.
|
|
81
|
+
if g < 0 then
|
|
82
|
+
redis.call('HSET', key, f .. '_win', minute, f .. '_n', -g)
|
|
83
|
+
redis.call('PEXPIRE', key, KEY_MS)
|
|
84
|
+
end
|
|
85
|
+
return
|
|
86
|
+
end
|
|
87
|
+
redis.call('HSET', key, f .. '_n', math.max(0, hnum(key, f .. '_n', 0) - g))
|
|
88
|
+
redis.call('PEXPIRE', key, KEY_MS)
|
|
89
|
+
return
|
|
90
|
+
end
|
|
91
|
+
local per = 60000 / L
|
|
92
|
+
local tat = math.max(hnum(key, f .. '_tat', now), now)
|
|
93
|
+
local value = tat - g * per
|
|
94
|
+
if g > 0 then value = math.max(value, now) end
|
|
95
|
+
redis.call('HSET', key, f .. '_tat', tostring(value))
|
|
96
|
+
redis.call('PEXPIRE', key, KEY_MS) -- the key may have expired since the grant
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
-- The budgets of a deployment row for class cls: {key, field, limit, cost kind}.
|
|
100
|
+
local function budgets(alias, row, model, cls, account)
|
|
101
|
+
local list = {}
|
|
102
|
+
local limits = row.rate_limits
|
|
103
|
+
local rkey = mk(alias, 'rate:' .. row.key)
|
|
104
|
+
local share = 1
|
|
105
|
+
if cls == 'n' then share = 1 - num(model.priority_reserve, 0) else share = 1 - num(model.normal_min_share, 0) end
|
|
106
|
+
local function add(key, f, L, kind)
|
|
107
|
+
if present(L) and tonumber(L) > 0 then table.insert(list, {key, f, tonumber(L), kind}) end
|
|
108
|
+
end
|
|
109
|
+
if limits == 'both' or limits == 'rpm' then
|
|
110
|
+
add(rkey, 'rpm', row.rpm, 'r')
|
|
111
|
+
if share < 1 and present(row.rpm) then add(rkey, cls .. '_rpm', row.rpm * share, 'r') end
|
|
112
|
+
end
|
|
113
|
+
if limits == 'both' or limits == 'tpm' then
|
|
114
|
+
add(rkey, 'tpm', row.tpm, 't')
|
|
115
|
+
if share < 1 and present(row.tpm) then add(rkey, cls .. '_tpm', row.tpm * share, 't') end
|
|
116
|
+
end
|
|
117
|
+
if account then
|
|
118
|
+
local akey = P .. ':{a:' .. row.account .. '}:rate'
|
|
119
|
+
local ashare = 1
|
|
120
|
+
if cls == 'n' then ashare = 1 - num(row.account_priority_reserve, 0) end
|
|
121
|
+
add(akey, 'rpm', row.shared_rpm, 'r')
|
|
122
|
+
add(akey, 'tpm', row.shared_tpm, 't')
|
|
123
|
+
if ashare < 1 then
|
|
124
|
+
if present(row.shared_rpm) then add(akey, 'n_rpm', row.shared_rpm * ashare, 'r') end
|
|
125
|
+
if present(row.shared_tpm) then add(akey, 'n_tpm', row.shared_tpm * ashare, 't') end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
return list
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
local function lease_keys(alias, dep)
|
|
132
|
+
return mk(alias, 'lease:' .. dep .. ':p'), mk(alias, 'lease:' .. dep .. ':n')
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
local function limit_of(alias, row)
|
|
136
|
+
local key = mk(alias, 'ac:' .. row.key)
|
|
137
|
+
local cap = num(row.max_concurrency, 200)
|
|
138
|
+
local limit = hnum(key, 'limit', nil)
|
|
139
|
+
if limit == nil then
|
|
140
|
+
-- A new deployment takes its full share at once (plan E74).
|
|
141
|
+
limit = cap
|
|
142
|
+
redis.call('HSET', key, 'limit', tostring(limit))
|
|
143
|
+
end
|
|
144
|
+
return math.max(1, math.floor(math.min(limit, cap))), cap
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
local function cfg_of(alias)
|
|
148
|
+
local raw = redis.call('HGET', mk(alias, 'cfg'), 'numbers')
|
|
149
|
+
if not raw then return nil end
|
|
150
|
+
local doc = cjson.decode(raw)
|
|
151
|
+
local rows = {}
|
|
152
|
+
for _, row in ipairs(doc.deployments) do rows[row.key] = row end
|
|
153
|
+
return doc.model, rows, hnum(mk(alias, 'cfg'), 'changes', -1)
|
|
154
|
+
end
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
PICK_TAKE = (
|
|
158
|
+
_COMMON
|
|
159
|
+
+ r"""
|
|
160
|
+
local alias = req.alias
|
|
161
|
+
local model, rows, changes = cfg_of(alias)
|
|
162
|
+
if model == nil then return cjson.encode({r = 'no_config'}) end
|
|
163
|
+
if changes ~= num(req.changes, -2) then return cjson.encode({r = 'stale_config', changes = changes}) end
|
|
164
|
+
local cls = req.cls
|
|
165
|
+
local rid = req.request_id
|
|
166
|
+
local entry_key = mk(alias, 'inprog:' .. rid)
|
|
167
|
+
|
|
168
|
+
-- Finished or cancelled? Checked for every ask: a fallback ask (claim off) once started a
|
|
169
|
+
-- call that was already cancelled or answered elsewhere. Only the claim write needs `claim`.
|
|
170
|
+
local job_key = P .. ':{r:' .. rid .. '}:job'
|
|
171
|
+
if redis.call('EXISTS', P .. ':{r:' .. rid .. '}:res') == 1 then return cjson.encode({r = 'done'}) end
|
|
172
|
+
if redis.call('EXISTS', P .. ':{r:' .. rid .. '}:cancel') == 1 then return cjson.encode({r = 'cancelled'}) end
|
|
173
|
+
local state = redis.call('HGET', job_key, 'state')
|
|
174
|
+
if state == 'done' or state == 'failed' or state == 'expired' or state == 'cancelled' then
|
|
175
|
+
return cjson.encode({r = 'done'})
|
|
176
|
+
end
|
|
177
|
+
-- Running under another worker with a fresh heartbeat (on another model, or between its
|
|
178
|
+
-- retries when no entry exists): a redelivered message is a duplicate, not a new run.
|
|
179
|
+
if req.claim and state == 'running' then
|
|
180
|
+
local runner = redis.call('HGET', job_key, 'owner')
|
|
181
|
+
if runner and runner ~= req.worker
|
|
182
|
+
and now - hnum(job_key, 'hb', 0) < num(req.lease_ms, 60000) then
|
|
183
|
+
return cjson.encode({r = 'duplicate'})
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
local held = redis.call('HGET', entry_key, 'owner')
|
|
187
|
+
if held then
|
|
188
|
+
if held == req.owner then
|
|
189
|
+
return cjson.encode({r = 'granted', dep = redis.call('HGET', entry_key, 'dep'),
|
|
190
|
+
minute = hnum(entry_key, 'minute', minute), repeat_ = true})
|
|
191
|
+
end
|
|
192
|
+
local holder = redis.call('HGET', entry_key, 'worker')
|
|
193
|
+
local hb = hnum(job_key, 'hb', 0)
|
|
194
|
+
local fresh = now - hb < num(req.lease_ms, 60000) and redis.call('HGET', entry_key, 'started') ~= false
|
|
195
|
+
if fresh then return cjson.encode({r = 'duplicate'}) end
|
|
196
|
+
-- A dead owner (no heartbeat for a lease): drop its slot and entry (no refund).
|
|
197
|
+
local old_dep = redis.call('HGET', entry_key, 'dep')
|
|
198
|
+
local old_cls = redis.call('HGET', entry_key, 'cls')
|
|
199
|
+
if old_dep then redis.call('ZREM', mk(alias, 'lease:' .. old_dep .. ':' .. old_cls), held) end
|
|
200
|
+
redis.call('ZREM', mk(alias, 'inprog'), held)
|
|
201
|
+
redis.call('DEL', entry_key)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
-- Foreign call (model fallback): only while this model has no waiters of its own.
|
|
205
|
+
local foreign = present(req.foreign)
|
|
206
|
+
local wait_keys = {p = mk(alias, 'wait:p'), n = mk(alias, 'wait:n')}
|
|
207
|
+
local info_key = mk(alias, 'waitinfo')
|
|
208
|
+
for _, c in ipairs({'p', 'n'}) do
|
|
209
|
+
for _, id in ipairs(redis.call('ZRANGE', wait_keys[c], 0, 49)) do
|
|
210
|
+
local raw = redis.call('HGET', info_key, id)
|
|
211
|
+
if not raw or num(cjson.decode(raw).exp, 0) < now then
|
|
212
|
+
redis.call('ZREM', wait_keys[c], id)
|
|
213
|
+
redis.call('HDEL', info_key, id)
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
if foreign and (redis.call('ZCARD', wait_keys.p) + redis.call('ZCARD', wait_keys.n)) > 0 then
|
|
218
|
+
return cjson.encode({r = 'busy', reason = 'waiters'})
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
-- What older waiters set aside, per deployment (plan §3.18 reservation rule).
|
|
222
|
+
local reserved = {}
|
|
223
|
+
local function reserve_from(list, into)
|
|
224
|
+
into = into or reserved
|
|
225
|
+
for _, id in ipairs(list) do
|
|
226
|
+
if id ~= rid then
|
|
227
|
+
local raw = redis.call('HGET', info_key, id)
|
|
228
|
+
if raw then
|
|
229
|
+
local w = cjson.decode(raw)
|
|
230
|
+
if num(w.exp, 0) >= now then
|
|
231
|
+
for _, dep in ipairs(w.deps) do
|
|
232
|
+
local slot = into[dep] or {r = 0, t = 0, s = 0}
|
|
233
|
+
slot.r = slot.r + num(w.r, 1)
|
|
234
|
+
slot.t = slot.t + num(w.t, 0)
|
|
235
|
+
slot.s = slot.s + 1
|
|
236
|
+
into[dep] = slot
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
local pos = num(req.pos, now * 1000)
|
|
244
|
+
if not foreign then
|
|
245
|
+
-- %.17g: plain `..` formats with %.14g and misorders waiters within ~0.1 ms of each other.
|
|
246
|
+
local before = '(' .. string.format('%.17g', pos)
|
|
247
|
+
reserve_from(redis.call('ZRANGEBYSCORE', wait_keys[cls], '-inf', before, 'LIMIT', 0, 200))
|
|
248
|
+
end
|
|
249
|
+
-- What waiting priority calls need, per deployment: added to a normal call's reservation
|
|
250
|
+
-- per candidate (only where that candidate's normal use reached its floor), never to the shared
|
|
251
|
+
-- table, so the result no longer depends on the order the candidates are tried in.
|
|
252
|
+
local priority_reserved = {}
|
|
253
|
+
if cls == 'n' and not foreign then
|
|
254
|
+
reserve_from(redis.call('ZRANGE', wait_keys.p, 0, 199), priority_reserved)
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
local cost_r = num(req.cost_r, 1)
|
|
258
|
+
local cost_t = num(req.cost_t, 0)
|
|
259
|
+
local best_wait, best_reason = nil, nil
|
|
260
|
+
-- A breaker's one probe call is in flight: its answer comes within an attempt, so a call
|
|
261
|
+
-- waits for it instead of failing as `model_unavailable` (scenario P7 at 300 %: ten calls
|
|
262
|
+
-- that arrived during the probe failed after every region had recovered).
|
|
263
|
+
local probe_wait = false
|
|
264
|
+
local function note(ms, reason)
|
|
265
|
+
if best_wait == nil or ms < best_wait then best_wait, best_reason = ms, reason end
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
-- Evaluate one candidate: fits (with its new budget states and headroom) or the wait.
|
|
269
|
+
local function evaluate(cand)
|
|
270
|
+
local row = rows[cand.key]
|
|
271
|
+
if row == nil then return nil end
|
|
272
|
+
local health = mk(alias, 'health')
|
|
273
|
+
if row.enabled == false or row.account_enabled == false or row.outside_residency or row.blocked
|
|
274
|
+
or redis.call('HEXISTS', health, cand.key .. '|until') == 1 then
|
|
275
|
+
return nil
|
|
276
|
+
end
|
|
277
|
+
local cb = mk(alias, 'cb:' .. cand.key)
|
|
278
|
+
local probe = false
|
|
279
|
+
if redis.call('HGET', cb, 'state') == 'open' then
|
|
280
|
+
if hnum(cb, 'until', 0) > now then return nil end
|
|
281
|
+
local who = redis.call('HGET', cb, 'probe')
|
|
282
|
+
if who and who ~= '' and hnum(cb, 'probe_at', 0) > now - num(row.breaker_probe_secs, 60) * 1000 then
|
|
283
|
+
probe_wait = true
|
|
284
|
+
return nil
|
|
285
|
+
end
|
|
286
|
+
probe = true
|
|
287
|
+
end
|
|
288
|
+
local res = reserved[cand.key] or {r = 0, t = 0, s = 0}
|
|
289
|
+
-- Normal calls also set aside what waiting priority calls need, unless normal use is below
|
|
290
|
+
-- normal_min_share this minute.
|
|
291
|
+
local rkey = mk(alias, 'rate:' .. cand.key)
|
|
292
|
+
local prio = priority_reserved[cand.key]
|
|
293
|
+
if prio then
|
|
294
|
+
local floor = num(model.normal_min_share, 0) * num(row.rpm, 0)
|
|
295
|
+
local used = 0
|
|
296
|
+
if hnum(rkey, 'seen_win', -1) == minute then used = hnum(rkey, 'seen_n', 0) end
|
|
297
|
+
if floor <= 0 or used >= floor then
|
|
298
|
+
res = {r = res.r + prio.r, t = res.t + prio.t, s = res.s + prio.s}
|
|
299
|
+
end
|
|
300
|
+
end
|
|
301
|
+
local states = {}
|
|
302
|
+
local headroom = nil
|
|
303
|
+
local most = nil
|
|
304
|
+
for _, b in ipairs(budgets(alias, row, model, cls, true)) do
|
|
305
|
+
local cost = cost_r
|
|
306
|
+
local R = res.r
|
|
307
|
+
if b[4] == 't' then cost, R = cost_t, res.t end
|
|
308
|
+
local mode = row.rate_window
|
|
309
|
+
local wait, value = room(b[1], b[2], b[3], cost, R, num(row.rate_burst, 0.1), mode)
|
|
310
|
+
if wait > 0 then
|
|
311
|
+
local reason = (b[4] == 'r') and 'rpm' or 'tpm'
|
|
312
|
+
if string.find(b[1], '{a:', 1, true) then reason = 'account'
|
|
313
|
+
elseif string.sub(b[2], 2, 2) == '_' then reason = 'reserve' end
|
|
314
|
+
if R > 0 then reason = 'not_head' end
|
|
315
|
+
return {wait = wait, reason = reason}
|
|
316
|
+
end
|
|
317
|
+
table.insert(states, {b[1], b[2], mode, value})
|
|
318
|
+
local free = avail(b[1], b[2], b[3], num(row.rate_burst, 0.1), mode) - R
|
|
319
|
+
local left = free - cost
|
|
320
|
+
-- Headroom is the share free before this call of what can be taken now: the minute (fixed)
|
|
321
|
+
-- or the bucket (smooth, never more than rate_burst of the minute). Before, not after: a
|
|
322
|
+
-- call larger than the bucket could otherwise never fall back to an idle model.
|
|
323
|
+
local size = b[3]
|
|
324
|
+
if mode ~= 'fixed' then size = math.max(1, math.floor(b[3] * num(row.rate_burst, 0.1))) end
|
|
325
|
+
local frac = free / size
|
|
326
|
+
if headroom == nil or frac < headroom then headroom = frac end
|
|
327
|
+
if b[4] == 'r' and not string.find(b[1], '{a:', 1, true) then most = left end
|
|
328
|
+
end
|
|
329
|
+
-- Concurrency: slots per class, after what older waiters set aside.
|
|
330
|
+
local lp, ln = lease_keys(alias, cand.key)
|
|
331
|
+
redis.call('ZREMRANGEBYSCORE', lp, '-inf', now)
|
|
332
|
+
redis.call('ZREMRANGEBYSCORE', ln, '-inf', now)
|
|
333
|
+
local in_p, in_n = redis.call('ZCARD', lp), redis.call('ZCARD', ln)
|
|
334
|
+
local limit = limit_of(alias, row)
|
|
335
|
+
local share = 1 - num(model.priority_reserve, 0)
|
|
336
|
+
if cls == 'p' then share = 1 - num(model.normal_min_share, 0) end
|
|
337
|
+
local cap = math.floor(share * limit)
|
|
338
|
+
if limit >= 2 and share > 0 then cap = math.max(1, cap) end
|
|
339
|
+
local mine = (cls == 'p') and in_p or in_n
|
|
340
|
+
if in_p + in_n + res.s >= limit or mine >= cap then
|
|
341
|
+
return {wait = num(T.slot_wait_ms, 50) + math.floor(num(req.rand, 0.5) * num(T.slot_jitter_ms, 150)), reason = 'slot'}
|
|
342
|
+
end
|
|
343
|
+
local slot_frac = (limit - in_p - in_n - res.s - 1) / limit
|
|
344
|
+
if headroom == nil or slot_frac < headroom then headroom = slot_frac end
|
|
345
|
+
return {fits = true, states = states, headroom = headroom, most = most or (limit - in_p - in_n),
|
|
346
|
+
probe = probe, limit = limit}
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
-- Sticky session: keep the session's deployment while it fits or would wait only briefly.
|
|
350
|
+
local sticky_key = nil
|
|
351
|
+
local sticky_dep = nil
|
|
352
|
+
if present(req.sticky) then
|
|
353
|
+
sticky_key = mk(alias, 'sticky:' .. req.sticky.hash)
|
|
354
|
+
sticky_dep = redis.call('GET', sticky_key)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
-- Latency-aware exclusion (plan §3.4): a deployment much slower than the model's median,
|
|
358
|
+
-- with enough samples and at least 3 to compare, becomes a last choice (never excluded alone).
|
|
359
|
+
local slow = {}
|
|
360
|
+
do
|
|
361
|
+
local hs = model.health or {}
|
|
362
|
+
local ratio, min_ms, min_n = num(hs.slow_ratio, 3), num(hs.slow_min_secs, 5) * 1000, num(hs.slow_min_samples, 20)
|
|
363
|
+
local health_key = mk(alias, 'health')
|
|
364
|
+
local values, lat_of = {}, {}
|
|
365
|
+
for _, cand in ipairs(req.candidates) do
|
|
366
|
+
local lat = hnum(health_key, cand.key .. '|lat', nil)
|
|
367
|
+
if lat and hnum(health_key, cand.key .. '|n', 0) >= min_n then
|
|
368
|
+
lat_of[cand.key] = lat
|
|
369
|
+
table.insert(values, lat)
|
|
370
|
+
end
|
|
371
|
+
end
|
|
372
|
+
if #values >= 3 then
|
|
373
|
+
table.sort(values)
|
|
374
|
+
local median = values[math.floor((#values + 1) / 2)]
|
|
375
|
+
for key, lat in pairs(lat_of) do
|
|
376
|
+
if lat > ratio * median and lat > min_ms then slow[key] = true end
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
local fitting = {confirmed = {}, fallback = {}}
|
|
382
|
+
local last = {}
|
|
383
|
+
local results = {}
|
|
384
|
+
local any_usable = false
|
|
385
|
+
-- `ordered` (the default) takes the first confirmed candidate that fits, so evaluation stops
|
|
386
|
+
-- there: evaluating all of them cost ~12 store reads each (30 deployments: p99 above 2 ms).
|
|
387
|
+
local first_fit = req.strategy == 'ordered' and not present(req.sticky)
|
|
388
|
+
for _, cand in ipairs(req.candidates) do
|
|
389
|
+
local res = evaluate(cand)
|
|
390
|
+
if res ~= nil then
|
|
391
|
+
any_usable = true
|
|
392
|
+
results[cand.key] = res
|
|
393
|
+
if res.fits then
|
|
394
|
+
if cand.last or slow[cand.key] then table.insert(last, cand) else table.insert(fitting[cand.tier], cand) end
|
|
395
|
+
if first_fit and cand.tier == 'confirmed' and not (cand.last or slow[cand.key]) then break end
|
|
396
|
+
else
|
|
397
|
+
note(res.wait, res.reason)
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
end
|
|
401
|
+
if not any_usable then
|
|
402
|
+
if probe_wait then return cjson.encode({r = 'wait', ms = num(T.probe_wait_ms, 200), reason = 'probe'}) end
|
|
403
|
+
return cjson.encode({r = 'none'})
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
local chosen = nil
|
|
407
|
+
if sticky_dep and results[sticky_dep] then
|
|
408
|
+
local s = results[sticky_dep]
|
|
409
|
+
if s.fits then
|
|
410
|
+
for _, cand in ipairs(req.candidates) do if cand.key == sticky_dep then chosen = cand end end
|
|
411
|
+
elseif s.wait <= num(req.sticky.move_ms, 1000) then
|
|
412
|
+
best_wait, best_reason = s.wait, s.reason
|
|
413
|
+
chosen = false
|
|
414
|
+
end
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
local function pick(list)
|
|
418
|
+
if #list == 0 then return nil end
|
|
419
|
+
local strategy = req.strategy
|
|
420
|
+
if strategy == 'round_robin' then
|
|
421
|
+
local rot = mk(alias, 'rot')
|
|
422
|
+
local best, best_seq = nil, nil
|
|
423
|
+
for _, cand in ipairs(list) do
|
|
424
|
+
local seq = hnum(rot, 'rr|' .. cand.key, 0)
|
|
425
|
+
if best_seq == nil or seq < best_seq then best, best_seq = cand, seq end
|
|
426
|
+
end
|
|
427
|
+
return best
|
|
428
|
+
elseif strategy == 'weighted' then
|
|
429
|
+
local rot = mk(alias, 'rot')
|
|
430
|
+
local total, best, best_cw = 0, nil, nil
|
|
431
|
+
for _, cand in ipairs(list) do
|
|
432
|
+
local w = num(rows[cand.key].weight, 1)
|
|
433
|
+
local cw = hnum(rot, 'cw|' .. cand.key, 0) + w
|
|
434
|
+
redis.call('HSET', rot, 'cw|' .. cand.key, tostring(cw))
|
|
435
|
+
total = total + w
|
|
436
|
+
if best_cw == nil or cw > best_cw then best, best_cw = cand, cw end
|
|
437
|
+
end
|
|
438
|
+
redis.call('HSET', rot, 'cw|' .. best.key, tostring(best_cw - total))
|
|
439
|
+
return best
|
|
440
|
+
elseif strategy == 'headroom' then
|
|
441
|
+
local mode = req.pick or 'percent'
|
|
442
|
+
if mode == 'spread' then
|
|
443
|
+
local total = 0
|
|
444
|
+
for _, cand in ipairs(list) do total = total + math.max(0.0001, results[cand.key].headroom) end
|
|
445
|
+
local x = num(req.rand, 0.5) * total
|
|
446
|
+
for _, cand in ipairs(list) do
|
|
447
|
+
x = x - math.max(0.0001, results[cand.key].headroom)
|
|
448
|
+
if x <= 0 then return cand end
|
|
449
|
+
end
|
|
450
|
+
return list[#list]
|
|
451
|
+
end
|
|
452
|
+
local best, best_score = nil, nil
|
|
453
|
+
for _, cand in ipairs(list) do
|
|
454
|
+
local score = (mode == 'max') and results[cand.key].most or results[cand.key].headroom
|
|
455
|
+
if best_score == nil or score > best_score then best, best_score = cand, score end
|
|
456
|
+
end
|
|
457
|
+
return best
|
|
458
|
+
end
|
|
459
|
+
return list[1]
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
if chosen == nil then
|
|
463
|
+
chosen = pick(fitting.confirmed) or pick(fitting.fallback) or pick(last)
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
if foreign and chosen and num(req.foreign.require_pct, 0) > 0 then
|
|
467
|
+
if results[chosen.key].headroom < num(req.foreign.require_pct, 0) / 100 then chosen = nil end
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
if not chosen then
|
|
471
|
+
if foreign then return cjson.encode({r = 'busy', reason = 'headroom'}) end
|
|
472
|
+
-- Refused: register (or refresh) as a waiter, with the deployments it can use.
|
|
473
|
+
local deps = {}
|
|
474
|
+
for _, cand in ipairs(req.candidates) do table.insert(deps, cand.key) end
|
|
475
|
+
redis.call('ZADD', wait_keys[cls], pos, rid)
|
|
476
|
+
redis.call('HSET', info_key, rid, cjson.encode({r = cost_r, t = cost_t, deps = deps,
|
|
477
|
+
exp = now + num(req.waiter_ms, 2000)}))
|
|
478
|
+
return cjson.encode({r = 'wait', ms = math.max(1, math.floor(best_wait or num(T.wait_ms, 100))),
|
|
479
|
+
reason = best_reason or 'slot'})
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
-- Take everything at once.
|
|
483
|
+
local row = rows[chosen.key]
|
|
484
|
+
local res = results[chosen.key]
|
|
485
|
+
for _, s in ipairs(res.states) do write(s[1], s[2], s[3], s[4]) end
|
|
486
|
+
local rkey = mk(alias, 'rate:' .. chosen.key)
|
|
487
|
+
if hnum(rkey, 'seen_win', -1) ~= minute then
|
|
488
|
+
redis.call('HSET', rkey, 'seen_win', minute, 'seen_calls', 0, 'seen_tokens', 0, 'seen_n', 0)
|
|
489
|
+
redis.call('PEXPIRE', rkey, KEY_MS)
|
|
490
|
+
end
|
|
491
|
+
redis.call('HINCRBY', rkey, 'seen_calls', cost_r)
|
|
492
|
+
redis.call('HINCRBY', rkey, 'seen_tokens', cost_t)
|
|
493
|
+
if cls == 'n' then redis.call('HINCRBY', rkey, 'seen_n', cost_r) end
|
|
494
|
+
local lp, ln = lease_keys(alias, chosen.key)
|
|
495
|
+
local lease_ms = num(req.lease_ms, 60000)
|
|
496
|
+
redis.call('ZADD', (cls == 'p') and lp or ln, now + lease_ms, req.owner)
|
|
497
|
+
redis.call('ZADD', mk(alias, 'inprog'), now + lease_ms, req.owner)
|
|
498
|
+
redis.call('HSET', entry_key, 'owner', req.owner, 'worker', req.worker, 'dep', chosen.key,
|
|
499
|
+
'cls', cls, 'minute', minute, 'cost_r', cost_r, 'cost_t', cost_t, 'account', row.account,
|
|
500
|
+
'started', now, 'request', rid, 'served', alias)
|
|
501
|
+
redis.call('PEXPIRE', entry_key, lease_ms * 2)
|
|
502
|
+
if res.probe then
|
|
503
|
+
local cb = mk(alias, 'cb:' .. chosen.key)
|
|
504
|
+
redis.call('HSET', cb, 'probe', req.owner, 'probe_at', now)
|
|
505
|
+
end
|
|
506
|
+
-- Running, by this worker: a fallback grant too, so a redelivery of the call is refused
|
|
507
|
+
-- while it runs on the other model (it was granted again and ran twice).
|
|
508
|
+
if req.claim or redis.call('EXISTS', job_key) == 1 then
|
|
509
|
+
redis.call('HSET', job_key, 'state', 'running', 'owner', req.worker, 'hb', now)
|
|
510
|
+
end
|
|
511
|
+
redis.call('ZREM', wait_keys[cls], rid)
|
|
512
|
+
redis.call('HDEL', info_key, rid)
|
|
513
|
+
if foreign and present(req.foreign.from_alias) then
|
|
514
|
+
redis.call('ZREM', mk(req.foreign.from_alias, 'wait:' .. cls), rid)
|
|
515
|
+
redis.call('HDEL', mk(req.foreign.from_alias, 'waitinfo'), rid)
|
|
516
|
+
end
|
|
517
|
+
if sticky_key then
|
|
518
|
+
redis.call('SET', sticky_key, chosen.key, 'PX', num(req.sticky.ttl_ms, 1800000))
|
|
519
|
+
end
|
|
520
|
+
if req.strategy == 'round_robin' then
|
|
521
|
+
local rot = mk(alias, 'rot')
|
|
522
|
+
redis.call('HSET', rot, 'rr|' .. chosen.key, redis.call('HINCRBY', rot, 'rr_seq', 1))
|
|
523
|
+
end
|
|
524
|
+
return cjson.encode({r = 'granted', dep = chosen.key, minute = minute, limit = res.limit,
|
|
525
|
+
tier = chosen.tier})
|
|
526
|
+
"""
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
# Undo or finish one grant: give back budgets, release the slot and the entry.
|
|
530
|
+
_SETTLE = r"""
|
|
531
|
+
local function settle(alias, owner, rid, refund_r, refund_t, learn, outcome, latency)
|
|
532
|
+
local entry_key = mk(alias, 'inprog:' .. rid)
|
|
533
|
+
if redis.call('HGET', entry_key, 'owner') ~= owner then return 0 end
|
|
534
|
+
local dep = redis.call('HGET', entry_key, 'dep')
|
|
535
|
+
local cls = redis.call('HGET', entry_key, 'cls')
|
|
536
|
+
local granted_minute = hnum(entry_key, 'minute', minute)
|
|
537
|
+
local model, rows = cfg_of(alias)
|
|
538
|
+
local row = rows and rows[dep]
|
|
539
|
+
if row then
|
|
540
|
+
for _, b in ipairs(budgets(alias, row, model, cls, true)) do
|
|
541
|
+
local g = (b[4] == 'r') and refund_r or refund_t
|
|
542
|
+
give(b[1], b[2], b[3], g, row.rate_window, granted_minute)
|
|
543
|
+
end
|
|
544
|
+
local rkey = mk(alias, 'rate:' .. dep)
|
|
545
|
+
if granted_minute == minute and hnum(rkey, 'seen_win', -1) == minute then
|
|
546
|
+
redis.call('HINCRBY', rkey, 'seen_calls', -refund_r)
|
|
547
|
+
redis.call('HINCRBY', rkey, 'seen_tokens', -refund_t)
|
|
548
|
+
if cls == 'n' then redis.call('HINCRBY', rkey, 'seen_n', -refund_r) end
|
|
549
|
+
end
|
|
550
|
+
end
|
|
551
|
+
redis.call('ZREM', mk(alias, 'lease:' .. dep .. ':' .. cls), owner)
|
|
552
|
+
redis.call('ZREM', mk(alias, 'inprog'), owner)
|
|
553
|
+
-- A breaker probe that ended without a verdict (cut, given back, throttled) frees the probe
|
|
554
|
+
-- at once: it blocked every call for breaker_probe_secs, and they waited it out (scenario L4).
|
|
555
|
+
if outcome ~= 'ok' and outcome ~= 'fail' and redis.call('HGET', mk(alias, 'cb:' .. dep), 'probe') == owner then
|
|
556
|
+
redis.call('HSET', mk(alias, 'cb:' .. dep), 'probe', '')
|
|
557
|
+
end
|
|
558
|
+
redis.call('DEL', entry_key)
|
|
559
|
+
if learn and row then
|
|
560
|
+
local ac = mk(alias, 'ac:' .. dep)
|
|
561
|
+
local limit, cap = limit_of(alias, row)
|
|
562
|
+
if outcome == 'throttle' then
|
|
563
|
+
local thr = mk(alias, 'thr:' .. dep .. ':' .. minute)
|
|
564
|
+
redis.call('INCR', thr)
|
|
565
|
+
redis.call('PEXPIRE', thr, KEY_MS)
|
|
566
|
+
if hnum(ac, 'cut_at', 0) < now - num(T.limiter_cut_ms, 1000) then
|
|
567
|
+
redis.call('HSET', ac, 'limit', tostring(math.max(1, math.floor(limit * num(T.limiter_cut_factor, 0.7)))), 'cut_at', now)
|
|
568
|
+
end
|
|
569
|
+
elseif outcome == 'ok' then
|
|
570
|
+
redis.call('HSET', ac, 'limit', tostring(math.min(cap, limit + 1 / math.max(1, limit))))
|
|
571
|
+
end
|
|
572
|
+
local cb = mk(alias, 'cb:' .. dep)
|
|
573
|
+
if outcome == 'ok' then
|
|
574
|
+
redis.call('HSET', cb, 'state', 'closed', 'failures', 0, 'probe', '')
|
|
575
|
+
redis.call('DEL', mk(alias, 'terr:' .. dep))
|
|
576
|
+
elseif outcome == 'fail' then
|
|
577
|
+
-- Recent failures only: one more than 60 s after the last starts the count again (it
|
|
578
|
+
-- counted forever until a success, so 5 failures over hours opened the breaker).
|
|
579
|
+
if hnum(cb, 'fail_at', 0) < now - num(row.breaker_window_secs, 60) * 1000 then redis.call('HSET', cb, 'failures', 0) end
|
|
580
|
+
redis.call('HSET', cb, 'fail_at', now)
|
|
581
|
+
local fails = redis.call('HINCRBY', cb, 'failures', 1)
|
|
582
|
+
local probing = redis.call('HGET', cb, 'probe') == owner
|
|
583
|
+
if fails >= num(row.breaker_threshold, 5) or probing then
|
|
584
|
+
redis.call('HSET', cb, 'state', 'open', 'probe', '',
|
|
585
|
+
'until', now + math.floor(num(row.breaker_cooldown_secs, 15) * 1000))
|
|
586
|
+
end
|
|
587
|
+
end
|
|
588
|
+
if latency and latency >= 0 then
|
|
589
|
+
local health = mk(alias, 'health')
|
|
590
|
+
local old = hnum(health, dep .. '|lat', latency)
|
|
591
|
+
redis.call('HSET', health, dep .. '|lat', tostring(old * 0.8 + latency * 0.2))
|
|
592
|
+
redis.call('HINCRBY', health, dep .. '|n', 1)
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
return 1
|
|
596
|
+
end
|
|
597
|
+
"""
|
|
598
|
+
|
|
599
|
+
# req: alias, owner, request_id, refund_r, refund_t, outcome (ok|throttle|fail|none), latency_ms.
|
|
600
|
+
FINISH = (
|
|
601
|
+
_COMMON
|
|
602
|
+
+ _SETTLE
|
|
603
|
+
+ r"""
|
|
604
|
+
return settle(req.alias, req.owner, req.request_id, num(req.refund_r, 0), num(req.refund_t, 0),
|
|
605
|
+
true, req.outcome, num(req.latency_ms, -1))
|
|
606
|
+
"""
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
# req: alias, owner, request_id. Give everything back; no limiter or breaker learning.
|
|
610
|
+
UNDO = (
|
|
611
|
+
_COMMON
|
|
612
|
+
+ _SETTLE
|
|
613
|
+
+ r"""
|
|
614
|
+
local entry_key = mk(req.alias, 'inprog:' .. req.request_id)
|
|
615
|
+
if redis.call('HGET', entry_key, 'owner') ~= req.owner then return 0 end
|
|
616
|
+
return settle(req.alias, req.owner, req.request_id, hnum(entry_key, 'cost_r', 0),
|
|
617
|
+
hnum(entry_key, 'cost_t', 0), false, 'none', -1)
|
|
618
|
+
"""
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
# req: {lease_ms, worker, items: [{alias, owner, request_id}]}. Heartbeat of held grants.
|
|
622
|
+
RENEW = (
|
|
623
|
+
_COMMON
|
|
624
|
+
+ r"""
|
|
625
|
+
local renewed = 0
|
|
626
|
+
for _, item in ipairs(req.items) do
|
|
627
|
+
local entry_key = mk(item.alias, 'inprog:' .. item.request_id)
|
|
628
|
+
if redis.call('HGET', entry_key, 'owner') == item.owner then
|
|
629
|
+
local until_ = now + num(req.lease_ms, 60000)
|
|
630
|
+
local dep = redis.call('HGET', entry_key, 'dep')
|
|
631
|
+
local cls = redis.call('HGET', entry_key, 'cls')
|
|
632
|
+
-- No XX: a lease that expired during a missed beat comes back (the owner check above
|
|
633
|
+
-- guards it), so a still-running call holds its slot again.
|
|
634
|
+
redis.call('ZADD', mk(item.alias, 'lease:' .. dep .. ':' .. cls), until_, item.owner)
|
|
635
|
+
redis.call('ZADD', mk(item.alias, 'inprog'), 'XX', until_, item.owner)
|
|
636
|
+
redis.call('PEXPIRE', entry_key, num(req.lease_ms, 60000) * 2)
|
|
637
|
+
local job = P .. ':{r:' .. item.request_id .. '}:job'
|
|
638
|
+
if redis.call('EXISTS', job) == 1 then redis.call('HSET', job, 'hb', now) end
|
|
639
|
+
renewed = renewed + 1
|
|
640
|
+
end
|
|
641
|
+
end
|
|
642
|
+
return renewed
|
|
643
|
+
"""
|
|
644
|
+
)
|
|
645
|
+
|
|
646
|
+
# req: {lease_ms, items: [{alias, owner, request_id, dep, cls, cost_r, cost_t}]}. After a
|
|
647
|
+
# store reset, put back the slots and entries of calls still running (plan T10).
|
|
648
|
+
REREGISTER = (
|
|
649
|
+
_COMMON
|
|
650
|
+
+ r"""
|
|
651
|
+
local done = 0
|
|
652
|
+
for _, item in ipairs(req.items) do
|
|
653
|
+
local until_ = now + num(req.lease_ms, 60000)
|
|
654
|
+
redis.call('ZADD', mk(item.alias, 'lease:' .. item.dep .. ':' .. item.cls), until_, item.owner)
|
|
655
|
+
redis.call('ZADD', mk(item.alias, 'inprog'), until_, item.owner)
|
|
656
|
+
local entry_key = mk(item.alias, 'inprog:' .. item.request_id)
|
|
657
|
+
local worker = string.match(item.owner, '^.*|([^|]*)|[^|]*$') or ''
|
|
658
|
+
redis.call('HSET', entry_key, 'owner', item.owner, 'worker', worker, 'dep', item.dep,
|
|
659
|
+
'cls', item.cls, 'minute', minute, 'cost_r', item.cost_r, 'cost_t', item.cost_t,
|
|
660
|
+
'started', now, 'request', item.request_id, 'served', item.alias)
|
|
661
|
+
redis.call('PEXPIRE', entry_key, num(req.lease_ms, 60000) * 2)
|
|
662
|
+
done = done + 1
|
|
663
|
+
end
|
|
664
|
+
return done
|
|
665
|
+
"""
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
# req: {alias}. Remove in-progress entries whose owner stopped renewing (the dead worker's
|
|
669
|
+
# estimate stays charged for that minute; plan §3.18). Reply: how many were removed.
|
|
670
|
+
SWEEP = (
|
|
671
|
+
_COMMON
|
|
672
|
+
+ r"""
|
|
673
|
+
local removed = 0
|
|
674
|
+
local inprog = mk(req.alias, 'inprog')
|
|
675
|
+
for _, owner in ipairs(redis.call('ZRANGEBYSCORE', inprog, '-inf', now, 'LIMIT', 0, 200)) do
|
|
676
|
+
redis.call('ZREM', inprog, owner)
|
|
677
|
+
local rid = string.match(owner, '^(.*)|[^|]*|[^|]*$')
|
|
678
|
+
if rid then
|
|
679
|
+
local entry_key = mk(req.alias, 'inprog:' .. rid)
|
|
680
|
+
if redis.call('HGET', entry_key, 'owner') == owner then
|
|
681
|
+
local dep = redis.call('HGET', entry_key, 'dep')
|
|
682
|
+
local cls = redis.call('HGET', entry_key, 'cls')
|
|
683
|
+
if dep then
|
|
684
|
+
redis.call('ZREM', mk(req.alias, 'lease:' .. dep .. ':' .. cls), owner)
|
|
685
|
+
-- Its worker died mid-probe: free the probe (it blocked the breaker for 60 s).
|
|
686
|
+
local cb = mk(req.alias, 'cb:' .. dep)
|
|
687
|
+
if redis.call('HGET', cb, 'probe') == owner then redis.call('HSET', cb, 'probe', '') end
|
|
688
|
+
end
|
|
689
|
+
redis.call('DEL', entry_key)
|
|
690
|
+
end
|
|
691
|
+
end
|
|
692
|
+
removed = removed + 1
|
|
693
|
+
end
|
|
694
|
+
return removed
|
|
695
|
+
"""
|
|
696
|
+
)
|