llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,696 @@
1
+ """Lua of the admission scripts: PICK_TAKE, FINISH, UNDO, RENEW and SWEEP (plan §3.4, §3.18).
2
+
3
+ Each script takes the key prefix and one JSON argument and builds its key names itself, which
4
+ needs the single-primary store 0.3 supports. Times are the store's clock in ms. Budgets are GCRA
5
+ clocks (`smooth`) or minute counters (`fixed`); class clocks enforce the priority shares.
6
+ """
7
+
8
+ # ruff: noqa: E501 # Lua source in strings; wrapping its lines hurts reading it
9
+
10
+ from __future__ import annotations
11
+
12
+ _COMMON = r"""
13
+ local t = redis.call('TIME')
14
+ local now = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
15
+ local minute = math.floor(now / 60000)
16
+ local P = ARGV[1]
17
+ local req = cjson.decode(ARGV[2])
18
+ -- Timings and limits: `EngineTimings` (sent as `t` by every call) and the registry's breaker
19
+ -- settings (on each row); the numbers after them are the defaults.
20
+ local T = req.t or {}
21
+ local KEY_MS -- budget key lifetime: set below, once num() exists
22
+ local function mk(alias, name) return P .. ':{m:' .. alias .. '}:' .. name end
23
+ local function num(v, d) if v == nil or v == false or v == cjson.null then return d end return tonumber(v) end
24
+ local function hnum(key, field, d) return num(redis.call('HGET', key, field), d) end
25
+ local function present(v) return v ~= nil and v ~= false and v ~= cjson.null end
26
+ KEY_MS = num(T.budget_key_ms, 180000) -- budget and limiter keys live this long (> 2 minutes)
27
+
28
+ -- One budget: field prefix f, limit L per minute, burst fraction bf, window mode.
29
+ local function avail(key, f, L, bf, mode)
30
+ local burst = math.max(1, math.floor(L * bf))
31
+ if mode == 'fixed' then
32
+ local n = 0
33
+ if hnum(key, f .. '_win', -1) == minute then n = hnum(key, f .. '_n', 0) end
34
+ return L - n, n
35
+ end
36
+ local per = 60000 / L
37
+ local tat = math.max(hnum(key, f .. '_tat', now), now)
38
+ return (now + burst * per - tat) / per, tat
39
+ end
40
+
41
+ -- Room for cost c behind R reserved units: returns wait ms (0 = fits) and the new state.
42
+ local reset_minute = num(redis.call('GET', P .. ':reset_minute'), -1)
43
+ local function room(key, f, L, c, R, bf, mode)
44
+ local burst = math.max(1, math.floor(L * bf))
45
+ if mode == 'fixed' then
46
+ -- After a store reset, a fixed window holds until the next minute (plan T10).
47
+ if reset_minute == minute and hnum(key, f .. '_win', -1) ~= minute then
48
+ return (minute + 1) * 60000 - now, nil
49
+ end
50
+ local _, n = avail(key, f, L, bf, mode)
51
+ if n + R + c <= L or (n == 0 and R == 0 and c > L) then return 0, n + c end
52
+ return (minute + 1) * 60000 - now, nil
53
+ end
54
+ local per = 60000 / L
55
+ local tat = math.max(hnum(key, f .. '_tat', now), now)
56
+ if c > burst and R == 0 then
57
+ if tat <= now then return 0, now + c * per end
58
+ return tat - now, nil
59
+ end
60
+ local over = tat + (R + c) * per - now - burst * per
61
+ if over > 0 then return over, nil end
62
+ return 0, tat + c * per
63
+ end
64
+
65
+ local function write(key, f, mode, value)
66
+ if mode == 'fixed' then
67
+ redis.call('HSET', key, f .. '_win', minute, f .. '_n', value)
68
+ else
69
+ redis.call('HSET', key, f .. '_tat', tostring(value))
70
+ end
71
+ redis.call('PEXPIRE', key, KEY_MS)
72
+ end
73
+
74
+ -- Give back g units (negative g charges more). Fixed counters only inside the grant's minute.
75
+ local function give(key, f, L, g, mode, granted_minute)
76
+ if not L or L <= 0 or g == 0 then return end
77
+ if mode == 'fixed' then
78
+ if granted_minute ~= minute and g > 0 then return end
79
+ if hnum(key, f .. '_win', -1) ~= minute then
80
+ -- The minute turned: an overrun (g < 0) is charged to the new minute, not dropped.
81
+ if g < 0 then
82
+ redis.call('HSET', key, f .. '_win', minute, f .. '_n', -g)
83
+ redis.call('PEXPIRE', key, KEY_MS)
84
+ end
85
+ return
86
+ end
87
+ redis.call('HSET', key, f .. '_n', math.max(0, hnum(key, f .. '_n', 0) - g))
88
+ redis.call('PEXPIRE', key, KEY_MS)
89
+ return
90
+ end
91
+ local per = 60000 / L
92
+ local tat = math.max(hnum(key, f .. '_tat', now), now)
93
+ local value = tat - g * per
94
+ if g > 0 then value = math.max(value, now) end
95
+ redis.call('HSET', key, f .. '_tat', tostring(value))
96
+ redis.call('PEXPIRE', key, KEY_MS) -- the key may have expired since the grant
97
+ end
98
+
99
+ -- The budgets of a deployment row for class cls: {key, field, limit, cost kind}.
100
+ local function budgets(alias, row, model, cls, account)
101
+ local list = {}
102
+ local limits = row.rate_limits
103
+ local rkey = mk(alias, 'rate:' .. row.key)
104
+ local share = 1
105
+ if cls == 'n' then share = 1 - num(model.priority_reserve, 0) else share = 1 - num(model.normal_min_share, 0) end
106
+ local function add(key, f, L, kind)
107
+ if present(L) and tonumber(L) > 0 then table.insert(list, {key, f, tonumber(L), kind}) end
108
+ end
109
+ if limits == 'both' or limits == 'rpm' then
110
+ add(rkey, 'rpm', row.rpm, 'r')
111
+ if share < 1 and present(row.rpm) then add(rkey, cls .. '_rpm', row.rpm * share, 'r') end
112
+ end
113
+ if limits == 'both' or limits == 'tpm' then
114
+ add(rkey, 'tpm', row.tpm, 't')
115
+ if share < 1 and present(row.tpm) then add(rkey, cls .. '_tpm', row.tpm * share, 't') end
116
+ end
117
+ if account then
118
+ local akey = P .. ':{a:' .. row.account .. '}:rate'
119
+ local ashare = 1
120
+ if cls == 'n' then ashare = 1 - num(row.account_priority_reserve, 0) end
121
+ add(akey, 'rpm', row.shared_rpm, 'r')
122
+ add(akey, 'tpm', row.shared_tpm, 't')
123
+ if ashare < 1 then
124
+ if present(row.shared_rpm) then add(akey, 'n_rpm', row.shared_rpm * ashare, 'r') end
125
+ if present(row.shared_tpm) then add(akey, 'n_tpm', row.shared_tpm * ashare, 't') end
126
+ end
127
+ end
128
+ return list
129
+ end
130
+
131
+ local function lease_keys(alias, dep)
132
+ return mk(alias, 'lease:' .. dep .. ':p'), mk(alias, 'lease:' .. dep .. ':n')
133
+ end
134
+
135
+ local function limit_of(alias, row)
136
+ local key = mk(alias, 'ac:' .. row.key)
137
+ local cap = num(row.max_concurrency, 200)
138
+ local limit = hnum(key, 'limit', nil)
139
+ if limit == nil then
140
+ -- A new deployment takes its full share at once (plan E74).
141
+ limit = cap
142
+ redis.call('HSET', key, 'limit', tostring(limit))
143
+ end
144
+ return math.max(1, math.floor(math.min(limit, cap))), cap
145
+ end
146
+
147
+ local function cfg_of(alias)
148
+ local raw = redis.call('HGET', mk(alias, 'cfg'), 'numbers')
149
+ if not raw then return nil end
150
+ local doc = cjson.decode(raw)
151
+ local rows = {}
152
+ for _, row in ipairs(doc.deployments) do rows[row.key] = row end
153
+ return doc.model, rows, hnum(mk(alias, 'cfg'), 'changes', -1)
154
+ end
155
+ """
156
+
157
+ PICK_TAKE = (
158
+ _COMMON
159
+ + r"""
160
+ local alias = req.alias
161
+ local model, rows, changes = cfg_of(alias)
162
+ if model == nil then return cjson.encode({r = 'no_config'}) end
163
+ if changes ~= num(req.changes, -2) then return cjson.encode({r = 'stale_config', changes = changes}) end
164
+ local cls = req.cls
165
+ local rid = req.request_id
166
+ local entry_key = mk(alias, 'inprog:' .. rid)
167
+
168
+ -- Finished or cancelled? Checked for every ask: a fallback ask (claim off) once started a
169
+ -- call that was already cancelled or answered elsewhere. Only the claim write needs `claim`.
170
+ local job_key = P .. ':{r:' .. rid .. '}:job'
171
+ if redis.call('EXISTS', P .. ':{r:' .. rid .. '}:res') == 1 then return cjson.encode({r = 'done'}) end
172
+ if redis.call('EXISTS', P .. ':{r:' .. rid .. '}:cancel') == 1 then return cjson.encode({r = 'cancelled'}) end
173
+ local state = redis.call('HGET', job_key, 'state')
174
+ if state == 'done' or state == 'failed' or state == 'expired' or state == 'cancelled' then
175
+ return cjson.encode({r = 'done'})
176
+ end
177
+ -- Running under another worker with a fresh heartbeat (on another model, or between its
178
+ -- retries when no entry exists): a redelivered message is a duplicate, not a new run.
179
+ if req.claim and state == 'running' then
180
+ local runner = redis.call('HGET', job_key, 'owner')
181
+ if runner and runner ~= req.worker
182
+ and now - hnum(job_key, 'hb', 0) < num(req.lease_ms, 60000) then
183
+ return cjson.encode({r = 'duplicate'})
184
+ end
185
+ end
186
+ local held = redis.call('HGET', entry_key, 'owner')
187
+ if held then
188
+ if held == req.owner then
189
+ return cjson.encode({r = 'granted', dep = redis.call('HGET', entry_key, 'dep'),
190
+ minute = hnum(entry_key, 'minute', minute), repeat_ = true})
191
+ end
192
+ local holder = redis.call('HGET', entry_key, 'worker')
193
+ local hb = hnum(job_key, 'hb', 0)
194
+ local fresh = now - hb < num(req.lease_ms, 60000) and redis.call('HGET', entry_key, 'started') ~= false
195
+ if fresh then return cjson.encode({r = 'duplicate'}) end
196
+ -- A dead owner (no heartbeat for a lease): drop its slot and entry (no refund).
197
+ local old_dep = redis.call('HGET', entry_key, 'dep')
198
+ local old_cls = redis.call('HGET', entry_key, 'cls')
199
+ if old_dep then redis.call('ZREM', mk(alias, 'lease:' .. old_dep .. ':' .. old_cls), held) end
200
+ redis.call('ZREM', mk(alias, 'inprog'), held)
201
+ redis.call('DEL', entry_key)
202
+ end
203
+
204
+ -- Foreign call (model fallback): only while this model has no waiters of its own.
205
+ local foreign = present(req.foreign)
206
+ local wait_keys = {p = mk(alias, 'wait:p'), n = mk(alias, 'wait:n')}
207
+ local info_key = mk(alias, 'waitinfo')
208
+ for _, c in ipairs({'p', 'n'}) do
209
+ for _, id in ipairs(redis.call('ZRANGE', wait_keys[c], 0, 49)) do
210
+ local raw = redis.call('HGET', info_key, id)
211
+ if not raw or num(cjson.decode(raw).exp, 0) < now then
212
+ redis.call('ZREM', wait_keys[c], id)
213
+ redis.call('HDEL', info_key, id)
214
+ end
215
+ end
216
+ end
217
+ if foreign and (redis.call('ZCARD', wait_keys.p) + redis.call('ZCARD', wait_keys.n)) > 0 then
218
+ return cjson.encode({r = 'busy', reason = 'waiters'})
219
+ end
220
+
221
+ -- What older waiters set aside, per deployment (plan §3.18 reservation rule).
222
+ local reserved = {}
223
+ local function reserve_from(list, into)
224
+ into = into or reserved
225
+ for _, id in ipairs(list) do
226
+ if id ~= rid then
227
+ local raw = redis.call('HGET', info_key, id)
228
+ if raw then
229
+ local w = cjson.decode(raw)
230
+ if num(w.exp, 0) >= now then
231
+ for _, dep in ipairs(w.deps) do
232
+ local slot = into[dep] or {r = 0, t = 0, s = 0}
233
+ slot.r = slot.r + num(w.r, 1)
234
+ slot.t = slot.t + num(w.t, 0)
235
+ slot.s = slot.s + 1
236
+ into[dep] = slot
237
+ end
238
+ end
239
+ end
240
+ end
241
+ end
242
+ end
243
+ local pos = num(req.pos, now * 1000)
244
+ if not foreign then
245
+ -- %.17g: plain `..` formats with %.14g and misorders waiters within ~0.1 ms of each other.
246
+ local before = '(' .. string.format('%.17g', pos)
247
+ reserve_from(redis.call('ZRANGEBYSCORE', wait_keys[cls], '-inf', before, 'LIMIT', 0, 200))
248
+ end
249
+ -- What waiting priority calls need, per deployment: added to a normal call's reservation
250
+ -- per candidate (only where that candidate's normal use reached its floor), never to the shared
251
+ -- table, so the result no longer depends on the order the candidates are tried in.
252
+ local priority_reserved = {}
253
+ if cls == 'n' and not foreign then
254
+ reserve_from(redis.call('ZRANGE', wait_keys.p, 0, 199), priority_reserved)
255
+ end
256
+
257
+ local cost_r = num(req.cost_r, 1)
258
+ local cost_t = num(req.cost_t, 0)
259
+ local best_wait, best_reason = nil, nil
260
+ -- A breaker's one probe call is in flight: its answer comes within an attempt, so a call
261
+ -- waits for it instead of failing as `model_unavailable` (scenario P7 at 300 %: ten calls
262
+ -- that arrived during the probe failed after every region had recovered).
263
+ local probe_wait = false
264
+ local function note(ms, reason)
265
+ if best_wait == nil or ms < best_wait then best_wait, best_reason = ms, reason end
266
+ end
267
+
268
+ -- Evaluate one candidate: fits (with its new budget states and headroom) or the wait.
269
+ local function evaluate(cand)
270
+ local row = rows[cand.key]
271
+ if row == nil then return nil end
272
+ local health = mk(alias, 'health')
273
+ if row.enabled == false or row.account_enabled == false or row.outside_residency or row.blocked
274
+ or redis.call('HEXISTS', health, cand.key .. '|until') == 1 then
275
+ return nil
276
+ end
277
+ local cb = mk(alias, 'cb:' .. cand.key)
278
+ local probe = false
279
+ if redis.call('HGET', cb, 'state') == 'open' then
280
+ if hnum(cb, 'until', 0) > now then return nil end
281
+ local who = redis.call('HGET', cb, 'probe')
282
+ if who and who ~= '' and hnum(cb, 'probe_at', 0) > now - num(row.breaker_probe_secs, 60) * 1000 then
283
+ probe_wait = true
284
+ return nil
285
+ end
286
+ probe = true
287
+ end
288
+ local res = reserved[cand.key] or {r = 0, t = 0, s = 0}
289
+ -- Normal calls also set aside what waiting priority calls need, unless normal use is below
290
+ -- normal_min_share this minute.
291
+ local rkey = mk(alias, 'rate:' .. cand.key)
292
+ local prio = priority_reserved[cand.key]
293
+ if prio then
294
+ local floor = num(model.normal_min_share, 0) * num(row.rpm, 0)
295
+ local used = 0
296
+ if hnum(rkey, 'seen_win', -1) == minute then used = hnum(rkey, 'seen_n', 0) end
297
+ if floor <= 0 or used >= floor then
298
+ res = {r = res.r + prio.r, t = res.t + prio.t, s = res.s + prio.s}
299
+ end
300
+ end
301
+ local states = {}
302
+ local headroom = nil
303
+ local most = nil
304
+ for _, b in ipairs(budgets(alias, row, model, cls, true)) do
305
+ local cost = cost_r
306
+ local R = res.r
307
+ if b[4] == 't' then cost, R = cost_t, res.t end
308
+ local mode = row.rate_window
309
+ local wait, value = room(b[1], b[2], b[3], cost, R, num(row.rate_burst, 0.1), mode)
310
+ if wait > 0 then
311
+ local reason = (b[4] == 'r') and 'rpm' or 'tpm'
312
+ if string.find(b[1], '{a:', 1, true) then reason = 'account'
313
+ elseif string.sub(b[2], 2, 2) == '_' then reason = 'reserve' end
314
+ if R > 0 then reason = 'not_head' end
315
+ return {wait = wait, reason = reason}
316
+ end
317
+ table.insert(states, {b[1], b[2], mode, value})
318
+ local free = avail(b[1], b[2], b[3], num(row.rate_burst, 0.1), mode) - R
319
+ local left = free - cost
320
+ -- Headroom is the share free before this call of what can be taken now: the minute (fixed)
321
+ -- or the bucket (smooth, never more than rate_burst of the minute). Before, not after: a
322
+ -- call larger than the bucket could otherwise never fall back to an idle model.
323
+ local size = b[3]
324
+ if mode ~= 'fixed' then size = math.max(1, math.floor(b[3] * num(row.rate_burst, 0.1))) end
325
+ local frac = free / size
326
+ if headroom == nil or frac < headroom then headroom = frac end
327
+ if b[4] == 'r' and not string.find(b[1], '{a:', 1, true) then most = left end
328
+ end
329
+ -- Concurrency: slots per class, after what older waiters set aside.
330
+ local lp, ln = lease_keys(alias, cand.key)
331
+ redis.call('ZREMRANGEBYSCORE', lp, '-inf', now)
332
+ redis.call('ZREMRANGEBYSCORE', ln, '-inf', now)
333
+ local in_p, in_n = redis.call('ZCARD', lp), redis.call('ZCARD', ln)
334
+ local limit = limit_of(alias, row)
335
+ local share = 1 - num(model.priority_reserve, 0)
336
+ if cls == 'p' then share = 1 - num(model.normal_min_share, 0) end
337
+ local cap = math.floor(share * limit)
338
+ if limit >= 2 and share > 0 then cap = math.max(1, cap) end
339
+ local mine = (cls == 'p') and in_p or in_n
340
+ if in_p + in_n + res.s >= limit or mine >= cap then
341
+ return {wait = num(T.slot_wait_ms, 50) + math.floor(num(req.rand, 0.5) * num(T.slot_jitter_ms, 150)), reason = 'slot'}
342
+ end
343
+ local slot_frac = (limit - in_p - in_n - res.s - 1) / limit
344
+ if headroom == nil or slot_frac < headroom then headroom = slot_frac end
345
+ return {fits = true, states = states, headroom = headroom, most = most or (limit - in_p - in_n),
346
+ probe = probe, limit = limit}
347
+ end
348
+
349
+ -- Sticky session: keep the session's deployment while it fits or would wait only briefly.
350
+ local sticky_key = nil
351
+ local sticky_dep = nil
352
+ if present(req.sticky) then
353
+ sticky_key = mk(alias, 'sticky:' .. req.sticky.hash)
354
+ sticky_dep = redis.call('GET', sticky_key)
355
+ end
356
+
357
+ -- Latency-aware exclusion (plan §3.4): a deployment much slower than the model's median,
358
+ -- with enough samples and at least 3 to compare, becomes a last choice (never excluded alone).
359
+ local slow = {}
360
+ do
361
+ local hs = model.health or {}
362
+ local ratio, min_ms, min_n = num(hs.slow_ratio, 3), num(hs.slow_min_secs, 5) * 1000, num(hs.slow_min_samples, 20)
363
+ local health_key = mk(alias, 'health')
364
+ local values, lat_of = {}, {}
365
+ for _, cand in ipairs(req.candidates) do
366
+ local lat = hnum(health_key, cand.key .. '|lat', nil)
367
+ if lat and hnum(health_key, cand.key .. '|n', 0) >= min_n then
368
+ lat_of[cand.key] = lat
369
+ table.insert(values, lat)
370
+ end
371
+ end
372
+ if #values >= 3 then
373
+ table.sort(values)
374
+ local median = values[math.floor((#values + 1) / 2)]
375
+ for key, lat in pairs(lat_of) do
376
+ if lat > ratio * median and lat > min_ms then slow[key] = true end
377
+ end
378
+ end
379
+ end
380
+
381
+ local fitting = {confirmed = {}, fallback = {}}
382
+ local last = {}
383
+ local results = {}
384
+ local any_usable = false
385
+ -- `ordered` (the default) takes the first confirmed candidate that fits, so evaluation stops
386
+ -- there: evaluating all of them cost ~12 store reads each (30 deployments: p99 above 2 ms).
387
+ local first_fit = req.strategy == 'ordered' and not present(req.sticky)
388
+ for _, cand in ipairs(req.candidates) do
389
+ local res = evaluate(cand)
390
+ if res ~= nil then
391
+ any_usable = true
392
+ results[cand.key] = res
393
+ if res.fits then
394
+ if cand.last or slow[cand.key] then table.insert(last, cand) else table.insert(fitting[cand.tier], cand) end
395
+ if first_fit and cand.tier == 'confirmed' and not (cand.last or slow[cand.key]) then break end
396
+ else
397
+ note(res.wait, res.reason)
398
+ end
399
+ end
400
+ end
401
+ if not any_usable then
402
+ if probe_wait then return cjson.encode({r = 'wait', ms = num(T.probe_wait_ms, 200), reason = 'probe'}) end
403
+ return cjson.encode({r = 'none'})
404
+ end
405
+
406
+ local chosen = nil
407
+ if sticky_dep and results[sticky_dep] then
408
+ local s = results[sticky_dep]
409
+ if s.fits then
410
+ for _, cand in ipairs(req.candidates) do if cand.key == sticky_dep then chosen = cand end end
411
+ elseif s.wait <= num(req.sticky.move_ms, 1000) then
412
+ best_wait, best_reason = s.wait, s.reason
413
+ chosen = false
414
+ end
415
+ end
416
+
417
+ local function pick(list)
418
+ if #list == 0 then return nil end
419
+ local strategy = req.strategy
420
+ if strategy == 'round_robin' then
421
+ local rot = mk(alias, 'rot')
422
+ local best, best_seq = nil, nil
423
+ for _, cand in ipairs(list) do
424
+ local seq = hnum(rot, 'rr|' .. cand.key, 0)
425
+ if best_seq == nil or seq < best_seq then best, best_seq = cand, seq end
426
+ end
427
+ return best
428
+ elseif strategy == 'weighted' then
429
+ local rot = mk(alias, 'rot')
430
+ local total, best, best_cw = 0, nil, nil
431
+ for _, cand in ipairs(list) do
432
+ local w = num(rows[cand.key].weight, 1)
433
+ local cw = hnum(rot, 'cw|' .. cand.key, 0) + w
434
+ redis.call('HSET', rot, 'cw|' .. cand.key, tostring(cw))
435
+ total = total + w
436
+ if best_cw == nil or cw > best_cw then best, best_cw = cand, cw end
437
+ end
438
+ redis.call('HSET', rot, 'cw|' .. best.key, tostring(best_cw - total))
439
+ return best
440
+ elseif strategy == 'headroom' then
441
+ local mode = req.pick or 'percent'
442
+ if mode == 'spread' then
443
+ local total = 0
444
+ for _, cand in ipairs(list) do total = total + math.max(0.0001, results[cand.key].headroom) end
445
+ local x = num(req.rand, 0.5) * total
446
+ for _, cand in ipairs(list) do
447
+ x = x - math.max(0.0001, results[cand.key].headroom)
448
+ if x <= 0 then return cand end
449
+ end
450
+ return list[#list]
451
+ end
452
+ local best, best_score = nil, nil
453
+ for _, cand in ipairs(list) do
454
+ local score = (mode == 'max') and results[cand.key].most or results[cand.key].headroom
455
+ if best_score == nil or score > best_score then best, best_score = cand, score end
456
+ end
457
+ return best
458
+ end
459
+ return list[1]
460
+ end
461
+
462
+ if chosen == nil then
463
+ chosen = pick(fitting.confirmed) or pick(fitting.fallback) or pick(last)
464
+ end
465
+
466
+ if foreign and chosen and num(req.foreign.require_pct, 0) > 0 then
467
+ if results[chosen.key].headroom < num(req.foreign.require_pct, 0) / 100 then chosen = nil end
468
+ end
469
+
470
+ if not chosen then
471
+ if foreign then return cjson.encode({r = 'busy', reason = 'headroom'}) end
472
+ -- Refused: register (or refresh) as a waiter, with the deployments it can use.
473
+ local deps = {}
474
+ for _, cand in ipairs(req.candidates) do table.insert(deps, cand.key) end
475
+ redis.call('ZADD', wait_keys[cls], pos, rid)
476
+ redis.call('HSET', info_key, rid, cjson.encode({r = cost_r, t = cost_t, deps = deps,
477
+ exp = now + num(req.waiter_ms, 2000)}))
478
+ return cjson.encode({r = 'wait', ms = math.max(1, math.floor(best_wait or num(T.wait_ms, 100))),
479
+ reason = best_reason or 'slot'})
480
+ end
481
+
482
+ -- Take everything at once.
483
+ local row = rows[chosen.key]
484
+ local res = results[chosen.key]
485
+ for _, s in ipairs(res.states) do write(s[1], s[2], s[3], s[4]) end
486
+ local rkey = mk(alias, 'rate:' .. chosen.key)
487
+ if hnum(rkey, 'seen_win', -1) ~= minute then
488
+ redis.call('HSET', rkey, 'seen_win', minute, 'seen_calls', 0, 'seen_tokens', 0, 'seen_n', 0)
489
+ redis.call('PEXPIRE', rkey, KEY_MS)
490
+ end
491
+ redis.call('HINCRBY', rkey, 'seen_calls', cost_r)
492
+ redis.call('HINCRBY', rkey, 'seen_tokens', cost_t)
493
+ if cls == 'n' then redis.call('HINCRBY', rkey, 'seen_n', cost_r) end
494
+ local lp, ln = lease_keys(alias, chosen.key)
495
+ local lease_ms = num(req.lease_ms, 60000)
496
+ redis.call('ZADD', (cls == 'p') and lp or ln, now + lease_ms, req.owner)
497
+ redis.call('ZADD', mk(alias, 'inprog'), now + lease_ms, req.owner)
498
+ redis.call('HSET', entry_key, 'owner', req.owner, 'worker', req.worker, 'dep', chosen.key,
499
+ 'cls', cls, 'minute', minute, 'cost_r', cost_r, 'cost_t', cost_t, 'account', row.account,
500
+ 'started', now, 'request', rid, 'served', alias)
501
+ redis.call('PEXPIRE', entry_key, lease_ms * 2)
502
+ if res.probe then
503
+ local cb = mk(alias, 'cb:' .. chosen.key)
504
+ redis.call('HSET', cb, 'probe', req.owner, 'probe_at', now)
505
+ end
506
+ -- Running, by this worker: a fallback grant too, so a redelivery of the call is refused
507
+ -- while it runs on the other model (it was granted again and ran twice).
508
+ if req.claim or redis.call('EXISTS', job_key) == 1 then
509
+ redis.call('HSET', job_key, 'state', 'running', 'owner', req.worker, 'hb', now)
510
+ end
511
+ redis.call('ZREM', wait_keys[cls], rid)
512
+ redis.call('HDEL', info_key, rid)
513
+ if foreign and present(req.foreign.from_alias) then
514
+ redis.call('ZREM', mk(req.foreign.from_alias, 'wait:' .. cls), rid)
515
+ redis.call('HDEL', mk(req.foreign.from_alias, 'waitinfo'), rid)
516
+ end
517
+ if sticky_key then
518
+ redis.call('SET', sticky_key, chosen.key, 'PX', num(req.sticky.ttl_ms, 1800000))
519
+ end
520
+ if req.strategy == 'round_robin' then
521
+ local rot = mk(alias, 'rot')
522
+ redis.call('HSET', rot, 'rr|' .. chosen.key, redis.call('HINCRBY', rot, 'rr_seq', 1))
523
+ end
524
+ return cjson.encode({r = 'granted', dep = chosen.key, minute = minute, limit = res.limit,
525
+ tier = chosen.tier})
526
+ """
527
+ )
528
+
529
+ # Undo or finish one grant: give back budgets, release the slot and the entry.
530
+ _SETTLE = r"""
531
+ local function settle(alias, owner, rid, refund_r, refund_t, learn, outcome, latency)
532
+ local entry_key = mk(alias, 'inprog:' .. rid)
533
+ if redis.call('HGET', entry_key, 'owner') ~= owner then return 0 end
534
+ local dep = redis.call('HGET', entry_key, 'dep')
535
+ local cls = redis.call('HGET', entry_key, 'cls')
536
+ local granted_minute = hnum(entry_key, 'minute', minute)
537
+ local model, rows = cfg_of(alias)
538
+ local row = rows and rows[dep]
539
+ if row then
540
+ for _, b in ipairs(budgets(alias, row, model, cls, true)) do
541
+ local g = (b[4] == 'r') and refund_r or refund_t
542
+ give(b[1], b[2], b[3], g, row.rate_window, granted_minute)
543
+ end
544
+ local rkey = mk(alias, 'rate:' .. dep)
545
+ if granted_minute == minute and hnum(rkey, 'seen_win', -1) == minute then
546
+ redis.call('HINCRBY', rkey, 'seen_calls', -refund_r)
547
+ redis.call('HINCRBY', rkey, 'seen_tokens', -refund_t)
548
+ if cls == 'n' then redis.call('HINCRBY', rkey, 'seen_n', -refund_r) end
549
+ end
550
+ end
551
+ redis.call('ZREM', mk(alias, 'lease:' .. dep .. ':' .. cls), owner)
552
+ redis.call('ZREM', mk(alias, 'inprog'), owner)
553
+ -- A breaker probe that ended without a verdict (cut, given back, throttled) frees the probe
554
+ -- at once: it blocked every call for breaker_probe_secs, and they waited it out (scenario L4).
555
+ if outcome ~= 'ok' and outcome ~= 'fail' and redis.call('HGET', mk(alias, 'cb:' .. dep), 'probe') == owner then
556
+ redis.call('HSET', mk(alias, 'cb:' .. dep), 'probe', '')
557
+ end
558
+ redis.call('DEL', entry_key)
559
+ if learn and row then
560
+ local ac = mk(alias, 'ac:' .. dep)
561
+ local limit, cap = limit_of(alias, row)
562
+ if outcome == 'throttle' then
563
+ local thr = mk(alias, 'thr:' .. dep .. ':' .. minute)
564
+ redis.call('INCR', thr)
565
+ redis.call('PEXPIRE', thr, KEY_MS)
566
+ if hnum(ac, 'cut_at', 0) < now - num(T.limiter_cut_ms, 1000) then
567
+ redis.call('HSET', ac, 'limit', tostring(math.max(1, math.floor(limit * num(T.limiter_cut_factor, 0.7)))), 'cut_at', now)
568
+ end
569
+ elseif outcome == 'ok' then
570
+ redis.call('HSET', ac, 'limit', tostring(math.min(cap, limit + 1 / math.max(1, limit))))
571
+ end
572
+ local cb = mk(alias, 'cb:' .. dep)
573
+ if outcome == 'ok' then
574
+ redis.call('HSET', cb, 'state', 'closed', 'failures', 0, 'probe', '')
575
+ redis.call('DEL', mk(alias, 'terr:' .. dep))
576
+ elseif outcome == 'fail' then
577
+ -- Recent failures only: one more than 60 s after the last starts the count again (it
578
+ -- counted forever until a success, so 5 failures over hours opened the breaker).
579
+ if hnum(cb, 'fail_at', 0) < now - num(row.breaker_window_secs, 60) * 1000 then redis.call('HSET', cb, 'failures', 0) end
580
+ redis.call('HSET', cb, 'fail_at', now)
581
+ local fails = redis.call('HINCRBY', cb, 'failures', 1)
582
+ local probing = redis.call('HGET', cb, 'probe') == owner
583
+ if fails >= num(row.breaker_threshold, 5) or probing then
584
+ redis.call('HSET', cb, 'state', 'open', 'probe', '',
585
+ 'until', now + math.floor(num(row.breaker_cooldown_secs, 15) * 1000))
586
+ end
587
+ end
588
+ if latency and latency >= 0 then
589
+ local health = mk(alias, 'health')
590
+ local old = hnum(health, dep .. '|lat', latency)
591
+ redis.call('HSET', health, dep .. '|lat', tostring(old * 0.8 + latency * 0.2))
592
+ redis.call('HINCRBY', health, dep .. '|n', 1)
593
+ end
594
+ end
595
+ return 1
596
+ end
597
+ """
598
+
599
+ # req: alias, owner, request_id, refund_r, refund_t, outcome (ok|throttle|fail|none), latency_ms.
600
+ FINISH = (
601
+ _COMMON
602
+ + _SETTLE
603
+ + r"""
604
+ return settle(req.alias, req.owner, req.request_id, num(req.refund_r, 0), num(req.refund_t, 0),
605
+ true, req.outcome, num(req.latency_ms, -1))
606
+ """
607
+ )
608
+
609
+ # req: alias, owner, request_id. Give everything back; no limiter or breaker learning.
610
+ UNDO = (
611
+ _COMMON
612
+ + _SETTLE
613
+ + r"""
614
+ local entry_key = mk(req.alias, 'inprog:' .. req.request_id)
615
+ if redis.call('HGET', entry_key, 'owner') ~= req.owner then return 0 end
616
+ return settle(req.alias, req.owner, req.request_id, hnum(entry_key, 'cost_r', 0),
617
+ hnum(entry_key, 'cost_t', 0), false, 'none', -1)
618
+ """
619
+ )
620
+
621
+ # req: {lease_ms, worker, items: [{alias, owner, request_id}]}. Heartbeat of held grants.
622
+ RENEW = (
623
+ _COMMON
624
+ + r"""
625
+ local renewed = 0
626
+ for _, item in ipairs(req.items) do
627
+ local entry_key = mk(item.alias, 'inprog:' .. item.request_id)
628
+ if redis.call('HGET', entry_key, 'owner') == item.owner then
629
+ local until_ = now + num(req.lease_ms, 60000)
630
+ local dep = redis.call('HGET', entry_key, 'dep')
631
+ local cls = redis.call('HGET', entry_key, 'cls')
632
+ -- No XX: a lease that expired during a missed beat comes back (the owner check above
633
+ -- guards it), so a still-running call holds its slot again.
634
+ redis.call('ZADD', mk(item.alias, 'lease:' .. dep .. ':' .. cls), until_, item.owner)
635
+ redis.call('ZADD', mk(item.alias, 'inprog'), 'XX', until_, item.owner)
636
+ redis.call('PEXPIRE', entry_key, num(req.lease_ms, 60000) * 2)
637
+ local job = P .. ':{r:' .. item.request_id .. '}:job'
638
+ if redis.call('EXISTS', job) == 1 then redis.call('HSET', job, 'hb', now) end
639
+ renewed = renewed + 1
640
+ end
641
+ end
642
+ return renewed
643
+ """
644
+ )
645
+
646
+ # req: {lease_ms, items: [{alias, owner, request_id, dep, cls, cost_r, cost_t}]}. After a
647
+ # store reset, put back the slots and entries of calls still running (plan T10).
648
+ REREGISTER = (
649
+ _COMMON
650
+ + r"""
651
+ local done = 0
652
+ for _, item in ipairs(req.items) do
653
+ local until_ = now + num(req.lease_ms, 60000)
654
+ redis.call('ZADD', mk(item.alias, 'lease:' .. item.dep .. ':' .. item.cls), until_, item.owner)
655
+ redis.call('ZADD', mk(item.alias, 'inprog'), until_, item.owner)
656
+ local entry_key = mk(item.alias, 'inprog:' .. item.request_id)
657
+ local worker = string.match(item.owner, '^.*|([^|]*)|[^|]*$') or ''
658
+ redis.call('HSET', entry_key, 'owner', item.owner, 'worker', worker, 'dep', item.dep,
659
+ 'cls', item.cls, 'minute', minute, 'cost_r', item.cost_r, 'cost_t', item.cost_t,
660
+ 'started', now, 'request', item.request_id, 'served', item.alias)
661
+ redis.call('PEXPIRE', entry_key, num(req.lease_ms, 60000) * 2)
662
+ done = done + 1
663
+ end
664
+ return done
665
+ """
666
+ )
667
+
668
+ # req: {alias}. Remove in-progress entries whose owner stopped renewing (the dead worker's
669
+ # estimate stays charged for that minute; plan §3.18). Reply: how many were removed.
670
+ SWEEP = (
671
+ _COMMON
672
+ + r"""
673
+ local removed = 0
674
+ local inprog = mk(req.alias, 'inprog')
675
+ for _, owner in ipairs(redis.call('ZRANGEBYSCORE', inprog, '-inf', now, 'LIMIT', 0, 200)) do
676
+ redis.call('ZREM', inprog, owner)
677
+ local rid = string.match(owner, '^(.*)|[^|]*|[^|]*$')
678
+ if rid then
679
+ local entry_key = mk(req.alias, 'inprog:' .. rid)
680
+ if redis.call('HGET', entry_key, 'owner') == owner then
681
+ local dep = redis.call('HGET', entry_key, 'dep')
682
+ local cls = redis.call('HGET', entry_key, 'cls')
683
+ if dep then
684
+ redis.call('ZREM', mk(req.alias, 'lease:' .. dep .. ':' .. cls), owner)
685
+ -- Its worker died mid-probe: free the probe (it blocked the breaker for 60 s).
686
+ local cb = mk(req.alias, 'cb:' .. dep)
687
+ if redis.call('HGET', cb, 'probe') == owner then redis.call('HSET', cb, 'probe', '') end
688
+ end
689
+ redis.call('DEL', entry_key)
690
+ end
691
+ end
692
+ removed = removed + 1
693
+ end
694
+ return removed
695
+ """
696
+ )