llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""Per-request records: the duplicate-id claim, the result and the request status.
|
|
2
|
+
|
|
3
|
+
A call's record is written only if absent when it is sent and confirmed only after the send
|
|
4
|
+
succeeded, so a failed send never leaves a phantom (plan §3.18 R23). Results are written once
|
|
5
|
+
(and announced to the caller in the same script); a library id's result is deleted on its
|
|
6
|
+
first read, an app id's kept `result_ttl_secs`. A final status is never replaced. Scripts use
|
|
7
|
+
the store's clock.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
import orjson
|
|
17
|
+
|
|
18
|
+
from llm_queue.engine.envelope import Reply
|
|
19
|
+
from llm_queue.engine.keys import Keys
|
|
20
|
+
from llm_queue.engine.offload import MARKER, ResultOffload
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
import redis.asyncio
|
|
24
|
+
|
|
25
|
+
FINAL_STATES = ("done", "failed", "expired", "cancelled")
|
|
26
|
+
|
|
27
|
+
# KEYS: callers, epoch. ARGV: caller key prefix, caller ttl secs. Reply: 1 when any caller of
|
|
28
|
+
# the call is alive, or the store came back less than a TTL ago (keys not renewed yet).
|
|
29
|
+
CALLERS_ALIVE = """
|
|
30
|
+
local t = redis.call('TIME')
|
|
31
|
+
local now = tonumber(t[1])
|
|
32
|
+
local epoch = tonumber(redis.call('GET', KEYS[2]) or '')
|
|
33
|
+
if not epoch or now - epoch < tonumber(ARGV[2]) then return 1 end
|
|
34
|
+
local callers = redis.call('SMEMBERS', KEYS[1])
|
|
35
|
+
if #callers == 0 then return 1 end
|
|
36
|
+
for _, caller in ipairs(callers) do
|
|
37
|
+
if redis.call('EXISTS', ARGV[1] .. caller) == 1 then return 1 end
|
|
38
|
+
end
|
|
39
|
+
return 0
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
# KEYS: job, callers, status, result. ARGV: user hash, params hash, caller, unconfirmed ttl,
|
|
43
|
+
# status ttl, model, priority, takeover grace secs.
|
|
44
|
+
# Reply: new | attached | takeover | done | expired | conflict.
|
|
45
|
+
REGISTER = """
|
|
46
|
+
local job = redis.call('HGETALL', KEYS[1])
|
|
47
|
+
local clock = redis.call('TIME')
|
|
48
|
+
local now = tonumber(clock[1])
|
|
49
|
+
if #job == 0 then
|
|
50
|
+
redis.call('HSET', KEYS[1], 'state', 'queued', 'user', ARGV[1], 'params', ARGV[2],
|
|
51
|
+
'caller', ARGV[3], 'confirmed', '0', 'created', now)
|
|
52
|
+
redis.call('EXPIRE', KEYS[1], tonumber(ARGV[4]))
|
|
53
|
+
redis.call('SADD', KEYS[2], ARGV[3])
|
|
54
|
+
redis.call('EXPIRE', KEYS[2], tonumber(ARGV[4]))
|
|
55
|
+
if redis.call('HGET', KEYS[3], 'state') == false then
|
|
56
|
+
local t = redis.call('TIME')
|
|
57
|
+
redis.call('HSET', KEYS[3], 'state', 'waiting', 'model', ARGV[6], 'priority', ARGV[7],
|
|
58
|
+
'enqueued_at', t[1] .. '.' .. string.format('%06d', tonumber(t[2])))
|
|
59
|
+
redis.call('EXPIRE', KEYS[3], tonumber(ARGV[5]))
|
|
60
|
+
end
|
|
61
|
+
return 'new'
|
|
62
|
+
end
|
|
63
|
+
local fields = {}
|
|
64
|
+
for i = 1, #job, 2 do fields[job[i]] = job[i + 1] end
|
|
65
|
+
if fields['params'] == nil then
|
|
66
|
+
-- A record a worker's claim made (its caller's record was gone): the call is running;
|
|
67
|
+
-- attach to it rather than call it a conflict.
|
|
68
|
+
redis.call('HSET', KEYS[1], 'user', ARGV[1], 'params', ARGV[2], 'caller', ARGV[3],
|
|
69
|
+
'confirmed', '1', 'created', now)
|
|
70
|
+
redis.call('SADD', KEYS[2], ARGV[3])
|
|
71
|
+
return 'attached'
|
|
72
|
+
end
|
|
73
|
+
if fields['user'] ~= ARGV[1] or fields['params'] ~= ARGV[2] then return 'conflict' end
|
|
74
|
+
local state = fields['state']
|
|
75
|
+
if state == 'done' or state == 'failed' or state == 'expired' or state == 'cancelled' then
|
|
76
|
+
if redis.call('EXISTS', KEYS[4]) == 1 then return 'done' end
|
|
77
|
+
return 'expired'
|
|
78
|
+
end
|
|
79
|
+
-- An unconfirmed record older than the grace: its send failed or its sender died; take over.
|
|
80
|
+
if fields['confirmed'] == '0' and now - tonumber(fields['created'] or '0') >= tonumber(ARGV[8]) then
|
|
81
|
+
redis.call('HSET', KEYS[1], 'caller', ARGV[3], 'created', now)
|
|
82
|
+
redis.call('SADD', KEYS[2], ARGV[3])
|
|
83
|
+
return 'takeover'
|
|
84
|
+
end
|
|
85
|
+
redis.call('SADD', KEYS[2], ARGV[3])
|
|
86
|
+
return 'attached'
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
# KEYS: job, callers. ARGV: keep secs. The send succeeded: the record lives until the deadline
|
|
90
|
+
# plus the status TTL.
|
|
91
|
+
CONFIRM = """
|
|
92
|
+
redis.call('HSET', KEYS[1], 'confirmed', '1')
|
|
93
|
+
redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1]))
|
|
94
|
+
redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1]))
|
|
95
|
+
return 1
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
# KEYS: job, callers, status. ARGV: caller. The send failed: drop the unconfirmed record.
|
|
99
|
+
ABANDON = """
|
|
100
|
+
local confirmed = redis.call('HGET', KEYS[1], 'confirmed')
|
|
101
|
+
if confirmed == '0' and redis.call('HGET', KEYS[1], 'caller') == ARGV[1] then
|
|
102
|
+
redis.call('DEL', KEYS[1], KEYS[2], KEYS[3])
|
|
103
|
+
return 1
|
|
104
|
+
end
|
|
105
|
+
return 0
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
# KEYS: result, job, callers. ARGV: reply JSON, result ttl, final state, keep secs, channel
|
|
109
|
+
# prefix (`<prefix>:done:`), request id. Writes the result only if none exists, marks the claim
|
|
110
|
+
# final and wakes every caller waiting on this id. Reply: 1 written, 0 a result already existed.
|
|
111
|
+
RESULT_PUT = """
|
|
112
|
+
local written = redis.call('SET', KEYS[1], ARGV[1], 'NX', 'EX', tonumber(ARGV[2]))
|
|
113
|
+
if redis.call('EXISTS', KEYS[2]) == 1 then
|
|
114
|
+
redis.call('HSET', KEYS[2], 'state', ARGV[3])
|
|
115
|
+
redis.call('EXPIRE', KEYS[2], tonumber(ARGV[4]))
|
|
116
|
+
end
|
|
117
|
+
for _, caller in ipairs(redis.call('SMEMBERS', KEYS[3])) do
|
|
118
|
+
redis.call('PUBLISH', ARGV[5] .. caller, ARGV[6])
|
|
119
|
+
end
|
|
120
|
+
if written then return 1 end
|
|
121
|
+
return 0
|
|
122
|
+
"""
|
|
123
|
+
|
|
124
|
+
# KEYS: result, job. ARGV: dedupe secs. Read without deleting; after the first read a result
|
|
125
|
+
# of a library-made id lives only `dedupe secs` more (an app-supplied id keeps its TTL).
|
|
126
|
+
RESULT_TAKE = """
|
|
127
|
+
local value = redis.call('GET', KEYS[1])
|
|
128
|
+
if not value then return false end
|
|
129
|
+
if redis.call('HGET', KEYS[2], 'app') ~= '1' then
|
|
130
|
+
-- A library id has one reader, and the job record still says `done` to any redelivery:
|
|
131
|
+
-- with no dedupe window the body goes at once (bulk embeddings kept ~2 MB a call: B97).
|
|
132
|
+
if tonumber(ARGV[1]) <= 0 then
|
|
133
|
+
redis.call('DEL', KEYS[1])
|
|
134
|
+
else
|
|
135
|
+
local ttl = redis.call('TTL', KEYS[1])
|
|
136
|
+
if ttl > tonumber(ARGV[1]) then redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
return value
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
# KEYS: status. ARGV: state, ttl, then field / value pairs. A final state is never replaced; the
|
|
143
|
+
# TTL only grows; every write stamps the store's time as `<state>_at`.
|
|
144
|
+
STATUS_SET = """
|
|
145
|
+
local current = redis.call('HGET', KEYS[1], 'state')
|
|
146
|
+
if current == 'done' or current == 'failed' or current == 'expired' or current == 'cancelled' then
|
|
147
|
+
return 0
|
|
148
|
+
end
|
|
149
|
+
local t = redis.call('TIME')
|
|
150
|
+
local now = t[1] .. '.' .. string.format('%06d', tonumber(t[2]))
|
|
151
|
+
local args = {'HSET', KEYS[1], 'state', ARGV[1], ARGV[1] .. '_at', now}
|
|
152
|
+
for i = 3, #ARGV, 2 do
|
|
153
|
+
table.insert(args, ARGV[i])
|
|
154
|
+
table.insert(args, ARGV[i + 1])
|
|
155
|
+
end
|
|
156
|
+
redis.call(unpack(args))
|
|
157
|
+
local ttl = redis.call('TTL', KEYS[1])
|
|
158
|
+
if ttl < tonumber(ARGV[2]) then redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) end
|
|
159
|
+
return 1
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@dataclass(frozen=True, slots=True)
|
|
164
|
+
class Registration:
|
|
165
|
+
"""What `register` found: `new`, `attached`, `takeover`, `done`, `expired` or `conflict`."""
|
|
166
|
+
|
|
167
|
+
outcome: str
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class RequestStore:
|
|
171
|
+
"""Claim records, results and request status of one environment."""
|
|
172
|
+
|
|
173
|
+
def __init__(
|
|
174
|
+
self,
|
|
175
|
+
client: redis.asyncio.Redis,
|
|
176
|
+
keys: Keys,
|
|
177
|
+
*,
|
|
178
|
+
result_ttl_secs: int = 300,
|
|
179
|
+
result_dedupe_secs: int = 0,
|
|
180
|
+
status_ttl_secs: int = 1800,
|
|
181
|
+
unconfirmed_secs: int = 30,
|
|
182
|
+
takeover_grace_secs: int = 5,
|
|
183
|
+
offload: ResultOffload | None = None,
|
|
184
|
+
offload_bytes: int | None = None,
|
|
185
|
+
) -> None:
|
|
186
|
+
"""Register the scripts; no I/O until the first call."""
|
|
187
|
+
self._client = client
|
|
188
|
+
self.keys = keys
|
|
189
|
+
self.result_ttl = result_ttl_secs
|
|
190
|
+
self.dedupe = result_dedupe_secs
|
|
191
|
+
self.status_ttl = status_ttl_secs
|
|
192
|
+
self.unconfirmed = unconfirmed_secs
|
|
193
|
+
self.takeover_grace = takeover_grace_secs
|
|
194
|
+
self.offload = offload
|
|
195
|
+
self.offload_bytes = offload_bytes
|
|
196
|
+
self._register = client.register_script(REGISTER)
|
|
197
|
+
self._confirm = client.register_script(CONFIRM)
|
|
198
|
+
self._abandon = client.register_script(ABANDON)
|
|
199
|
+
self._put = client.register_script(RESULT_PUT)
|
|
200
|
+
self._take = client.register_script(RESULT_TAKE)
|
|
201
|
+
self._status = client.register_script(STATUS_SET)
|
|
202
|
+
self._alive = client.register_script(CALLERS_ALIVE)
|
|
203
|
+
|
|
204
|
+
def _req(self, request_id: str) -> tuple[str, str, str, str]:
|
|
205
|
+
k = self.keys
|
|
206
|
+
return (
|
|
207
|
+
k.request(request_id, "job"),
|
|
208
|
+
k.request(request_id, "callers"),
|
|
209
|
+
k.status(request_id),
|
|
210
|
+
k.request(request_id, "res"),
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
async def register(
|
|
214
|
+
self,
|
|
215
|
+
request_id: str,
|
|
216
|
+
*,
|
|
217
|
+
caller: str,
|
|
218
|
+
user_hash: str,
|
|
219
|
+
params_hash: str,
|
|
220
|
+
model: str,
|
|
221
|
+
priority: bool,
|
|
222
|
+
app_id: bool,
|
|
223
|
+
) -> Registration:
|
|
224
|
+
"""Register a call before it is sent (duplicate ids attach to the first run)."""
|
|
225
|
+
job, callers, status, result = self._req(request_id)
|
|
226
|
+
outcome = await self._register(
|
|
227
|
+
[job, callers, status, result],
|
|
228
|
+
[
|
|
229
|
+
user_hash,
|
|
230
|
+
params_hash,
|
|
231
|
+
caller,
|
|
232
|
+
self.unconfirmed,
|
|
233
|
+
self.status_ttl,
|
|
234
|
+
model,
|
|
235
|
+
"1" if priority else "0",
|
|
236
|
+
self.takeover_grace,
|
|
237
|
+
],
|
|
238
|
+
)
|
|
239
|
+
text = _text(outcome)
|
|
240
|
+
if text == "new" and app_id:
|
|
241
|
+
await self._client.hset(job, "app", "1")
|
|
242
|
+
return Registration(text)
|
|
243
|
+
|
|
244
|
+
async def confirm(self, request_id: str, *, keep_secs: float) -> None:
|
|
245
|
+
"""The send succeeded: keep the record until the deadline plus the status TTL."""
|
|
246
|
+
job, callers, _, _ = self._req(request_id)
|
|
247
|
+
await self._confirm([job, callers], [int(keep_secs) + self.status_ttl])
|
|
248
|
+
|
|
249
|
+
async def abandon(self, request_id: str, *, caller: str) -> None:
|
|
250
|
+
"""The send failed: remove this caller's unconfirmed record."""
|
|
251
|
+
job, callers, status, _ = self._req(request_id)
|
|
252
|
+
await self._abandon([job, callers, status], [caller])
|
|
253
|
+
|
|
254
|
+
async def put_result(self, reply: Reply, *, final_state: str) -> bool:
|
|
255
|
+
"""Store a result once and wake its callers; False when one was already stored."""
|
|
256
|
+
job, callers, _, result = self._req(reply.request_id)
|
|
257
|
+
data = reply.dumps()
|
|
258
|
+
if self.offload is not None and self.offload_bytes and len(data) > self.offload_bytes:
|
|
259
|
+
data = MARKER + (await self.offload.put(reply.request_id, data)).encode()
|
|
260
|
+
written = await self._put(
|
|
261
|
+
[result, job, callers],
|
|
262
|
+
[
|
|
263
|
+
data,
|
|
264
|
+
self.result_ttl,
|
|
265
|
+
final_state,
|
|
266
|
+
self.status_ttl,
|
|
267
|
+
f"{self.keys.prefix}:done:",
|
|
268
|
+
reply.request_id,
|
|
269
|
+
],
|
|
270
|
+
)
|
|
271
|
+
return bool(written)
|
|
272
|
+
|
|
273
|
+
async def take_result(self, request_id: str) -> Reply | None:
|
|
274
|
+
"""A stored result; a library id's is then deleted (or kept `result_dedupe_secs`)."""
|
|
275
|
+
job, _, _, result = self._req(request_id)
|
|
276
|
+
raw = await self._take([result, job], [self.dedupe])
|
|
277
|
+
if raw is None:
|
|
278
|
+
return None
|
|
279
|
+
if raw.startswith(MARKER) and self.offload is not None:
|
|
280
|
+
raw = await self.offload.get(raw[len(MARKER) :].decode())
|
|
281
|
+
if raw is None:
|
|
282
|
+
return None
|
|
283
|
+
return Reply.model_validate(orjson.loads(raw))
|
|
284
|
+
|
|
285
|
+
async def set_status(self, request_id: str, state: str, **fields: Any) -> bool:
|
|
286
|
+
"""Write a status (False when a final state was already there)."""
|
|
287
|
+
pairs: list[str] = []
|
|
288
|
+
for name, value in fields.items():
|
|
289
|
+
if value is not None:
|
|
290
|
+
pairs += [name, str(value)]
|
|
291
|
+
done = await self._status([self.keys.status(request_id)], [state, self.status_ttl, *pairs])
|
|
292
|
+
return bool(done)
|
|
293
|
+
|
|
294
|
+
async def status(self, request_id: str) -> dict[str, str]:
|
|
295
|
+
"""The status hash of a request (empty when unknown or expired)."""
|
|
296
|
+
raw = await self._client.hgetall(self.keys.status(request_id))
|
|
297
|
+
return {_text(k): _text(v) for k, v in (raw or {}).items()}
|
|
298
|
+
|
|
299
|
+
async def callers_alive(self, request_id: str, *, prefix: str, caller_ttl: int) -> bool:
|
|
300
|
+
"""False only when every caller of the call is gone (and the store is not new)."""
|
|
301
|
+
reply = await self._alive(
|
|
302
|
+
[self._req(request_id)[1], f"{prefix}:epoch"], [f"{prefix}:caller:", caller_ttl]
|
|
303
|
+
)
|
|
304
|
+
return bool(reply)
|
|
305
|
+
|
|
306
|
+
async def callers(self, request_id: str) -> Sequence[str]:
|
|
307
|
+
"""Callers waiting on a request id."""
|
|
308
|
+
return [_text(v) for v in await self._client.smembers(self._req(request_id)[1])]
|
|
309
|
+
|
|
310
|
+
async def job(self, request_id: str) -> Mapping[str, str]:
|
|
311
|
+
"""The claim record (for tests and status views)."""
|
|
312
|
+
raw = await self._client.hgetall(self._req(request_id)[0])
|
|
313
|
+
return {_text(k): _text(v) for k, v in (raw or {}).items()}
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _text(value: Any) -> str:
|
|
317
|
+
return value.decode() if isinstance(value, bytes) else str(value)
|