llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,447 @@
1
+ """A queue on any Redis-protocol store: a stream plus visibility, generation and failure records.
2
+
3
+ Order is the stream id (the store's clock at send). A received message is held (invisible) for
4
+ the visibility time; `return_` makes it visible again in its place without a failure;
5
+ a lapsed hold (the worker died) counts one failure, and `max_receives` failures move it to the
6
+ dead-letter stream, reported to the receiver so the caller can be told (plan §3.18). Every
7
+ operation is one Lua call. A model has two queues: `<name>` and `<name>-priority` (E92).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ from collections.abc import Sequence
14
+ from dataclasses import dataclass
15
+ from typing import TYPE_CHECKING, Any
16
+
17
+ from redis.exceptions import NoScriptError
18
+
19
+ from llm_queue.engine.keys import Keys
20
+
21
+ if TYPE_CHECKING:
22
+ import redis.asyncio
23
+
24
+ _NOW = """
25
+ local t = redis.call('TIME')
26
+ local now = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
27
+ """
28
+
29
+ # Keys of one queue, in this order for every script.
30
+ _PARTS = ("s", "vis", "gen", "fails", "st", "dlq", "meta")
31
+
32
+ # ARGV: body, max waiting (0 = no limit), retention ms, doorbell key. Reply: {'ok', id} or
33
+ # {'full', depth}. One doorbell token per send wakes exactly one idle worker blocked on it
34
+ # (no receive-poll delay, B34; pub/sub woke every process and cost grew with them, B22).
35
+ SEND = (
36
+ _NOW
37
+ + """
38
+ local limit = tonumber(ARGV[2])
39
+ local depth = redis.call('XLEN', KEYS[1])
40
+ if limit > 0 and depth >= limit then return {'full', depth} end
41
+ local id = redis.call('XADD', KEYS[1], 'MINID', '~', tostring(now - tonumber(ARGV[3])), '*',
42
+ 'body', ARGV[1])
43
+ redis.call('RPUSH', ARGV[4], '1')
44
+ redis.call('LTRIM', ARGV[4], -64, -1)
45
+ redis.call('PEXPIRE', ARGV[4], 60000)
46
+ return {'ok', id}
47
+ """
48
+ )
49
+
50
+ _DLQ = """
51
+ local function to_dlq(id, body, fails, keep_ms)
52
+ redis.call('XADD', KEYS[6], 'MINID', '~', tostring(now - keep_ms), '*',
53
+ 'body', body, 'src', id, 'fails', fails)
54
+ redis.call('XDEL', KEYS[1], id)
55
+ redis.call('ZREM', KEYS[2], id)
56
+ redis.call('HDEL', KEYS[3], id)
57
+ redis.call('HDEL', KEYS[4], id)
58
+ redis.call('HDEL', KEYS[5], id)
59
+ end
60
+ local function older(a, b)
61
+ local am, as = string.match(a, '(%d+)-(%d+)')
62
+ local bm, bs = string.match(b, '(%d+)-(%d+)')
63
+ if tonumber(am) ~= tonumber(bm) then return tonumber(am) < tonumber(bm) end
64
+ return tonumber(as) < tonumber(bs)
65
+ end
66
+ """
67
+
68
+ # ARGV: max messages, visibility ms, max receives, dead-letter retention ms.
69
+ # Reply: {taken: {id, generation, body, ...}, dead: {id, body, ...}}. Messages come oldest first,
70
+ # whether new or visible again; a lapsed hold counts one failure.
71
+ RECEIVE = (
72
+ _NOW
73
+ + _DLQ
74
+ + """
75
+ local want = tonumber(ARGV[1])
76
+ local maxr = tonumber(ARGV[3])
77
+ local keep = tonumber(ARGV[4])
78
+ local due = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', now, 'LIMIT', 0, 1000)
79
+ table.sort(due, older)
80
+ local cursor = redis.call('HGET', KEYS[7], 'cursor')
81
+ local start = cursor and ('(' .. cursor) or '-'
82
+ local fresh = redis.call('XRANGE', KEYS[1], start, '+', 'COUNT', want)
83
+ local taken, gone = {}, {}
84
+ local i, j = 1, 1
85
+ local last_new = nil
86
+ while #taken < want * 3 and (i <= #due or j <= #fresh) do
87
+ local pick_due
88
+ if i > #due then pick_due = false
89
+ elseif j > #fresh then pick_due = true
90
+ else pick_due = older(due[i], fresh[j][1]) end
91
+ local id, body
92
+ if pick_due then
93
+ id = due[i]
94
+ i = i + 1
95
+ local entry = redis.call('XRANGE', KEYS[1], id, id)
96
+ if #entry == 0 then
97
+ redis.call('ZREM', KEYS[2], id)
98
+ redis.call('HDEL', KEYS[3], id)
99
+ redis.call('HDEL', KEYS[4], id)
100
+ redis.call('HDEL', KEYS[5], id)
101
+ else
102
+ body = entry[1][2][2]
103
+ if redis.call('HGET', KEYS[5], id) == 'h' then
104
+ local fails = redis.call('HINCRBY', KEYS[4], id, 1)
105
+ if maxr > 0 and fails >= maxr then
106
+ to_dlq(id, body, fails, keep)
107
+ table.insert(gone, id)
108
+ table.insert(gone, body)
109
+ body = nil
110
+ end
111
+ end
112
+ end
113
+ else
114
+ id = fresh[j][1]
115
+ body = fresh[j][2][2]
116
+ last_new = id
117
+ j = j + 1
118
+ end
119
+ if body then
120
+ local gen = redis.call('HINCRBY', KEYS[3], id, 1)
121
+ redis.call('HSET', KEYS[5], id, 'h')
122
+ redis.call('ZADD', KEYS[2], now + tonumber(ARGV[2]), id)
123
+ table.insert(taken, id)
124
+ table.insert(taken, tostring(gen))
125
+ table.insert(taken, body)
126
+ if #taken >= want * 3 then break end
127
+ end
128
+ end
129
+ if last_new then redis.call('HSET', KEYS[7], 'cursor', last_new) end
130
+ return {taken, gone}
131
+ """
132
+ )
133
+
134
+ # ARGV: id, generation. Reply: 1 deleted, 0 the receipt is stale (another holder has it).
135
+ ACK = """
136
+ if redis.call('HGET', KEYS[3], ARGV[1]) ~= ARGV[2] then return 0 end
137
+ redis.call('XDEL', KEYS[1], ARGV[1])
138
+ redis.call('ZREM', KEYS[2], ARGV[1])
139
+ redis.call('HDEL', KEYS[3], ARGV[1])
140
+ redis.call('HDEL', KEYS[4], ARGV[1])
141
+ redis.call('HDEL', KEYS[5], ARGV[1])
142
+ return 1
143
+ """
144
+
145
+ # ARGV: id, generation, delay ms. Visible again in its place after the delay; no failure.
146
+ RETURN = (
147
+ _NOW
148
+ + """
149
+ if redis.call('HGET', KEYS[3], ARGV[1]) ~= ARGV[2] then return 0 end
150
+ redis.call('HSET', KEYS[5], ARGV[1], 'r')
151
+ redis.call('ZADD', KEYS[2], now + tonumber(ARGV[3]), ARGV[1])
152
+ return 1
153
+ """
154
+ )
155
+
156
+ # ARGV: id, generation, visibility ms. Extend a hold (the worker's heartbeat).
157
+ EXTEND = (
158
+ _NOW
159
+ + """
160
+ if redis.call('HGET', KEYS[3], ARGV[1]) ~= ARGV[2] then return 0 end
161
+ redis.call('ZADD', KEYS[2], 'XX', now + tonumber(ARGV[3]), ARGV[1])
162
+ return 1
163
+ """
164
+ )
165
+
166
+ # ARGV: id, generation, delay ms, max receives, dead-letter retention ms. The handler failed:
167
+ # count one failure; at `max_receives` the message goes to the dead-letter stream.
168
+ # Reply: 'retry', 'dead' or 'stale'.
169
+ FAIL = (
170
+ _NOW
171
+ + _DLQ
172
+ + """
173
+ if redis.call('HGET', KEYS[3], ARGV[1]) ~= ARGV[2] then return 'stale' end
174
+ local fails = redis.call('HINCRBY', KEYS[4], ARGV[1], 1)
175
+ local maxr = tonumber(ARGV[4])
176
+ if maxr > 0 and fails >= maxr then
177
+ local entry = redis.call('XRANGE', KEYS[1], ARGV[1], ARGV[1])
178
+ if #entry > 0 then to_dlq(ARGV[1], entry[1][2][2], fails, tonumber(ARGV[5])) end
179
+ return 'dead'
180
+ end
181
+ redis.call('HSET', KEYS[5], ARGV[1], 'r')
182
+ redis.call('ZADD', KEYS[2], now + tonumber(ARGV[3]), ARGV[1])
183
+ return 'retry'
184
+ """
185
+ )
186
+
187
+ # Reply: {messages in the stream, held, dead letters}.
188
+ DEPTH = """
189
+ local held = 0
190
+ for _, v in ipairs(redis.call('HVALS', KEYS[5])) do if v == 'h' then held = held + 1 end end
191
+ return {redis.call('XLEN', KEYS[1]), held, redis.call('XLEN', KEYS[6])}
192
+ """
193
+
194
+ # ARGV: max count. Dead letters back on the queue (new ids: they go to the back).
195
+ REDRIVE = """
196
+ local moved = 0
197
+ for _, entry in ipairs(redis.call('XRANGE', KEYS[6], '-', '+', 'COUNT', tonumber(ARGV[1]))) do
198
+ local fields = entry[2]
199
+ local body
200
+ for k = 1, #fields, 2 do if fields[k] == 'body' then body = fields[k + 1] end end
201
+ redis.call('XADD', KEYS[1], '*', 'body', body)
202
+ redis.call('XDEL', KEYS[6], entry[1])
203
+ moved = moved + 1
204
+ end
205
+ return moved
206
+ """
207
+
208
+
209
+ @dataclass(frozen=True, slots=True)
210
+ class Message:
211
+ """A received message: its stream id (its place), generation and body."""
212
+
213
+ queue: str
214
+ id: str
215
+ generation: int
216
+ body: bytes
217
+
218
+ @property
219
+ def position_ms(self) -> int:
220
+ """The store's time at send, in ms (deadlines and ages are measured from it)."""
221
+ return int(self.id.split("-", 1)[0])
222
+
223
+
224
+ @dataclass(frozen=True, slots=True)
225
+ class DeadLetter:
226
+ """A message moved to the dead-letter stream by this receive."""
227
+
228
+ queue: str
229
+ id: str
230
+ body: bytes
231
+
232
+
233
+ @dataclass(frozen=True, slots=True)
234
+ class QueueSettings:
235
+ """Per-queue numbers (from the model's `queue` setting and `max_waiting`)."""
236
+
237
+ visibility_secs: float = 60.0
238
+ max_receives: int = 10
239
+ retention_secs: int = 345_600
240
+ dlq_retention_secs: int = 86_400
241
+ max_waiting: int = 50_000
242
+
243
+
244
+ class QueueFull(Exception): # the plan's error kind is `queue_full`
245
+ """`max_waiting` calls already wait on this queue."""
246
+
247
+ def __init__(self, queue: str, depth: int) -> None:
248
+ """Keep the queue and its depth."""
249
+ super().__init__(f"queue {queue} is full ({depth} waiting)")
250
+ self.queue = queue
251
+ self.depth = depth
252
+
253
+
254
+ class NotReplicated(Exception): # the plan's error kind is `queue_unavailable`
255
+ """The send reached the primary but not `queue_wait_replicas` replicas in time."""
256
+
257
+
258
+ class StreamQueues:
259
+ """Send, receive, return, extend, fail and acknowledge on named queues of one prefix."""
260
+
261
+ def __init__(self, client: redis.asyncio.Redis, keys: Keys) -> None:
262
+ """Register the scripts; no I/O until the first call."""
263
+ self._client = client
264
+ self.keys = keys
265
+ self._send = client.register_script(SEND)
266
+ self._receive = client.register_script(RECEIVE)
267
+ self._ack = client.register_script(ACK)
268
+ self._return = client.register_script(RETURN)
269
+ self._extend = client.register_script(EXTEND)
270
+ self._fail = client.register_script(FAIL)
271
+ self._depth = client.register_script(DEPTH)
272
+ self._redrive = client.register_script(REDRIVE)
273
+
274
+ async def _send_and_wait(
275
+ self, queue: str, args: list[bytes | int | str], replicas: int, wait_ms: int
276
+ ) -> tuple[Any, Any]:
277
+ """SEND and WAIT on one connection (WAIT counts only that connection's writes)."""
278
+ async with self._client.pipeline(transaction=False) as pipe:
279
+ await self._send(self._k(queue), args, client=pipe)
280
+ pipe.execute_command("WAIT", replicas, wait_ms)
281
+ reply, acked = await pipe.execute()
282
+ return reply, acked
283
+
284
+ def bell(self, queue: str) -> str:
285
+ """The queue's doorbell: a list a send pushes one token on (see SEND)."""
286
+ return f"{self.keys.prefix}:bell:{queue}"
287
+
288
+ async def wait_bell(self, queues: Sequence[str], timeout_secs: float) -> bool:
289
+ """Block until one of the queues' doorbells rings (True) or `timeout_secs` pass."""
290
+ rung = await self._client.blpop([self.bell(q) for q in queues], timeout=timeout_secs)
291
+ return rung is not None
292
+
293
+ def _k(self, queue: str) -> list[str]:
294
+ return [self.keys.queue(queue, part) for part in _PARTS]
295
+
296
+ async def send(
297
+ self,
298
+ queue: str,
299
+ body: bytes,
300
+ settings: QueueSettings,
301
+ *,
302
+ wait_replicas: int = 0,
303
+ wait_ms: int = 100,
304
+ ) -> str:
305
+ """Append a message; returns its id. `QueueFull` above `max_waiting`.
306
+
307
+ With `wait_replicas`, the send and a `WAIT` go down one connection (WAIT counts that
308
+ connection's writes) and too few acknowledgements raise `NotReplicated`.
309
+ """
310
+ args: list[bytes | int | str] = [
311
+ body,
312
+ settings.max_waiting,
313
+ settings.retention_secs * 1000,
314
+ self.bell(queue),
315
+ ]
316
+ if not wait_replicas:
317
+ reply = await self._send(self._k(queue), args)
318
+ acked = None
319
+ else:
320
+ try:
321
+ reply, acked = await self._send_and_wait(queue, args, wait_replicas, wait_ms)
322
+ except NoScriptError:
323
+ # A failover moved the pipeline to a node without the script cached: a pipeline
324
+ # loads its scripts only when it starts, so run it once more (that reloads).
325
+ # Exactly once, like redis-py's own reload for plain script calls: a second
326
+ # NOSCRIPT means the cache was flushed again mid-call, not something to retry.
327
+ reply, acked = await self._send_and_wait(queue, args, wait_replicas, wait_ms)
328
+ if _text(reply[0]) == "full":
329
+ raise QueueFull(queue, int(reply[1]))
330
+ if acked is not None and int(acked) < wait_replicas:
331
+ raise NotReplicated(
332
+ f"queue {queue}: {acked} of {wait_replicas} replicas in {wait_ms} ms"
333
+ )
334
+ return _text(reply[1])
335
+
336
+ async def receive(
337
+ self, queue: str, count: int, settings: QueueSettings
338
+ ) -> tuple[list[Message], list[DeadLetter]]:
339
+ """Up to `count` messages, oldest first, held for the visibility time; and any dead."""
340
+ taken, dead = await self._receive(
341
+ self._k(queue),
342
+ [
343
+ count,
344
+ int(settings.visibility_secs * 1000),
345
+ settings.max_receives,
346
+ settings.dlq_retention_secs * 1000,
347
+ ],
348
+ )
349
+ messages = [
350
+ Message(queue, _text(taken[i]), int(taken[i + 1]), _bytes(taken[i + 2]))
351
+ for i in range(0, len(taken), 3)
352
+ ]
353
+ letters = [
354
+ DeadLetter(queue, _text(dead[i]), _bytes(dead[i + 1])) for i in range(0, len(dead), 2)
355
+ ]
356
+ return messages, letters
357
+
358
+ async def receive_wait(
359
+ self, queue: str, count: int, settings: QueueSettings, wait_secs: float
360
+ ) -> tuple[list[Message], list[DeadLetter]]:
361
+ """`receive`, polling with growing pauses (20 ms to 500 ms) for up to `wait_secs`."""
362
+ loop = asyncio.get_running_loop()
363
+ until = loop.time() + wait_secs
364
+ pause = 0.02
365
+ while True:
366
+ messages, dead = await self.receive(queue, count, settings)
367
+ if messages or dead or loop.time() >= until:
368
+ return messages, dead
369
+ await asyncio.sleep(min(pause, max(0.0, until - loop.time())))
370
+ pause = min(pause * 2, 0.5)
371
+
372
+ async def ack(self, message: Message) -> bool:
373
+ """Delete a handled message; False when the receipt is stale."""
374
+ return bool(await self._ack(self._k(message.queue), [message.id, message.generation]))
375
+
376
+ async def return_(self, message: Message, delay_secs: float = 0.0) -> bool:
377
+ """Make a message visible again in its place (after `delay_secs`), with no failure."""
378
+ return bool(
379
+ await self._return(
380
+ self._k(message.queue), [message.id, message.generation, int(delay_secs * 1000)]
381
+ )
382
+ )
383
+
384
+ async def extend(self, message: Message, visibility_secs: float) -> bool:
385
+ """Extend the hold of a message (heartbeat); False when the receipt is stale."""
386
+ return bool(
387
+ await self._extend(
388
+ self._k(message.queue),
389
+ [message.id, message.generation, int(visibility_secs * 1000)],
390
+ )
391
+ )
392
+
393
+ async def fail(self, message: Message, settings: QueueSettings, delay_secs: float = 0.0) -> str:
394
+ """Count a failure: `retry` (visible again after the delay), `dead` or `stale`."""
395
+ return _text(
396
+ await self._fail(
397
+ self._k(message.queue),
398
+ [
399
+ message.id,
400
+ message.generation,
401
+ int(delay_secs * 1000),
402
+ settings.max_receives,
403
+ settings.dlq_retention_secs * 1000,
404
+ ],
405
+ )
406
+ )
407
+
408
+ async def depth(self, queue: str) -> tuple[int, int, int]:
409
+ """(messages in the queue, held by workers, dead letters)."""
410
+ reply = await self._depth(self._k(queue), [])
411
+ return int(reply[0]), int(reply[1]), int(reply[2])
412
+
413
+ async def dead_letters(self, queue: str, count: int = 10) -> list[dict[str, Any]]:
414
+ """Dead letters, oldest first: id, source id, failures, body."""
415
+ entries = await self._client.xrange(self.keys.queue(queue, "dlq"), count=count)
416
+ out = []
417
+ for entry_id, fields in entries or ():
418
+ row = {_text(k): v for k, v in (fields or {}).items()}
419
+ out.append(
420
+ {
421
+ "id": _text(entry_id),
422
+ "src": _text(row.get("src", b"")),
423
+ "fails": int(row.get("fails", 0)),
424
+ "body": _bytes(row.get("body", b"")),
425
+ }
426
+ )
427
+ return out
428
+
429
+ async def redrive(self, queue: str, count: int = 1000) -> int:
430
+ """Move up to `count` dead letters back to the queue; returns how many."""
431
+ return int(await self._redrive(self._k(queue), [count]))
432
+
433
+ async def purge_dead(self, queue: str) -> None:
434
+ """Delete every dead letter of a queue."""
435
+ await self._client.delete(self.keys.queue(queue, "dlq"))
436
+
437
+ async def delete(self, names: Sequence[str]) -> None:
438
+ """Delete queues and everything in them (model removal)."""
439
+ await self._client.delete(*[k for name in names for k in self._k(name)])
440
+
441
+
442
+ def _text(value: Any) -> str:
443
+ return value.decode() if isinstance(value, bytes) else str(value)
444
+
445
+
446
+ def _bytes(value: Any) -> bytes:
447
+ return value if isinstance(value, bytes) else str(value).encode()