llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,317 @@
1
+ """Per-request records: the duplicate-id claim, the result and the request status.
2
+
3
+ A call's record is written only if absent when it is sent and confirmed only after the send
4
+ succeeded, so a failed send never leaves a phantom (plan §3.18 R23). Results are written once
5
+ (and announced to the caller in the same script); a library id's result is deleted on its
6
+ first read, an app id's kept `result_ttl_secs`. A final status is never replaced. Scripts use
7
+ the store's clock.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections.abc import Mapping, Sequence
13
+ from dataclasses import dataclass
14
+ from typing import TYPE_CHECKING, Any
15
+
16
+ import orjson
17
+
18
+ from llm_queue.engine.envelope import Reply
19
+ from llm_queue.engine.keys import Keys
20
+ from llm_queue.engine.offload import MARKER, ResultOffload
21
+
22
+ if TYPE_CHECKING:
23
+ import redis.asyncio
24
+
25
+ FINAL_STATES = ("done", "failed", "expired", "cancelled")
26
+
27
+ # KEYS: callers, epoch. ARGV: caller key prefix, caller ttl secs. Reply: 1 when any caller of
28
+ # the call is alive, or the store came back less than a TTL ago (keys not renewed yet).
29
+ CALLERS_ALIVE = """
30
+ local t = redis.call('TIME')
31
+ local now = tonumber(t[1])
32
+ local epoch = tonumber(redis.call('GET', KEYS[2]) or '')
33
+ if not epoch or now - epoch < tonumber(ARGV[2]) then return 1 end
34
+ local callers = redis.call('SMEMBERS', KEYS[1])
35
+ if #callers == 0 then return 1 end
36
+ for _, caller in ipairs(callers) do
37
+ if redis.call('EXISTS', ARGV[1] .. caller) == 1 then return 1 end
38
+ end
39
+ return 0
40
+ """
41
+
42
+ # KEYS: job, callers, status, result. ARGV: user hash, params hash, caller, unconfirmed ttl,
43
+ # status ttl, model, priority, takeover grace secs.
44
+ # Reply: new | attached | takeover | done | expired | conflict.
45
+ REGISTER = """
46
+ local job = redis.call('HGETALL', KEYS[1])
47
+ local clock = redis.call('TIME')
48
+ local now = tonumber(clock[1])
49
+ if #job == 0 then
50
+ redis.call('HSET', KEYS[1], 'state', 'queued', 'user', ARGV[1], 'params', ARGV[2],
51
+ 'caller', ARGV[3], 'confirmed', '0', 'created', now)
52
+ redis.call('EXPIRE', KEYS[1], tonumber(ARGV[4]))
53
+ redis.call('SADD', KEYS[2], ARGV[3])
54
+ redis.call('EXPIRE', KEYS[2], tonumber(ARGV[4]))
55
+ if redis.call('HGET', KEYS[3], 'state') == false then
56
+ local t = redis.call('TIME')
57
+ redis.call('HSET', KEYS[3], 'state', 'waiting', 'model', ARGV[6], 'priority', ARGV[7],
58
+ 'enqueued_at', t[1] .. '.' .. string.format('%06d', tonumber(t[2])))
59
+ redis.call('EXPIRE', KEYS[3], tonumber(ARGV[5]))
60
+ end
61
+ return 'new'
62
+ end
63
+ local fields = {}
64
+ for i = 1, #job, 2 do fields[job[i]] = job[i + 1] end
65
+ if fields['params'] == nil then
66
+ -- A record a worker's claim made (its caller's record was gone): the call is running;
67
+ -- attach to it rather than call it a conflict.
68
+ redis.call('HSET', KEYS[1], 'user', ARGV[1], 'params', ARGV[2], 'caller', ARGV[3],
69
+ 'confirmed', '1', 'created', now)
70
+ redis.call('SADD', KEYS[2], ARGV[3])
71
+ return 'attached'
72
+ end
73
+ if fields['user'] ~= ARGV[1] or fields['params'] ~= ARGV[2] then return 'conflict' end
74
+ local state = fields['state']
75
+ if state == 'done' or state == 'failed' or state == 'expired' or state == 'cancelled' then
76
+ if redis.call('EXISTS', KEYS[4]) == 1 then return 'done' end
77
+ return 'expired'
78
+ end
79
+ -- An unconfirmed record older than the grace: its send failed or its sender died; take over.
80
+ if fields['confirmed'] == '0' and now - tonumber(fields['created'] or '0') >= tonumber(ARGV[8]) then
81
+ redis.call('HSET', KEYS[1], 'caller', ARGV[3], 'created', now)
82
+ redis.call('SADD', KEYS[2], ARGV[3])
83
+ return 'takeover'
84
+ end
85
+ redis.call('SADD', KEYS[2], ARGV[3])
86
+ return 'attached'
87
+ """
88
+
89
+ # KEYS: job, callers. ARGV: keep secs. The send succeeded: the record lives until the deadline
90
+ # plus the status TTL.
91
+ CONFIRM = """
92
+ redis.call('HSET', KEYS[1], 'confirmed', '1')
93
+ redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1]))
94
+ redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1]))
95
+ return 1
96
+ """
97
+
98
+ # KEYS: job, callers, status. ARGV: caller. The send failed: drop the unconfirmed record.
99
+ ABANDON = """
100
+ local confirmed = redis.call('HGET', KEYS[1], 'confirmed')
101
+ if confirmed == '0' and redis.call('HGET', KEYS[1], 'caller') == ARGV[1] then
102
+ redis.call('DEL', KEYS[1], KEYS[2], KEYS[3])
103
+ return 1
104
+ end
105
+ return 0
106
+ """
107
+
108
+ # KEYS: result, job, callers. ARGV: reply JSON, result ttl, final state, keep secs, channel
109
+ # prefix (`<prefix>:done:`), request id. Writes the result only if none exists, marks the claim
110
+ # final and wakes every caller waiting on this id. Reply: 1 written, 0 a result already existed.
111
+ RESULT_PUT = """
112
+ local written = redis.call('SET', KEYS[1], ARGV[1], 'NX', 'EX', tonumber(ARGV[2]))
113
+ if redis.call('EXISTS', KEYS[2]) == 1 then
114
+ redis.call('HSET', KEYS[2], 'state', ARGV[3])
115
+ redis.call('EXPIRE', KEYS[2], tonumber(ARGV[4]))
116
+ end
117
+ for _, caller in ipairs(redis.call('SMEMBERS', KEYS[3])) do
118
+ redis.call('PUBLISH', ARGV[5] .. caller, ARGV[6])
119
+ end
120
+ if written then return 1 end
121
+ return 0
122
+ """
123
+
124
+ # KEYS: result, job. ARGV: dedupe secs. Read without deleting; after the first read a result
125
+ # of a library-made id lives only `dedupe secs` more (an app-supplied id keeps its TTL).
126
+ RESULT_TAKE = """
127
+ local value = redis.call('GET', KEYS[1])
128
+ if not value then return false end
129
+ if redis.call('HGET', KEYS[2], 'app') ~= '1' then
130
+ -- A library id has one reader, and the job record still says `done` to any redelivery:
131
+ -- with no dedupe window the body goes at once (bulk embeddings kept ~2 MB a call: B97).
132
+ if tonumber(ARGV[1]) <= 0 then
133
+ redis.call('DEL', KEYS[1])
134
+ else
135
+ local ttl = redis.call('TTL', KEYS[1])
136
+ if ttl > tonumber(ARGV[1]) then redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) end
137
+ end
138
+ end
139
+ return value
140
+ """
141
+
142
+ # KEYS: status. ARGV: state, ttl, then field / value pairs. A final state is never replaced; the
143
+ # TTL only grows; every write stamps the store's time as `<state>_at`.
144
+ STATUS_SET = """
145
+ local current = redis.call('HGET', KEYS[1], 'state')
146
+ if current == 'done' or current == 'failed' or current == 'expired' or current == 'cancelled' then
147
+ return 0
148
+ end
149
+ local t = redis.call('TIME')
150
+ local now = t[1] .. '.' .. string.format('%06d', tonumber(t[2]))
151
+ local args = {'HSET', KEYS[1], 'state', ARGV[1], ARGV[1] .. '_at', now}
152
+ for i = 3, #ARGV, 2 do
153
+ table.insert(args, ARGV[i])
154
+ table.insert(args, ARGV[i + 1])
155
+ end
156
+ redis.call(unpack(args))
157
+ local ttl = redis.call('TTL', KEYS[1])
158
+ if ttl < tonumber(ARGV[2]) then redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) end
159
+ return 1
160
+ """
161
+
162
+
163
+ @dataclass(frozen=True, slots=True)
164
+ class Registration:
165
+ """What `register` found: `new`, `attached`, `takeover`, `done`, `expired` or `conflict`."""
166
+
167
+ outcome: str
168
+
169
+
170
+ class RequestStore:
171
+ """Claim records, results and request status of one environment."""
172
+
173
+ def __init__(
174
+ self,
175
+ client: redis.asyncio.Redis,
176
+ keys: Keys,
177
+ *,
178
+ result_ttl_secs: int = 300,
179
+ result_dedupe_secs: int = 0,
180
+ status_ttl_secs: int = 1800,
181
+ unconfirmed_secs: int = 30,
182
+ takeover_grace_secs: int = 5,
183
+ offload: ResultOffload | None = None,
184
+ offload_bytes: int | None = None,
185
+ ) -> None:
186
+ """Register the scripts; no I/O until the first call."""
187
+ self._client = client
188
+ self.keys = keys
189
+ self.result_ttl = result_ttl_secs
190
+ self.dedupe = result_dedupe_secs
191
+ self.status_ttl = status_ttl_secs
192
+ self.unconfirmed = unconfirmed_secs
193
+ self.takeover_grace = takeover_grace_secs
194
+ self.offload = offload
195
+ self.offload_bytes = offload_bytes
196
+ self._register = client.register_script(REGISTER)
197
+ self._confirm = client.register_script(CONFIRM)
198
+ self._abandon = client.register_script(ABANDON)
199
+ self._put = client.register_script(RESULT_PUT)
200
+ self._take = client.register_script(RESULT_TAKE)
201
+ self._status = client.register_script(STATUS_SET)
202
+ self._alive = client.register_script(CALLERS_ALIVE)
203
+
204
+ def _req(self, request_id: str) -> tuple[str, str, str, str]:
205
+ k = self.keys
206
+ return (
207
+ k.request(request_id, "job"),
208
+ k.request(request_id, "callers"),
209
+ k.status(request_id),
210
+ k.request(request_id, "res"),
211
+ )
212
+
213
+ async def register(
214
+ self,
215
+ request_id: str,
216
+ *,
217
+ caller: str,
218
+ user_hash: str,
219
+ params_hash: str,
220
+ model: str,
221
+ priority: bool,
222
+ app_id: bool,
223
+ ) -> Registration:
224
+ """Register a call before it is sent (duplicate ids attach to the first run)."""
225
+ job, callers, status, result = self._req(request_id)
226
+ outcome = await self._register(
227
+ [job, callers, status, result],
228
+ [
229
+ user_hash,
230
+ params_hash,
231
+ caller,
232
+ self.unconfirmed,
233
+ self.status_ttl,
234
+ model,
235
+ "1" if priority else "0",
236
+ self.takeover_grace,
237
+ ],
238
+ )
239
+ text = _text(outcome)
240
+ if text == "new" and app_id:
241
+ await self._client.hset(job, "app", "1")
242
+ return Registration(text)
243
+
244
+ async def confirm(self, request_id: str, *, keep_secs: float) -> None:
245
+ """The send succeeded: keep the record until the deadline plus the status TTL."""
246
+ job, callers, _, _ = self._req(request_id)
247
+ await self._confirm([job, callers], [int(keep_secs) + self.status_ttl])
248
+
249
+ async def abandon(self, request_id: str, *, caller: str) -> None:
250
+ """The send failed: remove this caller's unconfirmed record."""
251
+ job, callers, status, _ = self._req(request_id)
252
+ await self._abandon([job, callers, status], [caller])
253
+
254
+ async def put_result(self, reply: Reply, *, final_state: str) -> bool:
255
+ """Store a result once and wake its callers; False when one was already stored."""
256
+ job, callers, _, result = self._req(reply.request_id)
257
+ data = reply.dumps()
258
+ if self.offload is not None and self.offload_bytes and len(data) > self.offload_bytes:
259
+ data = MARKER + (await self.offload.put(reply.request_id, data)).encode()
260
+ written = await self._put(
261
+ [result, job, callers],
262
+ [
263
+ data,
264
+ self.result_ttl,
265
+ final_state,
266
+ self.status_ttl,
267
+ f"{self.keys.prefix}:done:",
268
+ reply.request_id,
269
+ ],
270
+ )
271
+ return bool(written)
272
+
273
+ async def take_result(self, request_id: str) -> Reply | None:
274
+ """A stored result; a library id's is then deleted (or kept `result_dedupe_secs`)."""
275
+ job, _, _, result = self._req(request_id)
276
+ raw = await self._take([result, job], [self.dedupe])
277
+ if raw is None:
278
+ return None
279
+ if raw.startswith(MARKER) and self.offload is not None:
280
+ raw = await self.offload.get(raw[len(MARKER) :].decode())
281
+ if raw is None:
282
+ return None
283
+ return Reply.model_validate(orjson.loads(raw))
284
+
285
+ async def set_status(self, request_id: str, state: str, **fields: Any) -> bool:
286
+ """Write a status (False when a final state was already there)."""
287
+ pairs: list[str] = []
288
+ for name, value in fields.items():
289
+ if value is not None:
290
+ pairs += [name, str(value)]
291
+ done = await self._status([self.keys.status(request_id)], [state, self.status_ttl, *pairs])
292
+ return bool(done)
293
+
294
+ async def status(self, request_id: str) -> dict[str, str]:
295
+ """The status hash of a request (empty when unknown or expired)."""
296
+ raw = await self._client.hgetall(self.keys.status(request_id))
297
+ return {_text(k): _text(v) for k, v in (raw or {}).items()}
298
+
299
+ async def callers_alive(self, request_id: str, *, prefix: str, caller_ttl: int) -> bool:
300
+ """False only when every caller of the call is gone (and the store is not new)."""
301
+ reply = await self._alive(
302
+ [self._req(request_id)[1], f"{prefix}:epoch"], [f"{prefix}:caller:", caller_ttl]
303
+ )
304
+ return bool(reply)
305
+
306
+ async def callers(self, request_id: str) -> Sequence[str]:
307
+ """Callers waiting on a request id."""
308
+ return [_text(v) for v in await self._client.smembers(self._req(request_id)[1])]
309
+
310
+ async def job(self, request_id: str) -> Mapping[str, str]:
311
+ """The claim record (for tests and status views)."""
312
+ raw = await self._client.hgetall(self._req(request_id)[0])
313
+ return {_text(k): _text(v) for k, v in (raw or {}).items()}
314
+
315
+
316
+ def _text(value: Any) -> str:
317
+ return value.decode() if isinstance(value, bytes) else str(value)