llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,360 @@
1
+ """Providers on routes: one attempt of an operation, its usage, and error classification.
2
+
3
+ Chat runs on the provider's streaming API under the hood and is assembled into the plain reply
4
+ shape (Converse body, chat.completion), so the attempt timers apply to every chat call; the
5
+ caller still gets a normal reply. `register_provider` adds a provider for a custom account kind.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import base64
11
+ import contextlib
12
+ import time
13
+ from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
14
+ from dataclasses import dataclass
15
+ from typing import Any, Protocol
16
+
17
+ import orjson
18
+
19
+ from llm_queue.clients.pool import ClientPool, Route
20
+ from llm_queue.clients.timing import Timing, first, paced
21
+ from llm_queue.errors import ConfigError, ErrorInfo
22
+ from llm_queue.providers.classify import classify_bedrock, classify_openai
23
+
24
+ _TOKEN_HEADER = "x-amzn-bedrock-input-token-count"
25
+
26
+
27
+ @dataclass(frozen=True, slots=True)
28
+ class Usage:
29
+ """Tokens of one attempt as the provider reported them; None when it reported nothing."""
30
+
31
+ input_tokens: int | None = None
32
+ output_tokens: int | None = None
33
+ cache_read_tokens: int = 0
34
+ cache_write_tokens: int = 0
35
+
36
+
37
+ @dataclass(frozen=True, slots=True)
38
+ class Result:
39
+ """One successful attempt: the provider-native reply and its usage."""
40
+
41
+ response: dict[str, Any]
42
+ usage: Usage
43
+
44
+
45
+ Sink = Callable[[Any], Awaitable[None]]
46
+ """Receives each streamed event of an attempt (streaming to the caller, plan §3.13)."""
47
+
48
+
49
+ class Provider(Protocol):
50
+ """One provider kind: runs an operation on a route and classifies its failures."""
51
+
52
+ name: str
53
+
54
+ async def call(
55
+ self,
56
+ op: str,
57
+ params: Mapping[str, Any],
58
+ route: Route,
59
+ timing: Timing,
60
+ sink: Sink | None = None,
61
+ ) -> Result:
62
+ """Run one attempt; chat operations pass each streamed event to `sink`."""
63
+ ...
64
+
65
+ def classify(self, exc: BaseException) -> ErrorInfo:
66
+ """Throttle / retryable / fatal."""
67
+ ...
68
+
69
+
70
+ # ------------------------------------------------------------------------------- Bedrock
71
+
72
+
73
+ def _converse_body(events: list[Mapping[str, Any]]) -> dict[str, Any]:
74
+ """Assemble ConverseStream events into the body `converse` returns."""
75
+ role = "assistant"
76
+ blocks: dict[int, dict[str, Any]] = {}
77
+ tool_input: dict[int, str] = {}
78
+ body: dict[str, Any] = {}
79
+ for event in events:
80
+ if "messageStart" in event:
81
+ role = event["messageStart"].get("role", role)
82
+ elif "contentBlockStart" in event:
83
+ index = event["contentBlockStart"].get("contentBlockIndex", 0)
84
+ start = event["contentBlockStart"].get("start", {})
85
+ if "toolUse" in start:
86
+ blocks[index] = {"toolUse": dict(start["toolUse"])}
87
+ tool_input[index] = ""
88
+ elif "contentBlockDelta" in event:
89
+ index = event["contentBlockDelta"].get("contentBlockIndex", 0)
90
+ delta = event["contentBlockDelta"].get("delta", {})
91
+ if "text" in delta:
92
+ block = blocks.setdefault(index, {"text": ""})
93
+ block["text"] = block.get("text", "") + delta["text"]
94
+ elif "toolUse" in delta:
95
+ tool_input[index] = tool_input.get(index, "") + delta["toolUse"].get("input", "")
96
+ blocks.setdefault(index, {"toolUse": {}})
97
+ elif "reasoningContent" in delta:
98
+ block = blocks.setdefault(index, {"reasoningContent": {"reasoningText": {}}})
99
+ reasoning = block["reasoningContent"].setdefault("reasoningText", {})
100
+ part = delta["reasoningContent"]
101
+ if "text" in part:
102
+ reasoning["text"] = reasoning.get("text", "") + part["text"]
103
+ if "signature" in part:
104
+ reasoning["signature"] = part["signature"]
105
+ elif "messageStop" in event:
106
+ body["stopReason"] = event["messageStop"].get("stopReason")
107
+ extra = event["messageStop"].get("additionalModelResponseFields")
108
+ if extra is not None:
109
+ body["additionalModelResponseFields"] = extra
110
+ elif "metadata" in event:
111
+ for key in ("usage", "metrics", "trace"):
112
+ if key in event["metadata"]:
113
+ body[key] = event["metadata"][key]
114
+ for index, text in tool_input.items():
115
+ blocks[index]["toolUse"]["input"] = orjson.loads(text) if text else {}
116
+ content = [blocks[i] for i in sorted(blocks)]
117
+ return {"output": {"message": {"role": role, "content": content}}, **body}
118
+
119
+
120
+ def _bedrock_usage(usage: Mapping[str, Any] | None) -> Usage:
121
+ if not usage:
122
+ return Usage()
123
+ return Usage(
124
+ input_tokens=usage.get("inputTokens"),
125
+ output_tokens=usage.get("outputTokens"),
126
+ cache_read_tokens=int(usage.get("cacheReadInputTokens") or 0),
127
+ cache_write_tokens=int(usage.get("cacheWriteInputTokens") or 0),
128
+ )
129
+
130
+
131
+ class BedrockProvider:
132
+ """Bedrock runtime: `converse` (over ConverseStream) and `invoke_model` (embeddings)."""
133
+
134
+ name = "bedrock"
135
+
136
+ def __init__(self, pool: ClientPool) -> None:
137
+ """Use `pool` for clients."""
138
+ self._pool = pool
139
+
140
+ async def stream(
141
+ self, params: Mapping[str, Any], route: Route, timing: Timing
142
+ ) -> AsyncIterator[Mapping[str, Any]]:
143
+ """ConverseStream events of one attempt, each within the attempt timers."""
144
+ client = await self._pool.bedrock(route)
145
+ request = {**params, "modelId": route.model_id}
146
+ reply = await first(
147
+ client.converse_stream(**request), timing.first_response_secs, "first response"
148
+ )
149
+ stream = reply["stream"]
150
+ try:
151
+ async for event in paced(stream, timing, started=False):
152
+ yield event
153
+ finally:
154
+ # Always release the HTTP connection: a stream given up on an idle timeout (or a
155
+ # cancelled call) left it to garbage collection (aiohttp warned from __del__).
156
+ with contextlib.suppress(Exception):
157
+ stream.close()
158
+
159
+ async def call(
160
+ self,
161
+ op: str,
162
+ params: Mapping[str, Any],
163
+ route: Route,
164
+ timing: Timing,
165
+ sink: Sink | None = None,
166
+ ) -> Result:
167
+ """Run one `converse` or `invoke_model` attempt."""
168
+ if op == "converse":
169
+ events = []
170
+ async for event in self.stream(params, route, timing):
171
+ events.append(event)
172
+ if sink is not None:
173
+ await sink(event)
174
+ body = _converse_body(events)
175
+ return Result(body, _bedrock_usage(body.get("usage")))
176
+ if op == "invoke_model":
177
+ client = await self._pool.bedrock(route)
178
+ request = {**params, "modelId": route.model_id}
179
+ if isinstance(request.get("body"), Mapping | list):
180
+ request["body"] = orjson.dumps(request["body"])
181
+ request.setdefault("contentType", "application/json")
182
+ request.setdefault("accept", "application/json")
183
+ raw = await first(
184
+ client.invoke_model(**request), timing.first_response_secs, "first response"
185
+ )
186
+ async with raw["body"] as stream:
187
+ data = await first(stream.read(), timing.idle_secs, "body")
188
+ headers = raw.get("ResponseMetadata", {}).get("HTTPHeaders", {})
189
+ counted = headers.get(_TOKEN_HEADER)
190
+ response = {
191
+ "body_b64": base64.b64encode(data).decode("ascii"),
192
+ "contentType": raw.get("contentType", ""),
193
+ }
194
+ return Result(response, Usage(input_tokens=int(counted) if counted else None))
195
+ raise ValueError(f"Bedrock does not support op {op!r}")
196
+
197
+ def classify(self, exc: BaseException) -> ErrorInfo:
198
+ """By Bedrock error family (`classify_bedrock`)."""
199
+ return classify_bedrock(exc)
200
+
201
+
202
+ def decode_invoke_body(response: Mapping[str, Any]) -> Any:
203
+ """The model body of an `invoke_model` result: parsed JSON, or raw bytes."""
204
+ data = base64.b64decode(response["body_b64"])
205
+ if "json" in str(response.get("contentType", "")).lower():
206
+ return orjson.loads(data)
207
+ return data
208
+
209
+
210
+ # ------------------------------------------------------------------------------- OpenAI
211
+
212
+
213
+ def _chat_body(chunks: list[Any]) -> dict[str, Any]:
214
+ """Assemble chat.completion.chunk objects into one chat.completion body."""
215
+ text: list[str] = []
216
+ calls: dict[int, dict[str, Any]] = {}
217
+ finish = None
218
+ usage = None
219
+ ident, model, created = "", "", int(time.time())
220
+ for chunk in chunks:
221
+ ident, model, created = chunk.id or ident, chunk.model or model, chunk.created or created
222
+ if chunk.usage is not None:
223
+ usage = chunk.usage.model_dump(mode="json", exclude_none=True)
224
+ for choice in chunk.choices or []:
225
+ delta = choice.delta
226
+ if delta.content:
227
+ text.append(delta.content)
228
+ for call in delta.tool_calls or []:
229
+ slot = calls.setdefault(
230
+ call.index,
231
+ {"id": "", "type": "function", "function": {"name": "", "arguments": ""}},
232
+ )
233
+ if call.id:
234
+ slot["id"] = call.id
235
+ if call.function is not None:
236
+ slot["function"]["name"] += call.function.name or ""
237
+ slot["function"]["arguments"] += call.function.arguments or ""
238
+ if choice.finish_reason:
239
+ finish = choice.finish_reason
240
+ message: dict[str, Any] = {"role": "assistant", "content": "".join(text) or None}
241
+ if calls:
242
+ message["tool_calls"] = [calls[i] for i in sorted(calls)]
243
+ body: dict[str, Any] = {
244
+ "id": ident,
245
+ "object": "chat.completion",
246
+ "created": created,
247
+ "model": model,
248
+ "choices": [{"index": 0, "message": message, "finish_reason": finish}],
249
+ }
250
+ if usage is not None:
251
+ body["usage"] = usage
252
+ return body
253
+
254
+
255
+ def _openai_usage(usage: Mapping[str, Any] | None) -> Usage:
256
+ if not usage:
257
+ return Usage()
258
+ cached = (usage.get("prompt_tokens_details") or {}).get("cached_tokens") or 0
259
+ return Usage(
260
+ input_tokens=usage.get("prompt_tokens"),
261
+ output_tokens=usage.get("completion_tokens", 0),
262
+ cache_read_tokens=int(cached),
263
+ )
264
+
265
+
266
+ class OpenAIProvider:
267
+ """OpenAI-compatible servers: `chat` (over streaming) and `embeddings`."""
268
+
269
+ name = "openai"
270
+
271
+ def __init__(self, pool: ClientPool) -> None:
272
+ """Use `pool` for clients."""
273
+ self._pool = pool
274
+
275
+ async def stream(
276
+ self, params: Mapping[str, Any], route: Route, timing: Timing
277
+ ) -> AsyncIterator[Any]:
278
+ """Chat completion chunks of one attempt, each within the attempt timers."""
279
+ client = self._pool.openai(route)
280
+ request = {
281
+ **params,
282
+ "model": route.model_id,
283
+ "stream": True,
284
+ "stream_options": {"include_usage": True},
285
+ }
286
+ reply = await first(
287
+ client.chat.completions.create(**request), timing.first_response_secs, "first response"
288
+ )
289
+ try:
290
+ async for chunk in paced(reply, timing, started=False):
291
+ yield chunk
292
+ finally: # release the connection on a timeout or cancel too (as for Bedrock)
293
+ with contextlib.suppress(Exception):
294
+ await reply.close()
295
+
296
+ async def call(
297
+ self,
298
+ op: str,
299
+ params: Mapping[str, Any],
300
+ route: Route,
301
+ timing: Timing,
302
+ sink: Sink | None = None,
303
+ ) -> Result:
304
+ """Run one `chat` or `embeddings` attempt."""
305
+ if op == "chat":
306
+ chunks = []
307
+ async for chunk in self.stream(params, route, timing):
308
+ chunks.append(chunk)
309
+ if sink is not None:
310
+ await sink(chunk)
311
+ body = _chat_body(chunks)
312
+ return Result(body, _openai_usage(body.get("usage")))
313
+ if op == "embeddings":
314
+ client = self._pool.openai(route)
315
+ reply = await first(
316
+ client.embeddings.create(**{**params, "model": route.model_id}),
317
+ timing.first_response_secs,
318
+ "response",
319
+ )
320
+ body = reply.model_dump(mode="json", exclude_unset=True)
321
+ return Result(body, _openai_usage(body.get("usage")))
322
+ raise ValueError(f"OpenAI-compatible provider does not support op {op!r}")
323
+
324
+ def classify(self, exc: BaseException) -> ErrorInfo:
325
+ """By HTTP status (`classify_openai`)."""
326
+ return classify_openai(exc)
327
+
328
+
329
+ # ------------------------------------------------------------------------------- registry
330
+
331
+ ProviderFactory = Callable[[ClientPool], Provider]
332
+ _FACTORIES: dict[str, ProviderFactory] = {"bedrock": BedrockProvider, "openai": OpenAIProvider}
333
+
334
+
335
+ def register_provider(name: str, factory: ProviderFactory) -> None:
336
+ """Add a provider kind (pair it with `registry.register_account_kind`)."""
337
+ if not name or not callable(factory):
338
+ raise ConfigError("register_provider needs a name and a callable factory")
339
+ _FACTORIES[name] = factory
340
+
341
+
342
+ class Providers:
343
+ """One provider instance per kind over a shared `ClientPool`."""
344
+
345
+ def __init__(self, pool: ClientPool) -> None:
346
+ """Build lazily over `pool`."""
347
+ self.pool = pool
348
+ self._built: dict[str, Provider] = {}
349
+
350
+ def get(self, name: str) -> Provider:
351
+ """The provider for a kind; `ConfigError` for an unknown one."""
352
+ if name not in self._built:
353
+ if name not in _FACTORIES:
354
+ raise ConfigError(f"no provider registered for {name!r}")
355
+ self._built[name] = _FACTORIES[name](self.pool)
356
+ return self._built[name]
357
+
358
+ async def aclose(self) -> None:
359
+ """Close the pool's clients."""
360
+ await self.pool.aclose()
@@ -0,0 +1,52 @@
1
+ """Attempt timers: time to the first response, then an idle limit between chunks (plan E117).
2
+
3
+ A healthy long answer keeps sending chunks and is never cut; a region that accepts the
4
+ connection and never answers is given up after `first_response_secs`. Raised errors are
5
+ `AttemptTimeout`, a `TimeoutError`, so they classify as retryable.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ from collections.abc import AsyncIterator, Awaitable
12
+ from dataclasses import dataclass
13
+ from typing import TypeVar
14
+
15
+ _T = TypeVar("_T") # PEP 695 syntax needs 3.12; the package supports 3.11
16
+
17
+
18
+ class AttemptTimeout(TimeoutError):
19
+ """An attempt got no first response, or went idle mid-answer, for too long."""
20
+
21
+
22
+ @dataclass(frozen=True, slots=True)
23
+ class Timing:
24
+ """Limits of one attempt: until the first response, and between later chunks."""
25
+
26
+ first_response_secs: float = 60.0
27
+ idle_secs: float = 30.0
28
+
29
+
30
+ async def first(awaitable: Awaitable[_T], secs: float, what: str) -> _T:
31
+ """Await within `secs`, else raise `AttemptTimeout` naming `what`."""
32
+ try:
33
+ return await asyncio.wait_for(awaitable, secs)
34
+ except TimeoutError as exc:
35
+ raise AttemptTimeout(f"no {what} within {secs:g}s") from exc
36
+
37
+
38
+ async def paced(stream: AsyncIterator[_T], timing: Timing, *, started: bool) -> AsyncIterator[_T]:
39
+ """Yield a stream's items, each within the idle limit.
40
+
41
+ The first item gets the first-response limit unless `started` (the response already came).
42
+ """
43
+ iterator = stream.__aiter__()
44
+ limit = timing.idle_secs if started else timing.first_response_secs
45
+ what = "chunk" if started else "first response"
46
+ while True:
47
+ try:
48
+ item = await first(iterator.__anext__(), limit, what)
49
+ except StopAsyncIteration:
50
+ return
51
+ yield item
52
+ limit, what = timing.idle_secs, "chunk"
llm_queue/context.py ADDED
@@ -0,0 +1,54 @@
1
+ """Per-request context (priority, session, user, tenant) carried through tasks and threads.
2
+
3
+ Routers set it once with `request_context(...)`; queued calls and the LangChain adapters read
4
+ it when the call itself does not say. Raw threads must run under `contextvars.copy_context()`.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Iterator
10
+ from contextlib import contextmanager
11
+ from contextvars import ContextVar
12
+ from dataclasses import dataclass, replace
13
+
14
+
15
+ @dataclass(frozen=True, slots=True)
16
+ class RequestContext:
17
+ """What the current request is: its class and the ids used for sticky sessions and records."""
18
+
19
+ priority: bool = False
20
+ session_id: str | None = None
21
+ user_id: str | None = None
22
+ tenant_id: str | None = None
23
+
24
+
25
+ _CURRENT: ContextVar[RequestContext | None] = ContextVar("llm_queue_context", default=None)
26
+
27
+
28
+ def current_context() -> RequestContext:
29
+ """The active context, or the default (normal priority, no ids)."""
30
+ return _CURRENT.get() or RequestContext()
31
+
32
+
33
+ @contextmanager
34
+ def request_context(
35
+ *,
36
+ priority: bool | None = None,
37
+ session_id: str | None = None,
38
+ user_id: str | None = None,
39
+ tenant_id: str | None = None,
40
+ ) -> Iterator[RequestContext]:
41
+ """Set priority and ids for everything run inside the block; unset fields are inherited."""
42
+ base = current_context()
43
+ ctx = replace(
44
+ base,
45
+ priority=base.priority if priority is None else priority,
46
+ session_id=session_id if session_id is not None else base.session_id,
47
+ user_id=user_id if user_id is not None else base.user_id,
48
+ tenant_id=tenant_id if tenant_id is not None else base.tenant_id,
49
+ )
50
+ token = _CURRENT.set(ctx)
51
+ try:
52
+ yield ctx
53
+ finally:
54
+ _CURRENT.reset(token)