llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""Providers on routes: one attempt of an operation, its usage, and error classification.
|
|
2
|
+
|
|
3
|
+
Chat runs on the provider's streaming API under the hood and is assembled into the plain reply
|
|
4
|
+
shape (Converse body, chat.completion), so the attempt timers apply to every chat call; the
|
|
5
|
+
caller still gets a normal reply. `register_provider` adds a provider for a custom account kind.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
import contextlib
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import Any, Protocol
|
|
16
|
+
|
|
17
|
+
import orjson
|
|
18
|
+
|
|
19
|
+
from llm_queue.clients.pool import ClientPool, Route
|
|
20
|
+
from llm_queue.clients.timing import Timing, first, paced
|
|
21
|
+
from llm_queue.errors import ConfigError, ErrorInfo
|
|
22
|
+
from llm_queue.providers.classify import classify_bedrock, classify_openai
|
|
23
|
+
|
|
24
|
+
_TOKEN_HEADER = "x-amzn-bedrock-input-token-count"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True, slots=True)
|
|
28
|
+
class Usage:
|
|
29
|
+
"""Tokens of one attempt as the provider reported them; None when it reported nothing."""
|
|
30
|
+
|
|
31
|
+
input_tokens: int | None = None
|
|
32
|
+
output_tokens: int | None = None
|
|
33
|
+
cache_read_tokens: int = 0
|
|
34
|
+
cache_write_tokens: int = 0
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True, slots=True)
|
|
38
|
+
class Result:
|
|
39
|
+
"""One successful attempt: the provider-native reply and its usage."""
|
|
40
|
+
|
|
41
|
+
response: dict[str, Any]
|
|
42
|
+
usage: Usage
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
Sink = Callable[[Any], Awaitable[None]]
|
|
46
|
+
"""Receives each streamed event of an attempt (streaming to the caller, plan §3.13)."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Provider(Protocol):
|
|
50
|
+
"""One provider kind: runs an operation on a route and classifies its failures."""
|
|
51
|
+
|
|
52
|
+
name: str
|
|
53
|
+
|
|
54
|
+
async def call(
|
|
55
|
+
self,
|
|
56
|
+
op: str,
|
|
57
|
+
params: Mapping[str, Any],
|
|
58
|
+
route: Route,
|
|
59
|
+
timing: Timing,
|
|
60
|
+
sink: Sink | None = None,
|
|
61
|
+
) -> Result:
|
|
62
|
+
"""Run one attempt; chat operations pass each streamed event to `sink`."""
|
|
63
|
+
...
|
|
64
|
+
|
|
65
|
+
def classify(self, exc: BaseException) -> ErrorInfo:
|
|
66
|
+
"""Throttle / retryable / fatal."""
|
|
67
|
+
...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# ------------------------------------------------------------------------------- Bedrock
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _converse_body(events: list[Mapping[str, Any]]) -> dict[str, Any]:
|
|
74
|
+
"""Assemble ConverseStream events into the body `converse` returns."""
|
|
75
|
+
role = "assistant"
|
|
76
|
+
blocks: dict[int, dict[str, Any]] = {}
|
|
77
|
+
tool_input: dict[int, str] = {}
|
|
78
|
+
body: dict[str, Any] = {}
|
|
79
|
+
for event in events:
|
|
80
|
+
if "messageStart" in event:
|
|
81
|
+
role = event["messageStart"].get("role", role)
|
|
82
|
+
elif "contentBlockStart" in event:
|
|
83
|
+
index = event["contentBlockStart"].get("contentBlockIndex", 0)
|
|
84
|
+
start = event["contentBlockStart"].get("start", {})
|
|
85
|
+
if "toolUse" in start:
|
|
86
|
+
blocks[index] = {"toolUse": dict(start["toolUse"])}
|
|
87
|
+
tool_input[index] = ""
|
|
88
|
+
elif "contentBlockDelta" in event:
|
|
89
|
+
index = event["contentBlockDelta"].get("contentBlockIndex", 0)
|
|
90
|
+
delta = event["contentBlockDelta"].get("delta", {})
|
|
91
|
+
if "text" in delta:
|
|
92
|
+
block = blocks.setdefault(index, {"text": ""})
|
|
93
|
+
block["text"] = block.get("text", "") + delta["text"]
|
|
94
|
+
elif "toolUse" in delta:
|
|
95
|
+
tool_input[index] = tool_input.get(index, "") + delta["toolUse"].get("input", "")
|
|
96
|
+
blocks.setdefault(index, {"toolUse": {}})
|
|
97
|
+
elif "reasoningContent" in delta:
|
|
98
|
+
block = blocks.setdefault(index, {"reasoningContent": {"reasoningText": {}}})
|
|
99
|
+
reasoning = block["reasoningContent"].setdefault("reasoningText", {})
|
|
100
|
+
part = delta["reasoningContent"]
|
|
101
|
+
if "text" in part:
|
|
102
|
+
reasoning["text"] = reasoning.get("text", "") + part["text"]
|
|
103
|
+
if "signature" in part:
|
|
104
|
+
reasoning["signature"] = part["signature"]
|
|
105
|
+
elif "messageStop" in event:
|
|
106
|
+
body["stopReason"] = event["messageStop"].get("stopReason")
|
|
107
|
+
extra = event["messageStop"].get("additionalModelResponseFields")
|
|
108
|
+
if extra is not None:
|
|
109
|
+
body["additionalModelResponseFields"] = extra
|
|
110
|
+
elif "metadata" in event:
|
|
111
|
+
for key in ("usage", "metrics", "trace"):
|
|
112
|
+
if key in event["metadata"]:
|
|
113
|
+
body[key] = event["metadata"][key]
|
|
114
|
+
for index, text in tool_input.items():
|
|
115
|
+
blocks[index]["toolUse"]["input"] = orjson.loads(text) if text else {}
|
|
116
|
+
content = [blocks[i] for i in sorted(blocks)]
|
|
117
|
+
return {"output": {"message": {"role": role, "content": content}}, **body}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _bedrock_usage(usage: Mapping[str, Any] | None) -> Usage:
|
|
121
|
+
if not usage:
|
|
122
|
+
return Usage()
|
|
123
|
+
return Usage(
|
|
124
|
+
input_tokens=usage.get("inputTokens"),
|
|
125
|
+
output_tokens=usage.get("outputTokens"),
|
|
126
|
+
cache_read_tokens=int(usage.get("cacheReadInputTokens") or 0),
|
|
127
|
+
cache_write_tokens=int(usage.get("cacheWriteInputTokens") or 0),
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class BedrockProvider:
|
|
132
|
+
"""Bedrock runtime: `converse` (over ConverseStream) and `invoke_model` (embeddings)."""
|
|
133
|
+
|
|
134
|
+
name = "bedrock"
|
|
135
|
+
|
|
136
|
+
def __init__(self, pool: ClientPool) -> None:
|
|
137
|
+
"""Use `pool` for clients."""
|
|
138
|
+
self._pool = pool
|
|
139
|
+
|
|
140
|
+
async def stream(
|
|
141
|
+
self, params: Mapping[str, Any], route: Route, timing: Timing
|
|
142
|
+
) -> AsyncIterator[Mapping[str, Any]]:
|
|
143
|
+
"""ConverseStream events of one attempt, each within the attempt timers."""
|
|
144
|
+
client = await self._pool.bedrock(route)
|
|
145
|
+
request = {**params, "modelId": route.model_id}
|
|
146
|
+
reply = await first(
|
|
147
|
+
client.converse_stream(**request), timing.first_response_secs, "first response"
|
|
148
|
+
)
|
|
149
|
+
stream = reply["stream"]
|
|
150
|
+
try:
|
|
151
|
+
async for event in paced(stream, timing, started=False):
|
|
152
|
+
yield event
|
|
153
|
+
finally:
|
|
154
|
+
# Always release the HTTP connection: a stream given up on an idle timeout (or a
|
|
155
|
+
# cancelled call) left it to garbage collection (aiohttp warned from __del__).
|
|
156
|
+
with contextlib.suppress(Exception):
|
|
157
|
+
stream.close()
|
|
158
|
+
|
|
159
|
+
async def call(
|
|
160
|
+
self,
|
|
161
|
+
op: str,
|
|
162
|
+
params: Mapping[str, Any],
|
|
163
|
+
route: Route,
|
|
164
|
+
timing: Timing,
|
|
165
|
+
sink: Sink | None = None,
|
|
166
|
+
) -> Result:
|
|
167
|
+
"""Run one `converse` or `invoke_model` attempt."""
|
|
168
|
+
if op == "converse":
|
|
169
|
+
events = []
|
|
170
|
+
async for event in self.stream(params, route, timing):
|
|
171
|
+
events.append(event)
|
|
172
|
+
if sink is not None:
|
|
173
|
+
await sink(event)
|
|
174
|
+
body = _converse_body(events)
|
|
175
|
+
return Result(body, _bedrock_usage(body.get("usage")))
|
|
176
|
+
if op == "invoke_model":
|
|
177
|
+
client = await self._pool.bedrock(route)
|
|
178
|
+
request = {**params, "modelId": route.model_id}
|
|
179
|
+
if isinstance(request.get("body"), Mapping | list):
|
|
180
|
+
request["body"] = orjson.dumps(request["body"])
|
|
181
|
+
request.setdefault("contentType", "application/json")
|
|
182
|
+
request.setdefault("accept", "application/json")
|
|
183
|
+
raw = await first(
|
|
184
|
+
client.invoke_model(**request), timing.first_response_secs, "first response"
|
|
185
|
+
)
|
|
186
|
+
async with raw["body"] as stream:
|
|
187
|
+
data = await first(stream.read(), timing.idle_secs, "body")
|
|
188
|
+
headers = raw.get("ResponseMetadata", {}).get("HTTPHeaders", {})
|
|
189
|
+
counted = headers.get(_TOKEN_HEADER)
|
|
190
|
+
response = {
|
|
191
|
+
"body_b64": base64.b64encode(data).decode("ascii"),
|
|
192
|
+
"contentType": raw.get("contentType", ""),
|
|
193
|
+
}
|
|
194
|
+
return Result(response, Usage(input_tokens=int(counted) if counted else None))
|
|
195
|
+
raise ValueError(f"Bedrock does not support op {op!r}")
|
|
196
|
+
|
|
197
|
+
def classify(self, exc: BaseException) -> ErrorInfo:
|
|
198
|
+
"""By Bedrock error family (`classify_bedrock`)."""
|
|
199
|
+
return classify_bedrock(exc)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def decode_invoke_body(response: Mapping[str, Any]) -> Any:
|
|
203
|
+
"""The model body of an `invoke_model` result: parsed JSON, or raw bytes."""
|
|
204
|
+
data = base64.b64decode(response["body_b64"])
|
|
205
|
+
if "json" in str(response.get("contentType", "")).lower():
|
|
206
|
+
return orjson.loads(data)
|
|
207
|
+
return data
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# ------------------------------------------------------------------------------- OpenAI
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _chat_body(chunks: list[Any]) -> dict[str, Any]:
|
|
214
|
+
"""Assemble chat.completion.chunk objects into one chat.completion body."""
|
|
215
|
+
text: list[str] = []
|
|
216
|
+
calls: dict[int, dict[str, Any]] = {}
|
|
217
|
+
finish = None
|
|
218
|
+
usage = None
|
|
219
|
+
ident, model, created = "", "", int(time.time())
|
|
220
|
+
for chunk in chunks:
|
|
221
|
+
ident, model, created = chunk.id or ident, chunk.model or model, chunk.created or created
|
|
222
|
+
if chunk.usage is not None:
|
|
223
|
+
usage = chunk.usage.model_dump(mode="json", exclude_none=True)
|
|
224
|
+
for choice in chunk.choices or []:
|
|
225
|
+
delta = choice.delta
|
|
226
|
+
if delta.content:
|
|
227
|
+
text.append(delta.content)
|
|
228
|
+
for call in delta.tool_calls or []:
|
|
229
|
+
slot = calls.setdefault(
|
|
230
|
+
call.index,
|
|
231
|
+
{"id": "", "type": "function", "function": {"name": "", "arguments": ""}},
|
|
232
|
+
)
|
|
233
|
+
if call.id:
|
|
234
|
+
slot["id"] = call.id
|
|
235
|
+
if call.function is not None:
|
|
236
|
+
slot["function"]["name"] += call.function.name or ""
|
|
237
|
+
slot["function"]["arguments"] += call.function.arguments or ""
|
|
238
|
+
if choice.finish_reason:
|
|
239
|
+
finish = choice.finish_reason
|
|
240
|
+
message: dict[str, Any] = {"role": "assistant", "content": "".join(text) or None}
|
|
241
|
+
if calls:
|
|
242
|
+
message["tool_calls"] = [calls[i] for i in sorted(calls)]
|
|
243
|
+
body: dict[str, Any] = {
|
|
244
|
+
"id": ident,
|
|
245
|
+
"object": "chat.completion",
|
|
246
|
+
"created": created,
|
|
247
|
+
"model": model,
|
|
248
|
+
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
|
|
249
|
+
}
|
|
250
|
+
if usage is not None:
|
|
251
|
+
body["usage"] = usage
|
|
252
|
+
return body
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _openai_usage(usage: Mapping[str, Any] | None) -> Usage:
|
|
256
|
+
if not usage:
|
|
257
|
+
return Usage()
|
|
258
|
+
cached = (usage.get("prompt_tokens_details") or {}).get("cached_tokens") or 0
|
|
259
|
+
return Usage(
|
|
260
|
+
input_tokens=usage.get("prompt_tokens"),
|
|
261
|
+
output_tokens=usage.get("completion_tokens", 0),
|
|
262
|
+
cache_read_tokens=int(cached),
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class OpenAIProvider:
|
|
267
|
+
"""OpenAI-compatible servers: `chat` (over streaming) and `embeddings`."""
|
|
268
|
+
|
|
269
|
+
name = "openai"
|
|
270
|
+
|
|
271
|
+
def __init__(self, pool: ClientPool) -> None:
|
|
272
|
+
"""Use `pool` for clients."""
|
|
273
|
+
self._pool = pool
|
|
274
|
+
|
|
275
|
+
async def stream(
|
|
276
|
+
self, params: Mapping[str, Any], route: Route, timing: Timing
|
|
277
|
+
) -> AsyncIterator[Any]:
|
|
278
|
+
"""Chat completion chunks of one attempt, each within the attempt timers."""
|
|
279
|
+
client = self._pool.openai(route)
|
|
280
|
+
request = {
|
|
281
|
+
**params,
|
|
282
|
+
"model": route.model_id,
|
|
283
|
+
"stream": True,
|
|
284
|
+
"stream_options": {"include_usage": True},
|
|
285
|
+
}
|
|
286
|
+
reply = await first(
|
|
287
|
+
client.chat.completions.create(**request), timing.first_response_secs, "first response"
|
|
288
|
+
)
|
|
289
|
+
try:
|
|
290
|
+
async for chunk in paced(reply, timing, started=False):
|
|
291
|
+
yield chunk
|
|
292
|
+
finally: # release the connection on a timeout or cancel too (as for Bedrock)
|
|
293
|
+
with contextlib.suppress(Exception):
|
|
294
|
+
await reply.close()
|
|
295
|
+
|
|
296
|
+
async def call(
|
|
297
|
+
self,
|
|
298
|
+
op: str,
|
|
299
|
+
params: Mapping[str, Any],
|
|
300
|
+
route: Route,
|
|
301
|
+
timing: Timing,
|
|
302
|
+
sink: Sink | None = None,
|
|
303
|
+
) -> Result:
|
|
304
|
+
"""Run one `chat` or `embeddings` attempt."""
|
|
305
|
+
if op == "chat":
|
|
306
|
+
chunks = []
|
|
307
|
+
async for chunk in self.stream(params, route, timing):
|
|
308
|
+
chunks.append(chunk)
|
|
309
|
+
if sink is not None:
|
|
310
|
+
await sink(chunk)
|
|
311
|
+
body = _chat_body(chunks)
|
|
312
|
+
return Result(body, _openai_usage(body.get("usage")))
|
|
313
|
+
if op == "embeddings":
|
|
314
|
+
client = self._pool.openai(route)
|
|
315
|
+
reply = await first(
|
|
316
|
+
client.embeddings.create(**{**params, "model": route.model_id}),
|
|
317
|
+
timing.first_response_secs,
|
|
318
|
+
"response",
|
|
319
|
+
)
|
|
320
|
+
body = reply.model_dump(mode="json", exclude_unset=True)
|
|
321
|
+
return Result(body, _openai_usage(body.get("usage")))
|
|
322
|
+
raise ValueError(f"OpenAI-compatible provider does not support op {op!r}")
|
|
323
|
+
|
|
324
|
+
def classify(self, exc: BaseException) -> ErrorInfo:
|
|
325
|
+
"""By HTTP status (`classify_openai`)."""
|
|
326
|
+
return classify_openai(exc)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
# ------------------------------------------------------------------------------- registry
|
|
330
|
+
|
|
331
|
+
ProviderFactory = Callable[[ClientPool], Provider]
|
|
332
|
+
_FACTORIES: dict[str, ProviderFactory] = {"bedrock": BedrockProvider, "openai": OpenAIProvider}
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def register_provider(name: str, factory: ProviderFactory) -> None:
|
|
336
|
+
"""Add a provider kind (pair it with `registry.register_account_kind`)."""
|
|
337
|
+
if not name or not callable(factory):
|
|
338
|
+
raise ConfigError("register_provider needs a name and a callable factory")
|
|
339
|
+
_FACTORIES[name] = factory
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class Providers:
|
|
343
|
+
"""One provider instance per kind over a shared `ClientPool`."""
|
|
344
|
+
|
|
345
|
+
def __init__(self, pool: ClientPool) -> None:
|
|
346
|
+
"""Build lazily over `pool`."""
|
|
347
|
+
self.pool = pool
|
|
348
|
+
self._built: dict[str, Provider] = {}
|
|
349
|
+
|
|
350
|
+
def get(self, name: str) -> Provider:
|
|
351
|
+
"""The provider for a kind; `ConfigError` for an unknown one."""
|
|
352
|
+
if name not in self._built:
|
|
353
|
+
if name not in _FACTORIES:
|
|
354
|
+
raise ConfigError(f"no provider registered for {name!r}")
|
|
355
|
+
self._built[name] = _FACTORIES[name](self.pool)
|
|
356
|
+
return self._built[name]
|
|
357
|
+
|
|
358
|
+
async def aclose(self) -> None:
|
|
359
|
+
"""Close the pool's clients."""
|
|
360
|
+
await self.pool.aclose()
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Attempt timers: time to the first response, then an idle limit between chunks (plan E117).
|
|
2
|
+
|
|
3
|
+
A healthy long answer keeps sending chunks and is never cut; a region that accepts the
|
|
4
|
+
connection and never answers is given up after `first_response_secs`. Raised errors are
|
|
5
|
+
`AttemptTimeout`, a `TimeoutError`, so they classify as retryable.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
from collections.abc import AsyncIterator, Awaitable
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import TypeVar
|
|
14
|
+
|
|
15
|
+
_T = TypeVar("_T") # PEP 695 syntax needs 3.12; the package supports 3.11
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AttemptTimeout(TimeoutError):
|
|
19
|
+
"""An attempt got no first response, or went idle mid-answer, for too long."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True, slots=True)
|
|
23
|
+
class Timing:
|
|
24
|
+
"""Limits of one attempt: until the first response, and between later chunks."""
|
|
25
|
+
|
|
26
|
+
first_response_secs: float = 60.0
|
|
27
|
+
idle_secs: float = 30.0
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def first(awaitable: Awaitable[_T], secs: float, what: str) -> _T:
|
|
31
|
+
"""Await within `secs`, else raise `AttemptTimeout` naming `what`."""
|
|
32
|
+
try:
|
|
33
|
+
return await asyncio.wait_for(awaitable, secs)
|
|
34
|
+
except TimeoutError as exc:
|
|
35
|
+
raise AttemptTimeout(f"no {what} within {secs:g}s") from exc
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def paced(stream: AsyncIterator[_T], timing: Timing, *, started: bool) -> AsyncIterator[_T]:
|
|
39
|
+
"""Yield a stream's items, each within the idle limit.
|
|
40
|
+
|
|
41
|
+
The first item gets the first-response limit unless `started` (the response already came).
|
|
42
|
+
"""
|
|
43
|
+
iterator = stream.__aiter__()
|
|
44
|
+
limit = timing.idle_secs if started else timing.first_response_secs
|
|
45
|
+
what = "chunk" if started else "first response"
|
|
46
|
+
while True:
|
|
47
|
+
try:
|
|
48
|
+
item = await first(iterator.__anext__(), limit, what)
|
|
49
|
+
except StopAsyncIteration:
|
|
50
|
+
return
|
|
51
|
+
yield item
|
|
52
|
+
limit, what = timing.idle_secs, "chunk"
|
llm_queue/context.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Per-request context (priority, session, user, tenant) carried through tasks and threads.
|
|
2
|
+
|
|
3
|
+
Routers set it once with `request_context(...)`; queued calls and the LangChain adapters read
|
|
4
|
+
it when the call itself does not say. Raw threads must run under `contextvars.copy_context()`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from contextvars import ContextVar
|
|
12
|
+
from dataclasses import dataclass, replace
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True, slots=True)
|
|
16
|
+
class RequestContext:
|
|
17
|
+
"""What the current request is: its class and the ids used for sticky sessions and records."""
|
|
18
|
+
|
|
19
|
+
priority: bool = False
|
|
20
|
+
session_id: str | None = None
|
|
21
|
+
user_id: str | None = None
|
|
22
|
+
tenant_id: str | None = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
_CURRENT: ContextVar[RequestContext | None] = ContextVar("llm_queue_context", default=None)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def current_context() -> RequestContext:
|
|
29
|
+
"""The active context, or the default (normal priority, no ids)."""
|
|
30
|
+
return _CURRENT.get() or RequestContext()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@contextmanager
|
|
34
|
+
def request_context(
|
|
35
|
+
*,
|
|
36
|
+
priority: bool | None = None,
|
|
37
|
+
session_id: str | None = None,
|
|
38
|
+
user_id: str | None = None,
|
|
39
|
+
tenant_id: str | None = None,
|
|
40
|
+
) -> Iterator[RequestContext]:
|
|
41
|
+
"""Set priority and ids for everything run inside the block; unset fields are inherited."""
|
|
42
|
+
base = current_context()
|
|
43
|
+
ctx = replace(
|
|
44
|
+
base,
|
|
45
|
+
priority=base.priority if priority is None else priority,
|
|
46
|
+
session_id=session_id if session_id is not None else base.session_id,
|
|
47
|
+
user_id=user_id if user_id is not None else base.user_id,
|
|
48
|
+
tenant_id=tenant_id if tenant_id is not None else base.tenant_id,
|
|
49
|
+
)
|
|
50
|
+
token = _CURRENT.set(ctx)
|
|
51
|
+
try:
|
|
52
|
+
yield ctx
|
|
53
|
+
finally:
|
|
54
|
+
_CURRENT.reset(token)
|