llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Streaming to the caller (plan §3.13, F6, R19): chunk streams in the state store.
|
|
2
|
+
|
|
3
|
+
The worker serving a streaming call appends the provider's events to the request's chunk
|
|
4
|
+
stream (`P:{r:ID}:chunks`) through a `ChunkWriter`: the first event at once, later ones batched
|
|
5
|
+
by `stream_flush_ms`, then one `end` or `error` entry. Each caller process runs one
|
|
6
|
+
`ChunkReader` that reads all its open chunk streams with one blocking read.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import contextlib
|
|
13
|
+
import time
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
import orjson
|
|
18
|
+
|
|
19
|
+
from llm_queue._logging import get_logger
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
import redis.asyncio
|
|
23
|
+
|
|
24
|
+
logger = get_logger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class StreamTooLarge(Exception):
|
|
28
|
+
"""The chunk stream went above `stream_max_bytes` (plan §3.13: fail, never trim)."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def text_of(event: Mapping[str, Any]) -> str:
|
|
32
|
+
"""The answer text an event adds: a Bedrock `contentBlockDelta` or an OpenAI chunk delta."""
|
|
33
|
+
delta = event.get("contentBlockDelta", {}).get("delta", {})
|
|
34
|
+
if "text" in delta:
|
|
35
|
+
return str(delta["text"])
|
|
36
|
+
choices = event.get("choices") or []
|
|
37
|
+
if choices and isinstance(choices[0], Mapping):
|
|
38
|
+
content = (choices[0].get("delta") or {}).get("content")
|
|
39
|
+
return str(content) if content else ""
|
|
40
|
+
return ""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ChunkWriter:
|
|
44
|
+
"""Writes one attempt's events to a chunk stream; `sent` once any entry was written."""
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
client: redis.asyncio.Redis,
|
|
49
|
+
key: str,
|
|
50
|
+
attempt_id: str,
|
|
51
|
+
*,
|
|
52
|
+
flush_ms: int,
|
|
53
|
+
max_bytes: int,
|
|
54
|
+
ttl_secs: int,
|
|
55
|
+
) -> None:
|
|
56
|
+
"""Keep the target; nothing is written until the first event."""
|
|
57
|
+
self._client = client
|
|
58
|
+
self.key = key
|
|
59
|
+
self.attempt_id = attempt_id
|
|
60
|
+
self._flush_secs = flush_ms / 1000
|
|
61
|
+
self._max_bytes = max_bytes
|
|
62
|
+
self._ttl = ttl_secs
|
|
63
|
+
self._buffer: list[Any] = []
|
|
64
|
+
self._seq = 0
|
|
65
|
+
self._bytes = 0
|
|
66
|
+
self._last = 0.0
|
|
67
|
+
self._lock = asyncio.Lock()
|
|
68
|
+
self._timer: asyncio.Task[None] | None = None
|
|
69
|
+
self._failed: Exception | None = None
|
|
70
|
+
self.partial: list[str] = []
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def sent(self) -> bool:
|
|
74
|
+
"""True once the caller may have seen part of the answer."""
|
|
75
|
+
return self._seq > 0 or bool(self._buffer)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def partial_text(self) -> str:
|
|
79
|
+
"""The answer text written so far."""
|
|
80
|
+
return "".join(self.partial)
|
|
81
|
+
|
|
82
|
+
async def add(self, event: Any) -> None:
|
|
83
|
+
"""One provider event: written now if it is the first or the flush window has passed."""
|
|
84
|
+
if self._failed is not None:
|
|
85
|
+
raise self._failed
|
|
86
|
+
data = event.model_dump(exclude_none=True) if hasattr(event, "model_dump") else event
|
|
87
|
+
self._buffer.append(data)
|
|
88
|
+
if text := text_of(data):
|
|
89
|
+
self.partial.append(text)
|
|
90
|
+
if (
|
|
91
|
+
self._seq == 0
|
|
92
|
+
or self._flush_secs == 0
|
|
93
|
+
or time.monotonic() - self._last >= self._flush_secs
|
|
94
|
+
):
|
|
95
|
+
await self._flush()
|
|
96
|
+
elif self._timer is None or self._timer.done():
|
|
97
|
+
self._timer = asyncio.create_task(self._flush_later())
|
|
98
|
+
|
|
99
|
+
async def _flush_later(self) -> None:
|
|
100
|
+
await asyncio.sleep(self._flush_secs)
|
|
101
|
+
try:
|
|
102
|
+
await self._flush()
|
|
103
|
+
except Exception as exc: # StreamTooLarge or a store error: raised by the next add()/end()
|
|
104
|
+
self._failed = exc
|
|
105
|
+
|
|
106
|
+
async def _write(self, kind: str, data: Any) -> None:
|
|
107
|
+
payload = orjson.dumps(data, default=str)
|
|
108
|
+
self._bytes += len(payload)
|
|
109
|
+
if self._bytes > self._max_bytes:
|
|
110
|
+
raise StreamTooLarge(f"the stream is above {self._max_bytes} bytes")
|
|
111
|
+
seq = self._seq + 1 # counted only once written: a failed write leaves no gap
|
|
112
|
+
async with self._client.pipeline(transaction=False) as pipe:
|
|
113
|
+
pipe.xadd(self.key, {"seq": seq, "att": self.attempt_id, "t": kind, "d": payload})
|
|
114
|
+
pipe.expire(self.key, self._ttl) # every write: the first one may have failed
|
|
115
|
+
await pipe.execute()
|
|
116
|
+
self._seq = seq
|
|
117
|
+
self._last = time.monotonic()
|
|
118
|
+
|
|
119
|
+
async def _flush(self) -> None:
|
|
120
|
+
async with self._lock:
|
|
121
|
+
if self._buffer:
|
|
122
|
+
events, self._buffer = self._buffer, []
|
|
123
|
+
await self._write("chunk", events)
|
|
124
|
+
|
|
125
|
+
async def end(self, usage: Mapping[str, Any]) -> None:
|
|
126
|
+
"""Write what is left, then the `end` entry with the usage."""
|
|
127
|
+
await self._stop_timer()
|
|
128
|
+
await self._flush()
|
|
129
|
+
async with self._lock:
|
|
130
|
+
await self._write("end", {"usage": dict(usage)})
|
|
131
|
+
|
|
132
|
+
async def error(self, kind: str, message: str) -> None:
|
|
133
|
+
"""End the stream with an error that carries the partial text (best effort)."""
|
|
134
|
+
await self._stop_timer()
|
|
135
|
+
with contextlib.suppress(StreamTooLarge):
|
|
136
|
+
await self._flush()
|
|
137
|
+
async with self._lock:
|
|
138
|
+
self._bytes = 0 # the error entry is always written
|
|
139
|
+
await self._write(
|
|
140
|
+
"error", {"kind": kind, "message": message, "partial": self.partial_text}
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
async def _stop_timer(self) -> None:
|
|
144
|
+
if self._timer is not None and not self._timer.done():
|
|
145
|
+
self._timer.cancel()
|
|
146
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
147
|
+
await self._timer
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class ChunkReader:
|
|
151
|
+
"""One per caller process: one blocking read over every open chunk stream."""
|
|
152
|
+
|
|
153
|
+
def __init__(self, client: redis.asyncio.Redis, *, block_ms: int = 100) -> None:
|
|
154
|
+
"""Keep the client; the read task starts with the first open stream.
|
|
155
|
+
|
|
156
|
+
`block_ms`: an opened stream joins the shared read within this time
|
|
157
|
+
(`Tuning.engine.stream_read_block_ms`).
|
|
158
|
+
"""
|
|
159
|
+
self._client = client
|
|
160
|
+
self._block_ms = block_ms
|
|
161
|
+
self._open: dict[str, tuple[asyncio.Queue[dict[str, Any]], list[str]]] = {}
|
|
162
|
+
self._task: asyncio.Task[None] | None = None
|
|
163
|
+
|
|
164
|
+
def open(self, key: str) -> asyncio.Queue[dict[str, Any]]:
|
|
165
|
+
"""Start reading a chunk stream from its beginning; entries arrive on the queue."""
|
|
166
|
+
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
167
|
+
self._open[key] = (queue, ["0"])
|
|
168
|
+
if self._task is None or self._task.done():
|
|
169
|
+
self._task = asyncio.create_task(self._read())
|
|
170
|
+
return queue
|
|
171
|
+
|
|
172
|
+
def close(self, key: str) -> None:
|
|
173
|
+
"""Stop reading a chunk stream."""
|
|
174
|
+
self._open.pop(key, None)
|
|
175
|
+
|
|
176
|
+
async def aclose(self) -> None:
|
|
177
|
+
"""Stop the read task."""
|
|
178
|
+
self._open.clear()
|
|
179
|
+
if self._task is not None:
|
|
180
|
+
self._task.cancel()
|
|
181
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
182
|
+
await self._task
|
|
183
|
+
|
|
184
|
+
async def _read(self) -> None:
|
|
185
|
+
while self._open:
|
|
186
|
+
streams: dict[Any, Any] = {key: last[0] for key, (_, last) in self._open.items()}
|
|
187
|
+
try:
|
|
188
|
+
found: Any = await self._client.xread(streams, block=self._block_ms)
|
|
189
|
+
except asyncio.CancelledError:
|
|
190
|
+
raise
|
|
191
|
+
except Exception as exc:
|
|
192
|
+
logger.warning("chunk_reader_error", error=str(exc))
|
|
193
|
+
await asyncio.sleep(0.5)
|
|
194
|
+
continue
|
|
195
|
+
for raw_key, entries in found or []:
|
|
196
|
+
key = raw_key.decode() if isinstance(raw_key, bytes) else str(raw_key)
|
|
197
|
+
if key not in self._open:
|
|
198
|
+
continue
|
|
199
|
+
queue, last = self._open[key]
|
|
200
|
+
for entry_id, fields in entries:
|
|
201
|
+
last[0] = entry_id.decode() if isinstance(entry_id, bytes) else str(entry_id)
|
|
202
|
+
entry = {
|
|
203
|
+
(k.decode() if isinstance(k, bytes) else k): v for k, v in fields.items()
|
|
204
|
+
}
|
|
205
|
+
queue.put_nowait(
|
|
206
|
+
{
|
|
207
|
+
"seq": int(entry["seq"]),
|
|
208
|
+
"att": _text(entry.get("att", b"")),
|
|
209
|
+
"t": _text(entry["t"]),
|
|
210
|
+
"d": orjson.loads(entry["d"]),
|
|
211
|
+
}
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _text(value: Any) -> str:
|
|
216
|
+
return value.decode() if isinstance(value, bytes) else str(value)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Token accounting (plan §3.7a): the estimate before a call and what goes back after it.
|
|
2
|
+
|
|
3
|
+
The estimate is characters / 4 of every text in the request (tool definitions included) plus
|
|
4
|
+
the output reservation (`max_tokens`, as Bedrock reserves it). After the call the budget is
|
|
5
|
+
corrected to the real usage; a throttled attempt gives everything back; a failed one keeps its
|
|
6
|
+
input. `register_token_estimator` replaces the character count with a real tokenizer.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Callable, Mapping
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from llm_queue.clients.providers import Usage
|
|
17
|
+
from llm_queue.errors import ConfigError
|
|
18
|
+
|
|
19
|
+
Estimator = Callable[[str, Mapping[str, Any]], int]
|
|
20
|
+
_ESTIMATORS: dict[str, Estimator] = {}
|
|
21
|
+
# Request fields that carry numbers or switches, not text the model reads.
|
|
22
|
+
_SKIP = frozenset({"inferenceConfig", "modelId", "max_tokens", "temperature", "top_p", "seed"})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def register_token_estimator(name: str, fn: Estimator) -> None:
|
|
26
|
+
"""Add an estimator chosen by the `token_estimator` setting: fn(op, params) -> input tokens."""
|
|
27
|
+
if not name or not callable(fn):
|
|
28
|
+
raise ConfigError("register_token_estimator needs a name and a callable")
|
|
29
|
+
_ESTIMATORS[name] = fn
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _texts(node: Any, out: list[str]) -> None:
|
|
33
|
+
if isinstance(node, str):
|
|
34
|
+
out.append(node)
|
|
35
|
+
elif isinstance(node, Mapping):
|
|
36
|
+
for key, value in node.items():
|
|
37
|
+
if key not in _SKIP:
|
|
38
|
+
_texts(value, out)
|
|
39
|
+
elif isinstance(node, list | tuple):
|
|
40
|
+
for value in node:
|
|
41
|
+
_texts(value, out)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def input_tokens(op: str, params: Mapping[str, Any], estimator: str | None = None) -> int:
|
|
45
|
+
"""Estimated input tokens of a request (every text, tool schemas included)."""
|
|
46
|
+
if estimator and estimator in _ESTIMATORS:
|
|
47
|
+
return max(1, int(_ESTIMATORS[estimator](op, params)))
|
|
48
|
+
found: list[str] = []
|
|
49
|
+
_texts(params, found)
|
|
50
|
+
return max(1, math.ceil(sum(len(t) for t in found) / 4))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def max_tokens_of(op: str, params: Mapping[str, Any]) -> int | None:
|
|
54
|
+
"""The `max_tokens` a request carries, if any (Converse or OpenAI shape)."""
|
|
55
|
+
if op == "converse":
|
|
56
|
+
value = (params.get("inferenceConfig") or {}).get("maxTokens")
|
|
57
|
+
else:
|
|
58
|
+
value = params.get("max_tokens") or params.get("max_completion_tokens")
|
|
59
|
+
return int(value) if isinstance(value, int) else None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True, slots=True)
|
|
63
|
+
class Estimate:
|
|
64
|
+
"""What a call reserves: input tokens and the output reservation (already weighted)."""
|
|
65
|
+
|
|
66
|
+
input: int
|
|
67
|
+
output: int
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def total(self) -> int:
|
|
71
|
+
"""Tokens taken from TPM at admission."""
|
|
72
|
+
return self.input + self.output
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def estimate(
|
|
76
|
+
op: str,
|
|
77
|
+
params: Mapping[str, Any],
|
|
78
|
+
*,
|
|
79
|
+
embedding: bool,
|
|
80
|
+
default_output_tokens: int,
|
|
81
|
+
output_weight: float = 1.0,
|
|
82
|
+
estimator: str | None = None,
|
|
83
|
+
) -> Estimate:
|
|
84
|
+
"""The reservation of one call: input, plus `max_tokens` x the output weight for chat."""
|
|
85
|
+
inputs = input_tokens(op, params, estimator)
|
|
86
|
+
if embedding:
|
|
87
|
+
return Estimate(inputs, 0)
|
|
88
|
+
output = max_tokens_of(op, params) or default_output_tokens
|
|
89
|
+
return Estimate(inputs, math.ceil(output * output_weight))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def used_tokens(
|
|
93
|
+
usage: Usage, *, output_weight: float = 1.0, count_cached: bool = False
|
|
94
|
+
) -> int | None:
|
|
95
|
+
"""Tokens a finished call cost against TPM, or None when the provider reported nothing.
|
|
96
|
+
|
|
97
|
+
Bedrock: input + cache writes + output x the burndown rate; cache reads do not count.
|
|
98
|
+
"""
|
|
99
|
+
if usage.input_tokens is None and usage.output_tokens is None:
|
|
100
|
+
return None
|
|
101
|
+
total = (usage.input_tokens or 0) + usage.cache_write_tokens
|
|
102
|
+
total += math.ceil((usage.output_tokens or 0) * output_weight)
|
|
103
|
+
if count_cached:
|
|
104
|
+
total += usage.cache_read_tokens
|
|
105
|
+
return total
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(frozen=True, slots=True)
|
|
109
|
+
class Refund:
|
|
110
|
+
"""What goes back to the budgets after an attempt (negative tokens: charge more)."""
|
|
111
|
+
|
|
112
|
+
requests: int
|
|
113
|
+
tokens: int
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def refund(reserved: Estimate, outcome: str, used: int | None = None) -> Refund:
|
|
117
|
+
"""The refund rule of plan §3.7a.
|
|
118
|
+
|
|
119
|
+
`ok`: correct to the real usage (nothing back when usage is missing); `throttle`: the
|
|
120
|
+
request and every token back; `fail` (timeout, 5xx, cut stream): the output part back, the
|
|
121
|
+
input kept; `cut` (the client gave up mid-answer): nothing back.
|
|
122
|
+
"""
|
|
123
|
+
if outcome == "ok":
|
|
124
|
+
return Refund(0, 0 if used is None else reserved.total - used)
|
|
125
|
+
if outcome == "throttle":
|
|
126
|
+
return Refund(1, reserved.total)
|
|
127
|
+
if outcome == "fail":
|
|
128
|
+
return Refund(0, reserved.output)
|
|
129
|
+
return Refund(0, 0)
|
llm_queue/errors.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Exception base classes and provider-error classification shared by every layer.
|
|
2
|
+
|
|
3
|
+
Callers catch one type, `LLMQueueError` (calls raise `CallFailed`, a subclass with a `kind`);
|
|
4
|
+
workers turn provider exceptions into `ErrorInfo` so retry, budgets and health never depend on
|
|
5
|
+
SDK-specific exception classes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from enum import StrEnum
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class LLMQueueError(Exception):
|
|
15
|
+
"""Base class for every error this package raises to callers."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ConfigError(LLMQueueError):
|
|
19
|
+
"""Invalid configuration, raised at startup or on a refused edit (never mid-call)."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class UnknownModel(ConfigError, KeyError): # named for what it is
|
|
23
|
+
"""A model name that is neither an alias nor a model id in the registry.
|
|
24
|
+
|
|
25
|
+
Also a `KeyError`, so code that looks models up by key keeps working (HTTP 404, CLI).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __str__(self) -> str:
|
|
29
|
+
return f"unknown model {self.args[0]!r}"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class StoreUnavailable(LLMQueueError, ConnectionError): # named for what it is
|
|
33
|
+
"""The state store could not serve an admin operation (unreachable, read-only, full).
|
|
34
|
+
|
|
35
|
+
Raised by `cancel`, `status`, `send_job` and the registry's edits and reads; a call
|
|
36
|
+
raises `CallFailed` instead. Also a `ConnectionError`, so existing handlers keep working.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class MissingExtra(ConfigError, ImportError): # named for what it is
|
|
41
|
+
"""An optional feature's dependencies are not installed; names the extra to install.
|
|
42
|
+
|
|
43
|
+
Also an `ImportError`, so `except ImportError` around optional imports keeps working.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(self, feature: str, extra: str) -> None:
|
|
47
|
+
"""`feature` needs `pip install 'llm-queue[extra]'`."""
|
|
48
|
+
super().__init__(f"{feature} needs: pip install 'llm-queue[{extra}]'")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class UnsupportedEnvelopeError(LLMQueueError):
|
|
52
|
+
"""A message is newer than this worker understands; it is returned, never dropped."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class ErrorKind(StrEnum):
|
|
56
|
+
"""How the worker reacts to a provider error: give back and retry, retry, or give up."""
|
|
57
|
+
|
|
58
|
+
THROTTLE = "throttle"
|
|
59
|
+
RETRYABLE = "retryable"
|
|
60
|
+
FATAL = "fatal"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True, slots=True)
|
|
64
|
+
class ErrorInfo:
|
|
65
|
+
"""Provider-neutral classification of one failed attempt.
|
|
66
|
+
|
|
67
|
+
`code` keeps the provider's own spelling for logs only; decisions use `kind`, because
|
|
68
|
+
fakes and real services spell the same failure differently.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
kind: ErrorKind
|
|
72
|
+
code: str = ""
|
|
73
|
+
retry_after_secs: float | None = None
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""httpx transport that turns OpenAI SDK HTTP requests into LLM queue calls.
|
|
2
|
+
|
|
3
|
+
Given to `openai.AsyncOpenAI(http_client=httpx.AsyncClient(transport=...))`, it answers
|
|
4
|
+
`/chat/completions` with `llmq.chat` and `/embeddings` with `llmq.embed`, so the SDK and
|
|
5
|
+
LangChain parse replies exactly as from a real server. Queue errors become OpenAI-style error
|
|
6
|
+
bodies with the status an OpenAI server would send (rate limits are 429 with `retry-after`).
|
|
7
|
+
Chat replies carry how the queue served them under the extra body key `llmq_meta`. Needs the
|
|
8
|
+
`openai` extra (httpx); the core package never imports this module.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import array
|
|
14
|
+
import base64
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
from typing import TYPE_CHECKING, Any
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
import orjson
|
|
20
|
+
|
|
21
|
+
from llm_queue.engine.errors import CallFailed
|
|
22
|
+
from llm_queue.errors import ConfigError, LLMQueueError
|
|
23
|
+
from llm_queue.langchain._client import resolve_queue
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from llm_queue.langchain._client import QueueClient, ReplyLike
|
|
27
|
+
|
|
28
|
+
# Never resolved: every request is answered in-process by the transport.
|
|
29
|
+
QUEUED_BASE_URL = "http://llmq.invalid/v1"
|
|
30
|
+
# Extra key of a queued chat completion body: how the call was served.
|
|
31
|
+
META_KEY = "llmq_meta"
|
|
32
|
+
# Error kind -> (HTTP status, OpenAI error type).
|
|
33
|
+
_STATUS = {
|
|
34
|
+
"rate_limited": (429, "rate_limit_error"),
|
|
35
|
+
"queue_full": (429, "rate_limit_error"),
|
|
36
|
+
"retry_budget_exhausted": (429, "rate_limit_error"),
|
|
37
|
+
"deadline_exceeded": (504, "timeout"),
|
|
38
|
+
"queue_unavailable": (503, "service_unavailable"),
|
|
39
|
+
"model_unavailable": (503, "service_unavailable"),
|
|
40
|
+
"credentials_unavailable": (503, "service_unavailable"),
|
|
41
|
+
"model_disabled": (404, "invalid_request_error"),
|
|
42
|
+
"cancelled": (499, "cancelled"),
|
|
43
|
+
"worker_lost": (502, "server_error"),
|
|
44
|
+
"dead_lettered": (500, "server_error"),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _RequestRejected(Exception):
|
|
49
|
+
"""A request the transport cannot queue (bad body, streaming, unknown route)."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, status: int, message: str, code: str) -> None:
|
|
52
|
+
super().__init__(message)
|
|
53
|
+
self.status = status
|
|
54
|
+
self.code = code
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class QueuedHTTPTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
58
|
+
"""Answer OpenAI-compatible HTTP requests from the LLM queue instead of the network.
|
|
59
|
+
|
|
60
|
+
`model_alias` overrides the body's `model`. `llmq=None` uses the default queue, looked up per
|
|
61
|
+
request. Auth headers are ignored: workers hold the provider credentials.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
llmq: QueueClient | None = None,
|
|
67
|
+
model_alias: str | None = None,
|
|
68
|
+
*,
|
|
69
|
+
priority: bool | None = None,
|
|
70
|
+
allow_fallback: str = "all",
|
|
71
|
+
timeout_secs: float | None = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Keep the queue, alias and per-call overrides; nothing is resolved until a request."""
|
|
74
|
+
self._llmq = llmq
|
|
75
|
+
self._alias = model_alias
|
|
76
|
+
self._priority = priority
|
|
77
|
+
self._timeout = timeout_secs
|
|
78
|
+
self._fallback = allow_fallback
|
|
79
|
+
|
|
80
|
+
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
81
|
+
"""Route one SDK request through the async queue API."""
|
|
82
|
+
try:
|
|
83
|
+
route, alias, body = self._parse(request, await request.aread())
|
|
84
|
+
queue = resolve_queue(self._llmq)
|
|
85
|
+
if route == "chat":
|
|
86
|
+
reply = await queue.chat(
|
|
87
|
+
alias,
|
|
88
|
+
body,
|
|
89
|
+
priority=self._priority,
|
|
90
|
+
allow_fallback=self._fallback,
|
|
91
|
+
timeout_secs=self._timeout,
|
|
92
|
+
)
|
|
93
|
+
data = _chat_response(reply)
|
|
94
|
+
else:
|
|
95
|
+
texts = _embedding_inputs(body)
|
|
96
|
+
embedded = await queue.embed(
|
|
97
|
+
alias, texts, priority=self._priority, timeout_secs=self._timeout
|
|
98
|
+
)
|
|
99
|
+
data = _embedding_response(alias, embedded.vectors, body.get("encoding_format"))
|
|
100
|
+
except (_RequestRejected, LLMQueueError) as exc:
|
|
101
|
+
return _error_response(request, exc)
|
|
102
|
+
return httpx.Response(200, json=data, request=request)
|
|
103
|
+
|
|
104
|
+
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
|
105
|
+
"""Route one SDK request through the sync queue API (direct path)."""
|
|
106
|
+
try:
|
|
107
|
+
route, alias, body = self._parse(request, request.read())
|
|
108
|
+
queue = resolve_queue(self._llmq)
|
|
109
|
+
if route == "chat":
|
|
110
|
+
chat_sync = getattr(queue, "chat_sync", None) or queue.converse_sync
|
|
111
|
+
reply = chat_sync(
|
|
112
|
+
alias,
|
|
113
|
+
body,
|
|
114
|
+
priority=self._priority,
|
|
115
|
+
allow_fallback=self._fallback,
|
|
116
|
+
timeout_secs=self._timeout,
|
|
117
|
+
)
|
|
118
|
+
data = _chat_response(reply)
|
|
119
|
+
else:
|
|
120
|
+
texts = _embedding_inputs(body)
|
|
121
|
+
embedded = queue.embed_sync(
|
|
122
|
+
alias, texts, priority=self._priority, timeout_secs=self._timeout
|
|
123
|
+
)
|
|
124
|
+
data = _embedding_response(alias, embedded.vectors, body.get("encoding_format"))
|
|
125
|
+
except (_RequestRejected, LLMQueueError) as exc:
|
|
126
|
+
return _error_response(request, exc)
|
|
127
|
+
return httpx.Response(200, json=data, request=request)
|
|
128
|
+
|
|
129
|
+
def _parse(self, request: httpx.Request, raw: bytes) -> tuple[str, str, dict[str, Any]]:
|
|
130
|
+
"""Return (route, alias, JSON body), rejecting what the queue cannot serve."""
|
|
131
|
+
path = request.url.path.rstrip("/")
|
|
132
|
+
if request.method != "POST":
|
|
133
|
+
raise _RequestRejected(405, f"{request.method} {path} is not queued", "method")
|
|
134
|
+
if path.endswith("/chat/completions"):
|
|
135
|
+
route = "chat"
|
|
136
|
+
elif path.endswith("/embeddings"):
|
|
137
|
+
route = "embeddings"
|
|
138
|
+
else:
|
|
139
|
+
raise _RequestRejected(404, f"route {path} is not supported by the queue", "route")
|
|
140
|
+
try:
|
|
141
|
+
body = orjson.loads(raw or b"{}")
|
|
142
|
+
except ValueError as exc:
|
|
143
|
+
raise _RequestRejected(400, f"request body is not JSON: {exc}", "body") from exc
|
|
144
|
+
if not isinstance(body, dict):
|
|
145
|
+
raise _RequestRejected(400, "request body must be a JSON object", "body")
|
|
146
|
+
if body.get("stream"):
|
|
147
|
+
raise _RequestRejected(400, "streaming is not supported through the queue", "stream")
|
|
148
|
+
alias = self._alias or body.get("model")
|
|
149
|
+
if not isinstance(alias, str) or not alias:
|
|
150
|
+
raise _RequestRejected(400, "no model alias in the transport or the body", "model")
|
|
151
|
+
body.pop("model", None) # the registry decides the provider model id
|
|
152
|
+
return route, alias, body
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _chat_response(result: ReplyLike) -> dict[str, Any]:
|
|
156
|
+
"""The completion body plus `llmq_meta` (the SDK keeps unknown keys as extras)."""
|
|
157
|
+
return {**(result.response or {}), META_KEY: result.meta.model_dump(mode="json")}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _embedding_inputs(body: dict[str, Any]) -> list[str]:
|
|
161
|
+
"""Texts from an embeddings body; token-id inputs are rejected (the queue embeds text)."""
|
|
162
|
+
value = body.get("input")
|
|
163
|
+
if isinstance(value, str):
|
|
164
|
+
return [value]
|
|
165
|
+
if isinstance(value, list) and all(isinstance(v, str) for v in value):
|
|
166
|
+
return value
|
|
167
|
+
raise _RequestRejected(400, "embeddings input must be a string or a list of strings", "input")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _embedding_response(
|
|
171
|
+
alias: str, vectors: Sequence[Sequence[float]], encoding_format: object
|
|
172
|
+
) -> dict[str, Any]:
|
|
173
|
+
"""OpenAI embeddings body; base64 is float32 in native order, as the SDK decodes it."""
|
|
174
|
+
as_base64 = encoding_format == "base64"
|
|
175
|
+
return {
|
|
176
|
+
"object": "list",
|
|
177
|
+
"model": alias,
|
|
178
|
+
"data": [
|
|
179
|
+
{
|
|
180
|
+
"object": "embedding",
|
|
181
|
+
"index": i,
|
|
182
|
+
"embedding": (
|
|
183
|
+
base64.b64encode(array.array("f", vec).tobytes()).decode("ascii")
|
|
184
|
+
if as_base64
|
|
185
|
+
else list(vec)
|
|
186
|
+
),
|
|
187
|
+
}
|
|
188
|
+
for i, vec in enumerate(vectors)
|
|
189
|
+
],
|
|
190
|
+
"usage": {"prompt_tokens": 0, "total_tokens": 0},
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _error_response(request: httpx.Request, exc: Exception) -> httpx.Response:
|
|
195
|
+
"""Map a queue error to the status and body an OpenAI server would send."""
|
|
196
|
+
headers = {"x-llmq-error": type(exc).__name__}
|
|
197
|
+
if isinstance(exc, _RequestRejected):
|
|
198
|
+
status, err_type, code = exc.status, "invalid_request_error", exc.code
|
|
199
|
+
elif isinstance(exc, CallFailed):
|
|
200
|
+
status, err_type = _STATUS.get(exc.kind, (400, "invalid_request_error"))
|
|
201
|
+
code = exc.kind
|
|
202
|
+
headers["x-llmq-error"] = exc.kind
|
|
203
|
+
if exc.retry_after_secs is not None:
|
|
204
|
+
headers["retry-after"] = f"{exc.retry_after_secs:g}"
|
|
205
|
+
headers["retry-after-ms"] = str(int(exc.retry_after_secs * 1000))
|
|
206
|
+
elif isinstance(exc, ConfigError | KeyError):
|
|
207
|
+
status, err_type, code = 404, "invalid_request_error", "model_not_found"
|
|
208
|
+
else:
|
|
209
|
+
status, err_type, code = 500, "server_error", "llmq_error"
|
|
210
|
+
body = {"error": {"message": str(exc), "type": err_type, "param": None, "code": code}}
|
|
211
|
+
return httpx.Response(status, json=body, headers=headers, request=request)
|
llm_queue/ids.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Ids for requests and jobs: UUIDv7 (RFC 9562) as 32 hex characters, from `uuid-utils`.
|
|
2
|
+
|
|
3
|
+
Time-ordered, so request ids sort by creation in logs, keys and stores, which random UUIDv4 do
|
|
4
|
+
not. `uuid-utils` (Rust) makes one in 0.49 us against 2.4 us for `uuid.uuid4().hex` (Python
|
|
5
|
+
3.12, measured 2026-11-13); the stdlib's own `uuid.uuid7` is 3.14+ only and took 3.1 us.
|
|
6
|
+
A v7 id starts with the time: short ids take its random tail, never its head.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from uuid_utils import uuid7
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def new_id() -> str:
|
|
15
|
+
"""A UUIDv7: 48-bit Unix milliseconds, version 7, 74 random bits, RFC variant."""
|
|
16
|
+
return uuid7().hex
|