llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,216 @@
1
+ """Streaming to the caller (plan §3.13, F6, R19): chunk streams in the state store.
2
+
3
+ The worker serving a streaming call appends the provider's events to the request's chunk
4
+ stream (`P:{r:ID}:chunks`) through a `ChunkWriter`: the first event at once, later ones batched
5
+ by `stream_flush_ms`, then one `end` or `error` entry. Each caller process runs one
6
+ `ChunkReader` that reads all its open chunk streams with one blocking read.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import contextlib
13
+ import time
14
+ from collections.abc import Mapping
15
+ from typing import TYPE_CHECKING, Any
16
+
17
+ import orjson
18
+
19
+ from llm_queue._logging import get_logger
20
+
21
+ if TYPE_CHECKING:
22
+ import redis.asyncio
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ class StreamTooLarge(Exception):
28
+ """The chunk stream went above `stream_max_bytes` (plan §3.13: fail, never trim)."""
29
+
30
+
31
+ def text_of(event: Mapping[str, Any]) -> str:
32
+ """The answer text an event adds: a Bedrock `contentBlockDelta` or an OpenAI chunk delta."""
33
+ delta = event.get("contentBlockDelta", {}).get("delta", {})
34
+ if "text" in delta:
35
+ return str(delta["text"])
36
+ choices = event.get("choices") or []
37
+ if choices and isinstance(choices[0], Mapping):
38
+ content = (choices[0].get("delta") or {}).get("content")
39
+ return str(content) if content else ""
40
+ return ""
41
+
42
+
43
+ class ChunkWriter:
44
+ """Writes one attempt's events to a chunk stream; `sent` once any entry was written."""
45
+
46
+ def __init__(
47
+ self,
48
+ client: redis.asyncio.Redis,
49
+ key: str,
50
+ attempt_id: str,
51
+ *,
52
+ flush_ms: int,
53
+ max_bytes: int,
54
+ ttl_secs: int,
55
+ ) -> None:
56
+ """Keep the target; nothing is written until the first event."""
57
+ self._client = client
58
+ self.key = key
59
+ self.attempt_id = attempt_id
60
+ self._flush_secs = flush_ms / 1000
61
+ self._max_bytes = max_bytes
62
+ self._ttl = ttl_secs
63
+ self._buffer: list[Any] = []
64
+ self._seq = 0
65
+ self._bytes = 0
66
+ self._last = 0.0
67
+ self._lock = asyncio.Lock()
68
+ self._timer: asyncio.Task[None] | None = None
69
+ self._failed: Exception | None = None
70
+ self.partial: list[str] = []
71
+
72
+ @property
73
+ def sent(self) -> bool:
74
+ """True once the caller may have seen part of the answer."""
75
+ return self._seq > 0 or bool(self._buffer)
76
+
77
+ @property
78
+ def partial_text(self) -> str:
79
+ """The answer text written so far."""
80
+ return "".join(self.partial)
81
+
82
+ async def add(self, event: Any) -> None:
83
+ """One provider event: written now if it is the first or the flush window has passed."""
84
+ if self._failed is not None:
85
+ raise self._failed
86
+ data = event.model_dump(exclude_none=True) if hasattr(event, "model_dump") else event
87
+ self._buffer.append(data)
88
+ if text := text_of(data):
89
+ self.partial.append(text)
90
+ if (
91
+ self._seq == 0
92
+ or self._flush_secs == 0
93
+ or time.monotonic() - self._last >= self._flush_secs
94
+ ):
95
+ await self._flush()
96
+ elif self._timer is None or self._timer.done():
97
+ self._timer = asyncio.create_task(self._flush_later())
98
+
99
+ async def _flush_later(self) -> None:
100
+ await asyncio.sleep(self._flush_secs)
101
+ try:
102
+ await self._flush()
103
+ except Exception as exc: # StreamTooLarge or a store error: raised by the next add()/end()
104
+ self._failed = exc
105
+
106
+ async def _write(self, kind: str, data: Any) -> None:
107
+ payload = orjson.dumps(data, default=str)
108
+ self._bytes += len(payload)
109
+ if self._bytes > self._max_bytes:
110
+ raise StreamTooLarge(f"the stream is above {self._max_bytes} bytes")
111
+ seq = self._seq + 1 # counted only once written: a failed write leaves no gap
112
+ async with self._client.pipeline(transaction=False) as pipe:
113
+ pipe.xadd(self.key, {"seq": seq, "att": self.attempt_id, "t": kind, "d": payload})
114
+ pipe.expire(self.key, self._ttl) # every write: the first one may have failed
115
+ await pipe.execute()
116
+ self._seq = seq
117
+ self._last = time.monotonic()
118
+
119
+ async def _flush(self) -> None:
120
+ async with self._lock:
121
+ if self._buffer:
122
+ events, self._buffer = self._buffer, []
123
+ await self._write("chunk", events)
124
+
125
+ async def end(self, usage: Mapping[str, Any]) -> None:
126
+ """Write what is left, then the `end` entry with the usage."""
127
+ await self._stop_timer()
128
+ await self._flush()
129
+ async with self._lock:
130
+ await self._write("end", {"usage": dict(usage)})
131
+
132
+ async def error(self, kind: str, message: str) -> None:
133
+ """End the stream with an error that carries the partial text (best effort)."""
134
+ await self._stop_timer()
135
+ with contextlib.suppress(StreamTooLarge):
136
+ await self._flush()
137
+ async with self._lock:
138
+ self._bytes = 0 # the error entry is always written
139
+ await self._write(
140
+ "error", {"kind": kind, "message": message, "partial": self.partial_text}
141
+ )
142
+
143
+ async def _stop_timer(self) -> None:
144
+ if self._timer is not None and not self._timer.done():
145
+ self._timer.cancel()
146
+ with contextlib.suppress(asyncio.CancelledError):
147
+ await self._timer
148
+
149
+
150
+ class ChunkReader:
151
+ """One per caller process: one blocking read over every open chunk stream."""
152
+
153
+ def __init__(self, client: redis.asyncio.Redis, *, block_ms: int = 100) -> None:
154
+ """Keep the client; the read task starts with the first open stream.
155
+
156
+ `block_ms`: an opened stream joins the shared read within this time
157
+ (`Tuning.engine.stream_read_block_ms`).
158
+ """
159
+ self._client = client
160
+ self._block_ms = block_ms
161
+ self._open: dict[str, tuple[asyncio.Queue[dict[str, Any]], list[str]]] = {}
162
+ self._task: asyncio.Task[None] | None = None
163
+
164
+ def open(self, key: str) -> asyncio.Queue[dict[str, Any]]:
165
+ """Start reading a chunk stream from its beginning; entries arrive on the queue."""
166
+ queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
167
+ self._open[key] = (queue, ["0"])
168
+ if self._task is None or self._task.done():
169
+ self._task = asyncio.create_task(self._read())
170
+ return queue
171
+
172
+ def close(self, key: str) -> None:
173
+ """Stop reading a chunk stream."""
174
+ self._open.pop(key, None)
175
+
176
+ async def aclose(self) -> None:
177
+ """Stop the read task."""
178
+ self._open.clear()
179
+ if self._task is not None:
180
+ self._task.cancel()
181
+ with contextlib.suppress(asyncio.CancelledError):
182
+ await self._task
183
+
184
+ async def _read(self) -> None:
185
+ while self._open:
186
+ streams: dict[Any, Any] = {key: last[0] for key, (_, last) in self._open.items()}
187
+ try:
188
+ found: Any = await self._client.xread(streams, block=self._block_ms)
189
+ except asyncio.CancelledError:
190
+ raise
191
+ except Exception as exc:
192
+ logger.warning("chunk_reader_error", error=str(exc))
193
+ await asyncio.sleep(0.5)
194
+ continue
195
+ for raw_key, entries in found or []:
196
+ key = raw_key.decode() if isinstance(raw_key, bytes) else str(raw_key)
197
+ if key not in self._open:
198
+ continue
199
+ queue, last = self._open[key]
200
+ for entry_id, fields in entries:
201
+ last[0] = entry_id.decode() if isinstance(entry_id, bytes) else str(entry_id)
202
+ entry = {
203
+ (k.decode() if isinstance(k, bytes) else k): v for k, v in fields.items()
204
+ }
205
+ queue.put_nowait(
206
+ {
207
+ "seq": int(entry["seq"]),
208
+ "att": _text(entry.get("att", b"")),
209
+ "t": _text(entry["t"]),
210
+ "d": orjson.loads(entry["d"]),
211
+ }
212
+ )
213
+
214
+
215
+ def _text(value: Any) -> str:
216
+ return value.decode() if isinstance(value, bytes) else str(value)
@@ -0,0 +1,129 @@
1
+ """Token accounting (plan §3.7a): the estimate before a call and what goes back after it.
2
+
3
+ The estimate is characters / 4 of every text in the request (tool definitions included) plus
4
+ the output reservation (`max_tokens`, as Bedrock reserves it). After the call the budget is
5
+ corrected to the real usage; a throttled attempt gives everything back; a failed one keeps its
6
+ input. `register_token_estimator` replaces the character count with a real tokenizer.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+ from collections.abc import Callable, Mapping
13
+ from dataclasses import dataclass
14
+ from typing import Any
15
+
16
+ from llm_queue.clients.providers import Usage
17
+ from llm_queue.errors import ConfigError
18
+
19
+ Estimator = Callable[[str, Mapping[str, Any]], int]
20
+ _ESTIMATORS: dict[str, Estimator] = {}
21
+ # Request fields that carry numbers or switches, not text the model reads.
22
+ _SKIP = frozenset({"inferenceConfig", "modelId", "max_tokens", "temperature", "top_p", "seed"})
23
+
24
+
25
+ def register_token_estimator(name: str, fn: Estimator) -> None:
26
+ """Add an estimator chosen by the `token_estimator` setting: fn(op, params) -> input tokens."""
27
+ if not name or not callable(fn):
28
+ raise ConfigError("register_token_estimator needs a name and a callable")
29
+ _ESTIMATORS[name] = fn
30
+
31
+
32
+ def _texts(node: Any, out: list[str]) -> None:
33
+ if isinstance(node, str):
34
+ out.append(node)
35
+ elif isinstance(node, Mapping):
36
+ for key, value in node.items():
37
+ if key not in _SKIP:
38
+ _texts(value, out)
39
+ elif isinstance(node, list | tuple):
40
+ for value in node:
41
+ _texts(value, out)
42
+
43
+
44
+ def input_tokens(op: str, params: Mapping[str, Any], estimator: str | None = None) -> int:
45
+ """Estimated input tokens of a request (every text, tool schemas included)."""
46
+ if estimator and estimator in _ESTIMATORS:
47
+ return max(1, int(_ESTIMATORS[estimator](op, params)))
48
+ found: list[str] = []
49
+ _texts(params, found)
50
+ return max(1, math.ceil(sum(len(t) for t in found) / 4))
51
+
52
+
53
+ def max_tokens_of(op: str, params: Mapping[str, Any]) -> int | None:
54
+ """The `max_tokens` a request carries, if any (Converse or OpenAI shape)."""
55
+ if op == "converse":
56
+ value = (params.get("inferenceConfig") or {}).get("maxTokens")
57
+ else:
58
+ value = params.get("max_tokens") or params.get("max_completion_tokens")
59
+ return int(value) if isinstance(value, int) else None
60
+
61
+
62
+ @dataclass(frozen=True, slots=True)
63
+ class Estimate:
64
+ """What a call reserves: input tokens and the output reservation (already weighted)."""
65
+
66
+ input: int
67
+ output: int
68
+
69
+ @property
70
+ def total(self) -> int:
71
+ """Tokens taken from TPM at admission."""
72
+ return self.input + self.output
73
+
74
+
75
+ def estimate(
76
+ op: str,
77
+ params: Mapping[str, Any],
78
+ *,
79
+ embedding: bool,
80
+ default_output_tokens: int,
81
+ output_weight: float = 1.0,
82
+ estimator: str | None = None,
83
+ ) -> Estimate:
84
+ """The reservation of one call: input, plus `max_tokens` x the output weight for chat."""
85
+ inputs = input_tokens(op, params, estimator)
86
+ if embedding:
87
+ return Estimate(inputs, 0)
88
+ output = max_tokens_of(op, params) or default_output_tokens
89
+ return Estimate(inputs, math.ceil(output * output_weight))
90
+
91
+
92
+ def used_tokens(
93
+ usage: Usage, *, output_weight: float = 1.0, count_cached: bool = False
94
+ ) -> int | None:
95
+ """Tokens a finished call cost against TPM, or None when the provider reported nothing.
96
+
97
+ Bedrock: input + cache writes + output x the burndown rate; cache reads do not count.
98
+ """
99
+ if usage.input_tokens is None and usage.output_tokens is None:
100
+ return None
101
+ total = (usage.input_tokens or 0) + usage.cache_write_tokens
102
+ total += math.ceil((usage.output_tokens or 0) * output_weight)
103
+ if count_cached:
104
+ total += usage.cache_read_tokens
105
+ return total
106
+
107
+
108
+ @dataclass(frozen=True, slots=True)
109
+ class Refund:
110
+ """What goes back to the budgets after an attempt (negative tokens: charge more)."""
111
+
112
+ requests: int
113
+ tokens: int
114
+
115
+
116
+ def refund(reserved: Estimate, outcome: str, used: int | None = None) -> Refund:
117
+ """The refund rule of plan §3.7a.
118
+
119
+ `ok`: correct to the real usage (nothing back when usage is missing); `throttle`: the
120
+ request and every token back; `fail` (timeout, 5xx, cut stream): the output part back, the
121
+ input kept; `cut` (the client gave up mid-answer): nothing back.
122
+ """
123
+ if outcome == "ok":
124
+ return Refund(0, 0 if used is None else reserved.total - used)
125
+ if outcome == "throttle":
126
+ return Refund(1, reserved.total)
127
+ if outcome == "fail":
128
+ return Refund(0, reserved.output)
129
+ return Refund(0, 0)
llm_queue/errors.py ADDED
@@ -0,0 +1,73 @@
1
+ """Exception base classes and provider-error classification shared by every layer.
2
+
3
+ Callers catch one type, `LLMQueueError` (calls raise `CallFailed`, a subclass with a `kind`);
4
+ workers turn provider exceptions into `ErrorInfo` so retry, budgets and health never depend on
5
+ SDK-specific exception classes.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass
11
+ from enum import StrEnum
12
+
13
+
14
+ class LLMQueueError(Exception):
15
+ """Base class for every error this package raises to callers."""
16
+
17
+
18
+ class ConfigError(LLMQueueError):
19
+ """Invalid configuration, raised at startup or on a refused edit (never mid-call)."""
20
+
21
+
22
+ class UnknownModel(ConfigError, KeyError): # named for what it is
23
+ """A model name that is neither an alias nor a model id in the registry.
24
+
25
+ Also a `KeyError`, so code that looks models up by key keeps working (HTTP 404, CLI).
26
+ """
27
+
28
+ def __str__(self) -> str:
29
+ return f"unknown model {self.args[0]!r}"
30
+
31
+
32
+ class StoreUnavailable(LLMQueueError, ConnectionError): # named for what it is
33
+ """The state store could not serve an admin operation (unreachable, read-only, full).
34
+
35
+ Raised by `cancel`, `status`, `send_job` and the registry's edits and reads; a call
36
+ raises `CallFailed` instead. Also a `ConnectionError`, so existing handlers keep working.
37
+ """
38
+
39
+
40
+ class MissingExtra(ConfigError, ImportError): # named for what it is
41
+ """An optional feature's dependencies are not installed; names the extra to install.
42
+
43
+ Also an `ImportError`, so `except ImportError` around optional imports keeps working.
44
+ """
45
+
46
+ def __init__(self, feature: str, extra: str) -> None:
47
+ """`feature` needs `pip install 'llm-queue[extra]'`."""
48
+ super().__init__(f"{feature} needs: pip install 'llm-queue[{extra}]'")
49
+
50
+
51
+ class UnsupportedEnvelopeError(LLMQueueError):
52
+ """A message is newer than this worker understands; it is returned, never dropped."""
53
+
54
+
55
+ class ErrorKind(StrEnum):
56
+ """How the worker reacts to a provider error: give back and retry, retry, or give up."""
57
+
58
+ THROTTLE = "throttle"
59
+ RETRYABLE = "retryable"
60
+ FATAL = "fatal"
61
+
62
+
63
+ @dataclass(frozen=True, slots=True)
64
+ class ErrorInfo:
65
+ """Provider-neutral classification of one failed attempt.
66
+
67
+ `code` keeps the provider's own spelling for logs only; decisions use `kind`, because
68
+ fakes and real services spell the same failure differently.
69
+ """
70
+
71
+ kind: ErrorKind
72
+ code: str = ""
73
+ retry_after_secs: float | None = None
@@ -0,0 +1,211 @@
1
+ """httpx transport that turns OpenAI SDK HTTP requests into LLM queue calls.
2
+
3
+ Given to `openai.AsyncOpenAI(http_client=httpx.AsyncClient(transport=...))`, it answers
4
+ `/chat/completions` with `llmq.chat` and `/embeddings` with `llmq.embed`, so the SDK and
5
+ LangChain parse replies exactly as from a real server. Queue errors become OpenAI-style error
6
+ bodies with the status an OpenAI server would send (rate limits are 429 with `retry-after`).
7
+ Chat replies carry how the queue served them under the extra body key `llmq_meta`. Needs the
8
+ `openai` extra (httpx); the core package never imports this module.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import array
14
+ import base64
15
+ from collections.abc import Sequence
16
+ from typing import TYPE_CHECKING, Any
17
+
18
+ import httpx
19
+ import orjson
20
+
21
+ from llm_queue.engine.errors import CallFailed
22
+ from llm_queue.errors import ConfigError, LLMQueueError
23
+ from llm_queue.langchain._client import resolve_queue
24
+
25
+ if TYPE_CHECKING:
26
+ from llm_queue.langchain._client import QueueClient, ReplyLike
27
+
28
+ # Never resolved: every request is answered in-process by the transport.
29
+ QUEUED_BASE_URL = "http://llmq.invalid/v1"
30
+ # Extra key of a queued chat completion body: how the call was served.
31
+ META_KEY = "llmq_meta"
32
+ # Error kind -> (HTTP status, OpenAI error type).
33
+ _STATUS = {
34
+ "rate_limited": (429, "rate_limit_error"),
35
+ "queue_full": (429, "rate_limit_error"),
36
+ "retry_budget_exhausted": (429, "rate_limit_error"),
37
+ "deadline_exceeded": (504, "timeout"),
38
+ "queue_unavailable": (503, "service_unavailable"),
39
+ "model_unavailable": (503, "service_unavailable"),
40
+ "credentials_unavailable": (503, "service_unavailable"),
41
+ "model_disabled": (404, "invalid_request_error"),
42
+ "cancelled": (499, "cancelled"),
43
+ "worker_lost": (502, "server_error"),
44
+ "dead_lettered": (500, "server_error"),
45
+ }
46
+
47
+
48
+ class _RequestRejected(Exception):
49
+ """A request the transport cannot queue (bad body, streaming, unknown route)."""
50
+
51
+ def __init__(self, status: int, message: str, code: str) -> None:
52
+ super().__init__(message)
53
+ self.status = status
54
+ self.code = code
55
+
56
+
57
+ class QueuedHTTPTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
58
+ """Answer OpenAI-compatible HTTP requests from the LLM queue instead of the network.
59
+
60
+ `model_alias` overrides the body's `model`. `llmq=None` uses the default queue, looked up per
61
+ request. Auth headers are ignored: workers hold the provider credentials.
62
+ """
63
+
64
+ def __init__(
65
+ self,
66
+ llmq: QueueClient | None = None,
67
+ model_alias: str | None = None,
68
+ *,
69
+ priority: bool | None = None,
70
+ allow_fallback: str = "all",
71
+ timeout_secs: float | None = None,
72
+ ) -> None:
73
+ """Keep the queue, alias and per-call overrides; nothing is resolved until a request."""
74
+ self._llmq = llmq
75
+ self._alias = model_alias
76
+ self._priority = priority
77
+ self._timeout = timeout_secs
78
+ self._fallback = allow_fallback
79
+
80
+ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
81
+ """Route one SDK request through the async queue API."""
82
+ try:
83
+ route, alias, body = self._parse(request, await request.aread())
84
+ queue = resolve_queue(self._llmq)
85
+ if route == "chat":
86
+ reply = await queue.chat(
87
+ alias,
88
+ body,
89
+ priority=self._priority,
90
+ allow_fallback=self._fallback,
91
+ timeout_secs=self._timeout,
92
+ )
93
+ data = _chat_response(reply)
94
+ else:
95
+ texts = _embedding_inputs(body)
96
+ embedded = await queue.embed(
97
+ alias, texts, priority=self._priority, timeout_secs=self._timeout
98
+ )
99
+ data = _embedding_response(alias, embedded.vectors, body.get("encoding_format"))
100
+ except (_RequestRejected, LLMQueueError) as exc:
101
+ return _error_response(request, exc)
102
+ return httpx.Response(200, json=data, request=request)
103
+
104
+ def handle_request(self, request: httpx.Request) -> httpx.Response:
105
+ """Route one SDK request through the sync queue API (direct path)."""
106
+ try:
107
+ route, alias, body = self._parse(request, request.read())
108
+ queue = resolve_queue(self._llmq)
109
+ if route == "chat":
110
+ chat_sync = getattr(queue, "chat_sync", None) or queue.converse_sync
111
+ reply = chat_sync(
112
+ alias,
113
+ body,
114
+ priority=self._priority,
115
+ allow_fallback=self._fallback,
116
+ timeout_secs=self._timeout,
117
+ )
118
+ data = _chat_response(reply)
119
+ else:
120
+ texts = _embedding_inputs(body)
121
+ embedded = queue.embed_sync(
122
+ alias, texts, priority=self._priority, timeout_secs=self._timeout
123
+ )
124
+ data = _embedding_response(alias, embedded.vectors, body.get("encoding_format"))
125
+ except (_RequestRejected, LLMQueueError) as exc:
126
+ return _error_response(request, exc)
127
+ return httpx.Response(200, json=data, request=request)
128
+
129
+ def _parse(self, request: httpx.Request, raw: bytes) -> tuple[str, str, dict[str, Any]]:
130
+ """Return (route, alias, JSON body), rejecting what the queue cannot serve."""
131
+ path = request.url.path.rstrip("/")
132
+ if request.method != "POST":
133
+ raise _RequestRejected(405, f"{request.method} {path} is not queued", "method")
134
+ if path.endswith("/chat/completions"):
135
+ route = "chat"
136
+ elif path.endswith("/embeddings"):
137
+ route = "embeddings"
138
+ else:
139
+ raise _RequestRejected(404, f"route {path} is not supported by the queue", "route")
140
+ try:
141
+ body = orjson.loads(raw or b"{}")
142
+ except ValueError as exc:
143
+ raise _RequestRejected(400, f"request body is not JSON: {exc}", "body") from exc
144
+ if not isinstance(body, dict):
145
+ raise _RequestRejected(400, "request body must be a JSON object", "body")
146
+ if body.get("stream"):
147
+ raise _RequestRejected(400, "streaming is not supported through the queue", "stream")
148
+ alias = self._alias or body.get("model")
149
+ if not isinstance(alias, str) or not alias:
150
+ raise _RequestRejected(400, "no model alias in the transport or the body", "model")
151
+ body.pop("model", None) # the registry decides the provider model id
152
+ return route, alias, body
153
+
154
+
155
+ def _chat_response(result: ReplyLike) -> dict[str, Any]:
156
+ """The completion body plus `llmq_meta` (the SDK keeps unknown keys as extras)."""
157
+ return {**(result.response or {}), META_KEY: result.meta.model_dump(mode="json")}
158
+
159
+
160
+ def _embedding_inputs(body: dict[str, Any]) -> list[str]:
161
+ """Texts from an embeddings body; token-id inputs are rejected (the queue embeds text)."""
162
+ value = body.get("input")
163
+ if isinstance(value, str):
164
+ return [value]
165
+ if isinstance(value, list) and all(isinstance(v, str) for v in value):
166
+ return value
167
+ raise _RequestRejected(400, "embeddings input must be a string or a list of strings", "input")
168
+
169
+
170
+ def _embedding_response(
171
+ alias: str, vectors: Sequence[Sequence[float]], encoding_format: object
172
+ ) -> dict[str, Any]:
173
+ """OpenAI embeddings body; base64 is float32 in native order, as the SDK decodes it."""
174
+ as_base64 = encoding_format == "base64"
175
+ return {
176
+ "object": "list",
177
+ "model": alias,
178
+ "data": [
179
+ {
180
+ "object": "embedding",
181
+ "index": i,
182
+ "embedding": (
183
+ base64.b64encode(array.array("f", vec).tobytes()).decode("ascii")
184
+ if as_base64
185
+ else list(vec)
186
+ ),
187
+ }
188
+ for i, vec in enumerate(vectors)
189
+ ],
190
+ "usage": {"prompt_tokens": 0, "total_tokens": 0},
191
+ }
192
+
193
+
194
+ def _error_response(request: httpx.Request, exc: Exception) -> httpx.Response:
195
+ """Map a queue error to the status and body an OpenAI server would send."""
196
+ headers = {"x-llmq-error": type(exc).__name__}
197
+ if isinstance(exc, _RequestRejected):
198
+ status, err_type, code = exc.status, "invalid_request_error", exc.code
199
+ elif isinstance(exc, CallFailed):
200
+ status, err_type = _STATUS.get(exc.kind, (400, "invalid_request_error"))
201
+ code = exc.kind
202
+ headers["x-llmq-error"] = exc.kind
203
+ if exc.retry_after_secs is not None:
204
+ headers["retry-after"] = f"{exc.retry_after_secs:g}"
205
+ headers["retry-after-ms"] = str(int(exc.retry_after_secs * 1000))
206
+ elif isinstance(exc, ConfigError | KeyError):
207
+ status, err_type, code = 404, "invalid_request_error", "model_not_found"
208
+ else:
209
+ status, err_type, code = 500, "server_error", "llmq_error"
210
+ body = {"error": {"message": str(exc), "type": err_type, "param": None, "code": code}}
211
+ return httpx.Response(status, json=body, headers=headers, request=request)
llm_queue/ids.py ADDED
@@ -0,0 +1,16 @@
1
+ """Ids for requests and jobs: UUIDv7 (RFC 9562) as 32 hex characters, from `uuid-utils`.
2
+
3
+ Time-ordered, so request ids sort by creation in logs, keys and stores, which random UUIDv4 do
4
+ not. `uuid-utils` (Rust) makes one in 0.49 us against 2.4 us for `uuid.uuid4().hex` (Python
5
+ 3.12, measured 2026-11-13); the stdlib's own `uuid.uuid7` is 3.14+ only and took 3.1 us.
6
+ A v7 id starts with the time: short ids take its random tail, never its head.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from uuid_utils import uuid7
12
+
13
+
14
+ def new_id() -> str:
15
+ """A UUIDv7: 48-bit Unix milliseconds, version 7, 74 random bits, RFC variant."""
16
+ return uuid7().hex