llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,61 @@
1
+ """LangChain adapters that send model calls through the LLM queue (extras `langchain-*`).
2
+
3
+ The adapter classes are loaded on first attribute access, so importing this package (for
4
+ example to call `set_default_queue` from `LLMQueue.start()`) never imports langchain, boto3 or
5
+ httpx. `QueuedChatBedrockConverse` / `QueuedBedrockEmbeddings` need `[langchain-aws]`;
6
+ `QueuedChatOpenAI` / `QueuedOpenAIEmbeddings` need `[openai,langchain-openai]`.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from importlib import import_module
12
+ from typing import TYPE_CHECKING, Any
13
+
14
+ from llm_queue.errors import MissingExtra
15
+ from llm_queue.langchain._client import (
16
+ EmbedResultLike,
17
+ QueueClient,
18
+ ReplyLike,
19
+ get_default_queue,
20
+ set_default_queue,
21
+ )
22
+
23
+ if TYPE_CHECKING:
24
+ from llm_queue.langchain.bedrock import (
25
+ QueuedBedrockEmbeddings,
26
+ QueuedChatBedrockConverse,
27
+ )
28
+ from llm_queue.langchain.openai import QueuedChatOpenAI, QueuedOpenAIEmbeddings
29
+
30
+ # Lazy exports: attribute name -> (module, extra that provides its dependencies).
31
+ _LAZY = {
32
+ "QueuedChatBedrockConverse": ("bedrock", "langchain-aws"),
33
+ "QueuedBedrockEmbeddings": ("bedrock", "langchain-aws"),
34
+ "QueuedChatOpenAI": ("openai", "openai,langchain-openai"),
35
+ "QueuedOpenAIEmbeddings": ("openai", "openai,langchain-openai"),
36
+ }
37
+
38
+ __all__ = [
39
+ "EmbedResultLike",
40
+ "QueueClient",
41
+ "QueuedBedrockEmbeddings",
42
+ "QueuedChatBedrockConverse",
43
+ "QueuedChatOpenAI",
44
+ "QueuedOpenAIEmbeddings",
45
+ "ReplyLike",
46
+ "get_default_queue",
47
+ "set_default_queue",
48
+ ]
49
+
50
+
51
+ def __getattr__(name: str) -> Any:
52
+ """Import an adapter's module on first use, naming the extra to install if it is missing."""
53
+ if name not in _LAZY:
54
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
55
+ module, extra = _LAZY[name]
56
+ try:
57
+ value = getattr(import_module(f"{__name__}.{module}"), name)
58
+ except ImportError as exc:
59
+ raise MissingExtra(name, extra) from exc
60
+ globals()[name] = value
61
+ return value
@@ -0,0 +1,94 @@
1
+ """The client interface the adapters call, and the process-wide default queue.
2
+
3
+ Adapters type against `QueueClient` instead of importing `LLMQueue`, so they work with any object
4
+ that has the same methods (tests use a fake). Classes created at import time find the running
5
+ queue on first call through `get_default_queue`. This module never imports langchain or httpx.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Mapping, Sequence
11
+ from typing import Any, Protocol, cast, runtime_checkable
12
+
13
+ from llm_queue.errors import LLMQueueError
14
+
15
+
16
+ class ReplyLike(Protocol):
17
+ """What `converse` / `chat` return: the provider-native body and how it was served."""
18
+
19
+ @property
20
+ def response(self) -> dict[str, Any] | None: ...
21
+
22
+ @property
23
+ def meta(self) -> Any: ...
24
+
25
+
26
+ class EmbedResultLike(Protocol):
27
+ """What `embed` returns: one vector per text, in input order."""
28
+
29
+ @property
30
+ def vectors(self) -> list[list[float]]: ...
31
+
32
+
33
+ @runtime_checkable
34
+ class QueueClient(Protocol):
35
+ """The subset of `LLMQueue` the LangChain adapters and the HTTP transport use.
36
+
37
+ `priority=None` takes it from `request_context`. Sync methods run on the direct path.
38
+ """
39
+
40
+ async def converse(
41
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
42
+ ) -> ReplyLike: ...
43
+
44
+ def converse_sync(
45
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
46
+ ) -> ReplyLike: ...
47
+
48
+ async def chat(
49
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
50
+ ) -> ReplyLike: ...
51
+
52
+ async def embed(
53
+ self,
54
+ model: str,
55
+ texts: Sequence[str],
56
+ *,
57
+ priority: bool | None = None,
58
+ input_type: str | None = None,
59
+ **kw: Any,
60
+ ) -> EmbedResultLike: ...
61
+
62
+ def embed_sync(
63
+ self,
64
+ model: str,
65
+ texts: Sequence[str],
66
+ *,
67
+ priority: bool | None = None,
68
+ input_type: str | None = None,
69
+ **kw: Any,
70
+ ) -> EmbedResultLike: ...
71
+
72
+
73
+ _default_queue: QueueClient | None = None
74
+
75
+
76
+ def set_default_queue(llmq: QueueClient | None) -> None:
77
+ """Register the queue that adapters built without `llmq=` use; `None` clears it."""
78
+ global _default_queue
79
+ _default_queue = llmq
80
+
81
+
82
+ def get_default_queue() -> QueueClient:
83
+ """The registered default queue; raises `LLMQueueError` when none has been set yet."""
84
+ if _default_queue is None:
85
+ raise LLMQueueError(
86
+ "no LLM queue: pass llmq=... to the adapter or call "
87
+ "llm_queue.langchain.set_default_queue(llmq) (LLMQueue.start() does this)"
88
+ )
89
+ return _default_queue
90
+
91
+
92
+ def resolve_queue(explicit: Any) -> QueueClient:
93
+ """The adapter's own queue if it has one, else the default, looked up on every call."""
94
+ return cast("QueueClient", explicit) if explicit is not None else get_default_queue()
@@ -0,0 +1,312 @@
1
+ """Queued Bedrock adapters: `ChatBedrockConverse` and `BedrockEmbeddings` that call the LLM queue.
2
+
3
+ The chat class runs the parent's own `_generate` against a placeholder client that hands the
4
+ Converse request to the queue, so messages, tools, cache points, reasoning and usage are built
5
+ and parsed exactly as the installed langchain-aws does (0.2 and 1.x lines alike). No boto3
6
+ client is ever created: both classes get a placeholder client that refuses direct calls.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import contextlib
12
+ import os
13
+ from collections.abc import Callable
14
+ from contextvars import ContextVar
15
+ from typing import TYPE_CHECKING, Any, Literal
16
+
17
+ from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
18
+ from pydantic import Field, model_validator
19
+
20
+ from llm_queue.errors import LLMQueueError
21
+ from llm_queue.langchain._client import get_default_queue, resolve_queue
22
+
23
+ if TYPE_CHECKING:
24
+ from langchain_core.callbacks import (
25
+ AsyncCallbackManagerForLLMRun,
26
+ CallbackManagerForLLMRun,
27
+ )
28
+ from langchain_core.messages import BaseMessage
29
+ from langchain_core.outputs import ChatResult
30
+
31
+ from llm_queue.langchain._client import QueueClient, ReplyLike
32
+
33
+ # Per-call Converse handler of the placeholder client (a ContextVar: concurrent calls on one
34
+ # model instance never see each other's handler).
35
+ _Handler = Callable[[dict[str, Any]], dict[str, Any]]
36
+ _HANDLER: ContextVar[_Handler | None] = ContextVar("llmq_converse_handler", default=None)
37
+
38
+
39
+ class _Captured(Exception): # control flow, not an error
40
+ """Raised by the capturing handler to stop the parent right after it built the request."""
41
+
42
+ def __init__(self, request: dict[str, Any]) -> None:
43
+ super().__init__("captured")
44
+ self.request = request
45
+
46
+
47
+ class _QueueOnlyClient:
48
+ """Placeholder for the boto3 client: stops the parent creating one and refuses direct calls.
49
+
50
+ `converse` answers only inside a queued call (through the current handler); any other path
51
+ that would reach Bedrock directly (e.g. a parent `_stream`) fails loudly.
52
+ """
53
+
54
+ def converse(self, **request: Any) -> dict[str, Any]:
55
+ handler = _HANDLER.get()
56
+ if handler is None:
57
+ self._refuse()
58
+ assert handler is not None
59
+ return handler(request)
60
+
61
+ def _refuse(self, *_args: Any, **_kwargs: Any) -> Any:
62
+ raise LLMQueueError("this model sends every call through the LLM queue")
63
+
64
+ converse_stream = invoke_model = get_inference_profile = _refuse
65
+
66
+
67
+ def _queue_only_values(values: Any, fields: Any, *slots: str) -> Any:
68
+ """Fill the given boto3 client slots with the placeholder unless the caller supplied them.
69
+
70
+ Only slots that are fields of the installed langchain-aws class: 0.2.24 has no
71
+ `bedrock_client`, and an unknown key moves into `model_kwargs`, which reach the Converse
72
+ request (the placeholder then broke envelope serialization in an app).
73
+ """
74
+ if isinstance(values, dict):
75
+ for slot in slots:
76
+ if slot in fields:
77
+ values.setdefault(slot, _QueueOnlyClient())
78
+ return values
79
+
80
+
81
+ def _looks_like_bedrock_id(model_id: str) -> bool:
82
+ """Bedrock ids are `provider.model[...]` (optionally region-prefixed) or ARNs."""
83
+ return "." in model_id or model_id.startswith("arn:")
84
+
85
+
86
+ def _alias_values(values: Any) -> Any:
87
+ """Resolve `model_alias` / `model_id` and reject a queue alias passed as the model id.
88
+
89
+ Without `model_id`, it is looked up from the registry of the given queue, else of the
90
+ started default queue.
91
+ """
92
+ if not isinstance(values, dict):
93
+ return values
94
+ model_id = values.get("model_id", values.get("model"))
95
+ alias = values.get("model_alias")
96
+ if model_id is None and alias is not None:
97
+ queue = values.get("llmq")
98
+ if queue is None:
99
+ with contextlib.suppress(LLMQueueError):
100
+ queue = get_default_queue()
101
+ registry = getattr(queue, "registry", None)
102
+ if registry is None:
103
+ raise ValueError(
104
+ f"model_alias={alias!r} needs model_id='<Bedrock model id>', llmq=<LLMQueue>, "
105
+ "or a started LLMQueue so it can be looked up from the registry"
106
+ )
107
+ values["model_id"] = registry.current.model(alias).model_id
108
+ elif isinstance(model_id, str) and alias is None and not _looks_like_bedrock_id(model_id):
109
+ raise ValueError(
110
+ f"model={model_id!r} looks like a queue alias, not a Bedrock model id; pass "
111
+ f"model_alias={model_id!r} and model_id='<Bedrock model id>' "
112
+ "(e.g. 'openai.gpt-oss-120b-1:0'), or model_alias plus llmq=<LLMQueue>"
113
+ )
114
+ return values
115
+
116
+
117
+ def _service_tier(tier: Any) -> dict[str, Any]:
118
+ return tier if isinstance(tier, dict) else {"type": tier}
119
+
120
+
121
+ class QueuedChatBedrockConverse(ChatBedrockConverse):
122
+ """`ChatBedrockConverse` whose Converse calls go through the LLM queue.
123
+
124
+ `model_alias` is the registry alias (`gpt120b`); `model_id` is the Bedrock id (looked up
125
+ from the registry when omitted) and drives the parent's provider-specific request building.
126
+ `priority=None` takes the class from `request_context`. Streaming is off (one full message).
127
+ """
128
+
129
+ llmq: Any = Field(default=None, exclude=True)
130
+ model_alias: str | None = None
131
+ allow_fallback: str = "all"
132
+ priority: bool | None = None
133
+ # langchain-aws 1.x types it as these values (0.2.x has no such field: ours adds it).
134
+ service_tier: Literal["priority", "default", "flex", "reserved"] | None = None
135
+ queue_timeout_secs: float | None = None
136
+
137
+ @model_validator(mode="before")
138
+ @classmethod
139
+ def _queue_defaults(cls, values: Any) -> Any:
140
+ values = _queue_only_values(
141
+ _alias_values(values), cls.model_fields, "client", "bedrock_client"
142
+ )
143
+ if isinstance(values, dict):
144
+ values["disable_streaming"] = True
145
+ # Traces (Langfuse, LangSmith) keep the parent's run name, so dashboards still match.
146
+ values.setdefault("name", ChatBedrockConverse.__name__)
147
+ return values
148
+
149
+ def _queue(self) -> QueueClient:
150
+ return resolve_queue(self.llmq)
151
+
152
+ @property
153
+ def _alias(self) -> str:
154
+ return self.model_alias or self.model_id
155
+
156
+ def _run_parent(
157
+ self,
158
+ messages: list[BaseMessage],
159
+ stop: list[str] | None,
160
+ kwargs: dict[str, Any],
161
+ handler: _Handler,
162
+ ) -> ChatResult:
163
+ token = _HANDLER.set(handler)
164
+ try:
165
+ return super()._generate(messages, stop, None, **kwargs)
166
+ finally:
167
+ _HANDLER.reset(token)
168
+
169
+ def _capture(
170
+ self, messages: list[BaseMessage], stop: list[str] | None, kwargs: dict[str, Any]
171
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
172
+ """(Converse request without `modelId`, kwargs for the parse pass) from the parent."""
173
+ call_kwargs = dict(kwargs)
174
+ tier = call_kwargs.pop("service_tier", None) # langchain-aws 0.2 has no such kwarg
175
+
176
+ def capture(request: dict[str, Any]) -> dict[str, Any]:
177
+ raise _Captured(request)
178
+
179
+ try:
180
+ self._run_parent(messages, stop, call_kwargs, capture)
181
+ except _Captured as captured:
182
+ request = dict(captured.request)
183
+ else: # pragma: no cover - the parent always calls converse
184
+ raise LLMQueueError("ChatBedrockConverse did not build a Converse request")
185
+ request.pop("modelId", None) # the queue alias picks the model and target
186
+ if tier is not None:
187
+ request["serviceTier"] = _service_tier(tier)
188
+ elif self.service_tier and "serviceTier" not in request:
189
+ request["serviceTier"] = _service_tier(self.service_tier)
190
+ return request, call_kwargs
191
+
192
+ def _parse(
193
+ self,
194
+ messages: list[BaseMessage],
195
+ stop: list[str] | None,
196
+ call_kwargs: dict[str, Any],
197
+ result: ReplyLike,
198
+ ) -> ChatResult:
199
+ """Parse like the parent (same call, canned reply), then add how the queue served it."""
200
+ response = result.response or {}
201
+ chat = self._run_parent(messages, stop, call_kwargs, lambda _request: response)
202
+ meta = result.meta.model_dump()
203
+ for generation in chat.generations:
204
+ generation.message.response_metadata.update(meta)
205
+ served = result.meta.served_model
206
+ if served and served != self._alias:
207
+ # Model fallback: report the model that answered (tracers price by `llm_output`).
208
+ registry = getattr(self._queue(), "registry", None)
209
+ served_id = registry.current.model(served).model_id if registry else served
210
+ chat.llm_output = {**(chat.llm_output or {}), "model_name": served_id}
211
+ return chat
212
+
213
+ def _generate(
214
+ self,
215
+ messages: list[BaseMessage],
216
+ stop: list[str] | None = None,
217
+ run_manager: CallbackManagerForLLMRun | None = None,
218
+ **kwargs: Any,
219
+ ) -> ChatResult:
220
+ """Sync path: `converse_sync` (direct under the shared limiter, see README)."""
221
+ request, call_kwargs = self._capture(messages, stop, kwargs)
222
+ result = self._queue().converse_sync(
223
+ self._alias,
224
+ request,
225
+ priority=self.priority,
226
+ allow_fallback=self.allow_fallback,
227
+ timeout_secs=self.queue_timeout_secs,
228
+ )
229
+ return self._parse(messages, stop, call_kwargs, result)
230
+
231
+ async def _agenerate(
232
+ self,
233
+ messages: list[BaseMessage],
234
+ stop: list[str] | None = None,
235
+ run_manager: AsyncCallbackManagerForLLMRun | None = None,
236
+ **kwargs: Any,
237
+ ) -> ChatResult:
238
+ """Native async path: one queued Converse call (the parent only has a thread fallback)."""
239
+ request, call_kwargs = self._capture(messages, stop, kwargs)
240
+ result = await self._queue().converse(
241
+ self._alias,
242
+ request,
243
+ priority=self.priority,
244
+ allow_fallback=self.allow_fallback,
245
+ timeout_secs=self.queue_timeout_secs,
246
+ )
247
+ return self._parse(messages, stop, call_kwargs, result)
248
+
249
+
250
+ class QueuedBedrockEmbeddings(BedrockEmbeddings):
251
+ """`BedrockEmbeddings` whose calls go through the queue's `embed` op (batched per provider).
252
+
253
+ `model_alias` is the queue alias (defaults to `model_id`; the queue builds the provider
254
+ body). Queries use `input_type="search_query"`, documents `"search_document"`; the provider
255
+ ignores it where the model has no such field. `normalize` is applied locally.
256
+ """
257
+
258
+ llmq: Any = Field(default=None, exclude=True)
259
+ model_alias: str | None = None
260
+ priority: bool | None = None
261
+ queue_timeout_secs: float | None = None
262
+
263
+ @model_validator(mode="before")
264
+ @classmethod
265
+ def _queue_defaults(cls, values: Any) -> Any:
266
+ return _queue_only_values(values, cls.model_fields, "client")
267
+
268
+ def _queue(self) -> QueueClient:
269
+ return resolve_queue(self.llmq)
270
+
271
+ def _prepare(self, texts: list[str]) -> list[str]:
272
+ """Same newline handling as the parent, which says newlines hurt embedding quality."""
273
+ return [text.replace(os.linesep, " ") for text in texts]
274
+
275
+ def _finish(self, vectors: list[list[float]]) -> list[list[float]]:
276
+ return [self._normalize_vector(v) for v in vectors] if self.normalize else vectors
277
+
278
+ def _sync(self, texts: list[str], input_type: str) -> list[list[float]]:
279
+ result = self._queue().embed_sync(
280
+ self.model_alias or self.model_id,
281
+ self._prepare(texts),
282
+ priority=self.priority,
283
+ input_type=input_type,
284
+ timeout_secs=self.queue_timeout_secs,
285
+ )
286
+ return self._finish(result.vectors)
287
+
288
+ async def _async(self, texts: list[str], input_type: str) -> list[list[float]]:
289
+ result = await self._queue().embed(
290
+ self.model_alias or self.model_id,
291
+ self._prepare(texts),
292
+ priority=self.priority,
293
+ input_type=input_type,
294
+ timeout_secs=self.queue_timeout_secs,
295
+ )
296
+ return self._finish(result.vectors)
297
+
298
+ def embed_documents(self, texts: list[str]) -> list[list[float]]:
299
+ """Embed documents on the sync path (`embed_sync`)."""
300
+ return self._sync(texts, "search_document")
301
+
302
+ def embed_query(self, text: str) -> list[float]:
303
+ """Embed one query on the sync path (`embed_sync`)."""
304
+ return self._sync([text], "search_query")[0]
305
+
306
+ async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
307
+ """Embed documents through the queue as one call (the parent fans out per text)."""
308
+ return await self._async(texts, "search_document")
309
+
310
+ async def aembed_query(self, text: str) -> list[float]:
311
+ """Embed one query through the queue."""
312
+ return (await self._async([text], "search_query"))[0]
@@ -0,0 +1,102 @@
1
+ """Queued OpenAI-compatible adapters: `ChatOpenAI` / `OpenAIEmbeddings` over `QueuedHTTPTransport`.
2
+
3
+ Both SDK clients get an httpx client whose transport answers from the LLM queue (D88), so every
4
+ LangChain/SDK code path is queued and parsed unchanged. Streaming and SDK retries are forced off:
5
+ the queue's workers own retry and fallback. Chat replies get how the queue served
6
+ them (served model, deployment, attempts, queue wait) in `response_metadata`.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import TYPE_CHECKING, Any
12
+
13
+ import httpx
14
+ from langchain_openai import ChatOpenAI, OpenAIEmbeddings
15
+ from pydantic import Field, model_validator
16
+
17
+ from llm_queue.http_transport import META_KEY, QUEUED_BASE_URL, QueuedHTTPTransport
18
+
19
+ if TYPE_CHECKING:
20
+ from langchain_core.outputs import ChatResult
21
+
22
+ # Placeholder only: the SDK requires a key, but workers hold the real provider credentials.
23
+ _PLACEHOLDER_API_KEY = "llmq-queued"
24
+
25
+
26
+ def _queued_client_values(values: Any) -> Any:
27
+ """Point both SDK clients at a queued transport built from the raw constructor values."""
28
+ if not isinstance(values, dict):
29
+ return values
30
+ transport = QueuedHTTPTransport(
31
+ values.get("llmq"),
32
+ values.get("model_alias"),
33
+ priority=values.get("priority"),
34
+ allow_fallback=values.get("allow_fallback", "all"),
35
+ timeout_secs=values.get("queue_timeout_secs"),
36
+ )
37
+ values["http_async_client"] = httpx.AsyncClient(transport=transport, base_url=QUEUED_BASE_URL)
38
+ values["http_client"] = httpx.Client(transport=transport, base_url=QUEUED_BASE_URL)
39
+ values.pop("openai_api_base", None)
40
+ values["base_url"] = QUEUED_BASE_URL
41
+ values["openai_proxy"] = None # an OPENAI_PROXY env var would conflict with http_client
42
+ if values.get("api_key") is None and values.get("openai_api_key") is None:
43
+ values["api_key"] = _PLACEHOLDER_API_KEY
44
+ values["max_retries"] = 0
45
+ return values
46
+
47
+
48
+ class QueuedChatOpenAI(ChatOpenAI):
49
+ """`ChatOpenAI` whose HTTP calls are answered by the LLM queue's `chat` op.
50
+
51
+ `model_alias` is the queue alias; when unset, the request's `model` (this class's `model`)
52
+ is used as the alias. Streaming is always off, so `astream()` yields one full message.
53
+ """
54
+
55
+ llmq: Any = Field(default=None, exclude=True)
56
+ model_alias: str | None = None
57
+ allow_fallback: str = "all"
58
+ priority: bool | None = None
59
+ queue_timeout_secs: float | None = None
60
+
61
+ @model_validator(mode="before")
62
+ @classmethod
63
+ def _queue_defaults(cls, values: Any) -> Any:
64
+ values = _queued_client_values(values)
65
+ if isinstance(values, dict):
66
+ values["streaming"] = False
67
+ values["disable_streaming"] = True
68
+ values.setdefault("name", ChatOpenAI.__name__) # trace run name as the parent's
69
+ return values
70
+
71
+ def _create_chat_result(
72
+ self, response: Any, generation_info: dict[str, Any] | None = None
73
+ ) -> ChatResult:
74
+ """Parse like the parent, then add the queue's `llmq_meta` to `response_metadata`."""
75
+ result = super()._create_chat_result(response, generation_info)
76
+ if isinstance(response, dict):
77
+ meta = response.get(META_KEY)
78
+ else:
79
+ meta = (getattr(response, "model_extra", None) or {}).get(META_KEY)
80
+ if isinstance(meta, dict):
81
+ for generation in result.generations:
82
+ generation.message.response_metadata.update(meta)
83
+ return result
84
+
85
+
86
+ class QueuedOpenAIEmbeddings(OpenAIEmbeddings):
87
+ """`OpenAIEmbeddings` whose HTTP calls are answered by the LLM queue's `embed` op.
88
+
89
+ `check_embedding_ctx_length` defaults to False because the queue embeds text, not tokens,
90
+ and non-OpenAI servers do not share OpenAI's tokenizer.
91
+ """
92
+
93
+ llmq: Any = Field(default=None, exclude=True)
94
+ model_alias: str | None = None
95
+ priority: bool | None = None
96
+ queue_timeout_secs: float | None = None
97
+ check_embedding_ctx_length: bool = False
98
+
99
+ @model_validator(mode="before")
100
+ @classmethod
101
+ def _queue_defaults(cls, values: Any) -> Any:
102
+ return _queued_client_values(values)
@@ -0,0 +1,43 @@
1
+ """Provider helpers shared by the clients: error classification, embedding bodies, host checks.
2
+
3
+ The providers themselves (one attempt on a route) live in `llm_queue.clients`. Importing
4
+ this package never imports aiobotocore or openai.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from llm_queue.providers.classify import (
10
+ classify_bedrock,
11
+ classify_openai,
12
+ error_family,
13
+ is_target_error,
14
+ parse_retry_after,
15
+ )
16
+ from llm_queue.providers.embeddings import (
17
+ COHERE_MAX_BATCH,
18
+ OPENAI_DEFAULT_CHUNK_SIZE,
19
+ TITAN_MAX_BATCH,
20
+ batches,
21
+ cohere_body,
22
+ embedding_body,
23
+ max_batch_size,
24
+ parse_embeddings,
25
+ titan_body,
26
+ )
27
+
28
+ __all__ = [
29
+ "COHERE_MAX_BATCH",
30
+ "OPENAI_DEFAULT_CHUNK_SIZE",
31
+ "TITAN_MAX_BATCH",
32
+ "batches",
33
+ "classify_bedrock",
34
+ "classify_openai",
35
+ "cohere_body",
36
+ "embedding_body",
37
+ "error_family",
38
+ "is_target_error",
39
+ "max_batch_size",
40
+ "parse_embeddings",
41
+ "parse_retry_after",
42
+ "titan_body",
43
+ ]