llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""LangChain adapters that send model calls through the LLM queue (extras `langchain-*`).
|
|
2
|
+
|
|
3
|
+
The adapter classes are loaded on first attribute access, so importing this package (for
|
|
4
|
+
example to call `set_default_queue` from `LLMQueue.start()`) never imports langchain, boto3 or
|
|
5
|
+
httpx. `QueuedChatBedrockConverse` / `QueuedBedrockEmbeddings` need `[langchain-aws]`;
|
|
6
|
+
`QueuedChatOpenAI` / `QueuedOpenAIEmbeddings` need `[openai,langchain-openai]`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from importlib import import_module
|
|
12
|
+
from typing import TYPE_CHECKING, Any
|
|
13
|
+
|
|
14
|
+
from llm_queue.errors import MissingExtra
|
|
15
|
+
from llm_queue.langchain._client import (
|
|
16
|
+
EmbedResultLike,
|
|
17
|
+
QueueClient,
|
|
18
|
+
ReplyLike,
|
|
19
|
+
get_default_queue,
|
|
20
|
+
set_default_queue,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from llm_queue.langchain.bedrock import (
|
|
25
|
+
QueuedBedrockEmbeddings,
|
|
26
|
+
QueuedChatBedrockConverse,
|
|
27
|
+
)
|
|
28
|
+
from llm_queue.langchain.openai import QueuedChatOpenAI, QueuedOpenAIEmbeddings
|
|
29
|
+
|
|
30
|
+
# Lazy exports: attribute name -> (module, extra that provides its dependencies).
|
|
31
|
+
_LAZY = {
|
|
32
|
+
"QueuedChatBedrockConverse": ("bedrock", "langchain-aws"),
|
|
33
|
+
"QueuedBedrockEmbeddings": ("bedrock", "langchain-aws"),
|
|
34
|
+
"QueuedChatOpenAI": ("openai", "openai,langchain-openai"),
|
|
35
|
+
"QueuedOpenAIEmbeddings": ("openai", "openai,langchain-openai"),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"EmbedResultLike",
|
|
40
|
+
"QueueClient",
|
|
41
|
+
"QueuedBedrockEmbeddings",
|
|
42
|
+
"QueuedChatBedrockConverse",
|
|
43
|
+
"QueuedChatOpenAI",
|
|
44
|
+
"QueuedOpenAIEmbeddings",
|
|
45
|
+
"ReplyLike",
|
|
46
|
+
"get_default_queue",
|
|
47
|
+
"set_default_queue",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def __getattr__(name: str) -> Any:
|
|
52
|
+
"""Import an adapter's module on first use, naming the extra to install if it is missing."""
|
|
53
|
+
if name not in _LAZY:
|
|
54
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
55
|
+
module, extra = _LAZY[name]
|
|
56
|
+
try:
|
|
57
|
+
value = getattr(import_module(f"{__name__}.{module}"), name)
|
|
58
|
+
except ImportError as exc:
|
|
59
|
+
raise MissingExtra(name, extra) from exc
|
|
60
|
+
globals()[name] = value
|
|
61
|
+
return value
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""The client interface the adapters call, and the process-wide default queue.
|
|
2
|
+
|
|
3
|
+
Adapters type against `QueueClient` instead of importing `LLMQueue`, so they work with any object
|
|
4
|
+
that has the same methods (tests use a fake). Classes created at import time find the running
|
|
5
|
+
queue on first call through `get_default_queue`. This module never imports langchain or httpx.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping, Sequence
|
|
11
|
+
from typing import Any, Protocol, cast, runtime_checkable
|
|
12
|
+
|
|
13
|
+
from llm_queue.errors import LLMQueueError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ReplyLike(Protocol):
|
|
17
|
+
"""What `converse` / `chat` return: the provider-native body and how it was served."""
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def response(self) -> dict[str, Any] | None: ...
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def meta(self) -> Any: ...
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class EmbedResultLike(Protocol):
|
|
27
|
+
"""What `embed` returns: one vector per text, in input order."""
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def vectors(self) -> list[list[float]]: ...
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@runtime_checkable
|
|
34
|
+
class QueueClient(Protocol):
|
|
35
|
+
"""The subset of `LLMQueue` the LangChain adapters and the HTTP transport use.
|
|
36
|
+
|
|
37
|
+
`priority=None` takes it from `request_context`. Sync methods run on the direct path.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
async def converse(
|
|
41
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
42
|
+
) -> ReplyLike: ...
|
|
43
|
+
|
|
44
|
+
def converse_sync(
|
|
45
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
46
|
+
) -> ReplyLike: ...
|
|
47
|
+
|
|
48
|
+
async def chat(
|
|
49
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
50
|
+
) -> ReplyLike: ...
|
|
51
|
+
|
|
52
|
+
async def embed(
|
|
53
|
+
self,
|
|
54
|
+
model: str,
|
|
55
|
+
texts: Sequence[str],
|
|
56
|
+
*,
|
|
57
|
+
priority: bool | None = None,
|
|
58
|
+
input_type: str | None = None,
|
|
59
|
+
**kw: Any,
|
|
60
|
+
) -> EmbedResultLike: ...
|
|
61
|
+
|
|
62
|
+
def embed_sync(
|
|
63
|
+
self,
|
|
64
|
+
model: str,
|
|
65
|
+
texts: Sequence[str],
|
|
66
|
+
*,
|
|
67
|
+
priority: bool | None = None,
|
|
68
|
+
input_type: str | None = None,
|
|
69
|
+
**kw: Any,
|
|
70
|
+
) -> EmbedResultLike: ...
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
_default_queue: QueueClient | None = None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def set_default_queue(llmq: QueueClient | None) -> None:
|
|
77
|
+
"""Register the queue that adapters built without `llmq=` use; `None` clears it."""
|
|
78
|
+
global _default_queue
|
|
79
|
+
_default_queue = llmq
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_default_queue() -> QueueClient:
|
|
83
|
+
"""The registered default queue; raises `LLMQueueError` when none has been set yet."""
|
|
84
|
+
if _default_queue is None:
|
|
85
|
+
raise LLMQueueError(
|
|
86
|
+
"no LLM queue: pass llmq=... to the adapter or call "
|
|
87
|
+
"llm_queue.langchain.set_default_queue(llmq) (LLMQueue.start() does this)"
|
|
88
|
+
)
|
|
89
|
+
return _default_queue
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def resolve_queue(explicit: Any) -> QueueClient:
|
|
93
|
+
"""The adapter's own queue if it has one, else the default, looked up on every call."""
|
|
94
|
+
return cast("QueueClient", explicit) if explicit is not None else get_default_queue()
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Queued Bedrock adapters: `ChatBedrockConverse` and `BedrockEmbeddings` that call the LLM queue.
|
|
2
|
+
|
|
3
|
+
The chat class runs the parent's own `_generate` against a placeholder client that hands the
|
|
4
|
+
Converse request to the queue, so messages, tools, cache points, reasoning and usage are built
|
|
5
|
+
and parsed exactly as the installed langchain-aws does (0.2 and 1.x lines alike). No boto3
|
|
6
|
+
client is ever created: both classes get a placeholder client that refuses direct calls.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import os
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from contextvars import ContextVar
|
|
15
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
16
|
+
|
|
17
|
+
from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
|
|
18
|
+
from pydantic import Field, model_validator
|
|
19
|
+
|
|
20
|
+
from llm_queue.errors import LLMQueueError
|
|
21
|
+
from llm_queue.langchain._client import get_default_queue, resolve_queue
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from langchain_core.callbacks import (
|
|
25
|
+
AsyncCallbackManagerForLLMRun,
|
|
26
|
+
CallbackManagerForLLMRun,
|
|
27
|
+
)
|
|
28
|
+
from langchain_core.messages import BaseMessage
|
|
29
|
+
from langchain_core.outputs import ChatResult
|
|
30
|
+
|
|
31
|
+
from llm_queue.langchain._client import QueueClient, ReplyLike
|
|
32
|
+
|
|
33
|
+
# Per-call Converse handler of the placeholder client (a ContextVar: concurrent calls on one
|
|
34
|
+
# model instance never see each other's handler).
|
|
35
|
+
_Handler = Callable[[dict[str, Any]], dict[str, Any]]
|
|
36
|
+
_HANDLER: ContextVar[_Handler | None] = ContextVar("llmq_converse_handler", default=None)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _Captured(Exception): # control flow, not an error
|
|
40
|
+
"""Raised by the capturing handler to stop the parent right after it built the request."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, request: dict[str, Any]) -> None:
|
|
43
|
+
super().__init__("captured")
|
|
44
|
+
self.request = request
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _QueueOnlyClient:
|
|
48
|
+
"""Placeholder for the boto3 client: stops the parent creating one and refuses direct calls.
|
|
49
|
+
|
|
50
|
+
`converse` answers only inside a queued call (through the current handler); any other path
|
|
51
|
+
that would reach Bedrock directly (e.g. a parent `_stream`) fails loudly.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def converse(self, **request: Any) -> dict[str, Any]:
|
|
55
|
+
handler = _HANDLER.get()
|
|
56
|
+
if handler is None:
|
|
57
|
+
self._refuse()
|
|
58
|
+
assert handler is not None
|
|
59
|
+
return handler(request)
|
|
60
|
+
|
|
61
|
+
def _refuse(self, *_args: Any, **_kwargs: Any) -> Any:
|
|
62
|
+
raise LLMQueueError("this model sends every call through the LLM queue")
|
|
63
|
+
|
|
64
|
+
converse_stream = invoke_model = get_inference_profile = _refuse
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _queue_only_values(values: Any, fields: Any, *slots: str) -> Any:
|
|
68
|
+
"""Fill the given boto3 client slots with the placeholder unless the caller supplied them.
|
|
69
|
+
|
|
70
|
+
Only slots that are fields of the installed langchain-aws class: 0.2.24 has no
|
|
71
|
+
`bedrock_client`, and an unknown key moves into `model_kwargs`, which reach the Converse
|
|
72
|
+
request (the placeholder then broke envelope serialization in an app).
|
|
73
|
+
"""
|
|
74
|
+
if isinstance(values, dict):
|
|
75
|
+
for slot in slots:
|
|
76
|
+
if slot in fields:
|
|
77
|
+
values.setdefault(slot, _QueueOnlyClient())
|
|
78
|
+
return values
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _looks_like_bedrock_id(model_id: str) -> bool:
|
|
82
|
+
"""Bedrock ids are `provider.model[...]` (optionally region-prefixed) or ARNs."""
|
|
83
|
+
return "." in model_id or model_id.startswith("arn:")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _alias_values(values: Any) -> Any:
|
|
87
|
+
"""Resolve `model_alias` / `model_id` and reject a queue alias passed as the model id.
|
|
88
|
+
|
|
89
|
+
Without `model_id`, it is looked up from the registry of the given queue, else of the
|
|
90
|
+
started default queue.
|
|
91
|
+
"""
|
|
92
|
+
if not isinstance(values, dict):
|
|
93
|
+
return values
|
|
94
|
+
model_id = values.get("model_id", values.get("model"))
|
|
95
|
+
alias = values.get("model_alias")
|
|
96
|
+
if model_id is None and alias is not None:
|
|
97
|
+
queue = values.get("llmq")
|
|
98
|
+
if queue is None:
|
|
99
|
+
with contextlib.suppress(LLMQueueError):
|
|
100
|
+
queue = get_default_queue()
|
|
101
|
+
registry = getattr(queue, "registry", None)
|
|
102
|
+
if registry is None:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"model_alias={alias!r} needs model_id='<Bedrock model id>', llmq=<LLMQueue>, "
|
|
105
|
+
"or a started LLMQueue so it can be looked up from the registry"
|
|
106
|
+
)
|
|
107
|
+
values["model_id"] = registry.current.model(alias).model_id
|
|
108
|
+
elif isinstance(model_id, str) and alias is None and not _looks_like_bedrock_id(model_id):
|
|
109
|
+
raise ValueError(
|
|
110
|
+
f"model={model_id!r} looks like a queue alias, not a Bedrock model id; pass "
|
|
111
|
+
f"model_alias={model_id!r} and model_id='<Bedrock model id>' "
|
|
112
|
+
"(e.g. 'openai.gpt-oss-120b-1:0'), or model_alias plus llmq=<LLMQueue>"
|
|
113
|
+
)
|
|
114
|
+
return values
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _service_tier(tier: Any) -> dict[str, Any]:
|
|
118
|
+
return tier if isinstance(tier, dict) else {"type": tier}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class QueuedChatBedrockConverse(ChatBedrockConverse):
|
|
122
|
+
"""`ChatBedrockConverse` whose Converse calls go through the LLM queue.
|
|
123
|
+
|
|
124
|
+
`model_alias` is the registry alias (`gpt120b`); `model_id` is the Bedrock id (looked up
|
|
125
|
+
from the registry when omitted) and drives the parent's provider-specific request building.
|
|
126
|
+
`priority=None` takes the class from `request_context`. Streaming is off (one full message).
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
llmq: Any = Field(default=None, exclude=True)
|
|
130
|
+
model_alias: str | None = None
|
|
131
|
+
allow_fallback: str = "all"
|
|
132
|
+
priority: bool | None = None
|
|
133
|
+
# langchain-aws 1.x types it as these values (0.2.x has no such field: ours adds it).
|
|
134
|
+
service_tier: Literal["priority", "default", "flex", "reserved"] | None = None
|
|
135
|
+
queue_timeout_secs: float | None = None
|
|
136
|
+
|
|
137
|
+
@model_validator(mode="before")
|
|
138
|
+
@classmethod
|
|
139
|
+
def _queue_defaults(cls, values: Any) -> Any:
|
|
140
|
+
values = _queue_only_values(
|
|
141
|
+
_alias_values(values), cls.model_fields, "client", "bedrock_client"
|
|
142
|
+
)
|
|
143
|
+
if isinstance(values, dict):
|
|
144
|
+
values["disable_streaming"] = True
|
|
145
|
+
# Traces (Langfuse, LangSmith) keep the parent's run name, so dashboards still match.
|
|
146
|
+
values.setdefault("name", ChatBedrockConverse.__name__)
|
|
147
|
+
return values
|
|
148
|
+
|
|
149
|
+
def _queue(self) -> QueueClient:
|
|
150
|
+
return resolve_queue(self.llmq)
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def _alias(self) -> str:
|
|
154
|
+
return self.model_alias or self.model_id
|
|
155
|
+
|
|
156
|
+
def _run_parent(
|
|
157
|
+
self,
|
|
158
|
+
messages: list[BaseMessage],
|
|
159
|
+
stop: list[str] | None,
|
|
160
|
+
kwargs: dict[str, Any],
|
|
161
|
+
handler: _Handler,
|
|
162
|
+
) -> ChatResult:
|
|
163
|
+
token = _HANDLER.set(handler)
|
|
164
|
+
try:
|
|
165
|
+
return super()._generate(messages, stop, None, **kwargs)
|
|
166
|
+
finally:
|
|
167
|
+
_HANDLER.reset(token)
|
|
168
|
+
|
|
169
|
+
def _capture(
|
|
170
|
+
self, messages: list[BaseMessage], stop: list[str] | None, kwargs: dict[str, Any]
|
|
171
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
172
|
+
"""(Converse request without `modelId`, kwargs for the parse pass) from the parent."""
|
|
173
|
+
call_kwargs = dict(kwargs)
|
|
174
|
+
tier = call_kwargs.pop("service_tier", None) # langchain-aws 0.2 has no such kwarg
|
|
175
|
+
|
|
176
|
+
def capture(request: dict[str, Any]) -> dict[str, Any]:
|
|
177
|
+
raise _Captured(request)
|
|
178
|
+
|
|
179
|
+
try:
|
|
180
|
+
self._run_parent(messages, stop, call_kwargs, capture)
|
|
181
|
+
except _Captured as captured:
|
|
182
|
+
request = dict(captured.request)
|
|
183
|
+
else: # pragma: no cover - the parent always calls converse
|
|
184
|
+
raise LLMQueueError("ChatBedrockConverse did not build a Converse request")
|
|
185
|
+
request.pop("modelId", None) # the queue alias picks the model and target
|
|
186
|
+
if tier is not None:
|
|
187
|
+
request["serviceTier"] = _service_tier(tier)
|
|
188
|
+
elif self.service_tier and "serviceTier" not in request:
|
|
189
|
+
request["serviceTier"] = _service_tier(self.service_tier)
|
|
190
|
+
return request, call_kwargs
|
|
191
|
+
|
|
192
|
+
def _parse(
|
|
193
|
+
self,
|
|
194
|
+
messages: list[BaseMessage],
|
|
195
|
+
stop: list[str] | None,
|
|
196
|
+
call_kwargs: dict[str, Any],
|
|
197
|
+
result: ReplyLike,
|
|
198
|
+
) -> ChatResult:
|
|
199
|
+
"""Parse like the parent (same call, canned reply), then add how the queue served it."""
|
|
200
|
+
response = result.response or {}
|
|
201
|
+
chat = self._run_parent(messages, stop, call_kwargs, lambda _request: response)
|
|
202
|
+
meta = result.meta.model_dump()
|
|
203
|
+
for generation in chat.generations:
|
|
204
|
+
generation.message.response_metadata.update(meta)
|
|
205
|
+
served = result.meta.served_model
|
|
206
|
+
if served and served != self._alias:
|
|
207
|
+
# Model fallback: report the model that answered (tracers price by `llm_output`).
|
|
208
|
+
registry = getattr(self._queue(), "registry", None)
|
|
209
|
+
served_id = registry.current.model(served).model_id if registry else served
|
|
210
|
+
chat.llm_output = {**(chat.llm_output or {}), "model_name": served_id}
|
|
211
|
+
return chat
|
|
212
|
+
|
|
213
|
+
def _generate(
|
|
214
|
+
self,
|
|
215
|
+
messages: list[BaseMessage],
|
|
216
|
+
stop: list[str] | None = None,
|
|
217
|
+
run_manager: CallbackManagerForLLMRun | None = None,
|
|
218
|
+
**kwargs: Any,
|
|
219
|
+
) -> ChatResult:
|
|
220
|
+
"""Sync path: `converse_sync` (direct under the shared limiter, see README)."""
|
|
221
|
+
request, call_kwargs = self._capture(messages, stop, kwargs)
|
|
222
|
+
result = self._queue().converse_sync(
|
|
223
|
+
self._alias,
|
|
224
|
+
request,
|
|
225
|
+
priority=self.priority,
|
|
226
|
+
allow_fallback=self.allow_fallback,
|
|
227
|
+
timeout_secs=self.queue_timeout_secs,
|
|
228
|
+
)
|
|
229
|
+
return self._parse(messages, stop, call_kwargs, result)
|
|
230
|
+
|
|
231
|
+
async def _agenerate(
|
|
232
|
+
self,
|
|
233
|
+
messages: list[BaseMessage],
|
|
234
|
+
stop: list[str] | None = None,
|
|
235
|
+
run_manager: AsyncCallbackManagerForLLMRun | None = None,
|
|
236
|
+
**kwargs: Any,
|
|
237
|
+
) -> ChatResult:
|
|
238
|
+
"""Native async path: one queued Converse call (the parent only has a thread fallback)."""
|
|
239
|
+
request, call_kwargs = self._capture(messages, stop, kwargs)
|
|
240
|
+
result = await self._queue().converse(
|
|
241
|
+
self._alias,
|
|
242
|
+
request,
|
|
243
|
+
priority=self.priority,
|
|
244
|
+
allow_fallback=self.allow_fallback,
|
|
245
|
+
timeout_secs=self.queue_timeout_secs,
|
|
246
|
+
)
|
|
247
|
+
return self._parse(messages, stop, call_kwargs, result)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class QueuedBedrockEmbeddings(BedrockEmbeddings):
|
|
251
|
+
"""`BedrockEmbeddings` whose calls go through the queue's `embed` op (batched per provider).
|
|
252
|
+
|
|
253
|
+
`model_alias` is the queue alias (defaults to `model_id`; the queue builds the provider
|
|
254
|
+
body). Queries use `input_type="search_query"`, documents `"search_document"`; the provider
|
|
255
|
+
ignores it where the model has no such field. `normalize` is applied locally.
|
|
256
|
+
"""
|
|
257
|
+
|
|
258
|
+
llmq: Any = Field(default=None, exclude=True)
|
|
259
|
+
model_alias: str | None = None
|
|
260
|
+
priority: bool | None = None
|
|
261
|
+
queue_timeout_secs: float | None = None
|
|
262
|
+
|
|
263
|
+
@model_validator(mode="before")
|
|
264
|
+
@classmethod
|
|
265
|
+
def _queue_defaults(cls, values: Any) -> Any:
|
|
266
|
+
return _queue_only_values(values, cls.model_fields, "client")
|
|
267
|
+
|
|
268
|
+
def _queue(self) -> QueueClient:
|
|
269
|
+
return resolve_queue(self.llmq)
|
|
270
|
+
|
|
271
|
+
def _prepare(self, texts: list[str]) -> list[str]:
|
|
272
|
+
"""Same newline handling as the parent, which says newlines hurt embedding quality."""
|
|
273
|
+
return [text.replace(os.linesep, " ") for text in texts]
|
|
274
|
+
|
|
275
|
+
def _finish(self, vectors: list[list[float]]) -> list[list[float]]:
|
|
276
|
+
return [self._normalize_vector(v) for v in vectors] if self.normalize else vectors
|
|
277
|
+
|
|
278
|
+
def _sync(self, texts: list[str], input_type: str) -> list[list[float]]:
|
|
279
|
+
result = self._queue().embed_sync(
|
|
280
|
+
self.model_alias or self.model_id,
|
|
281
|
+
self._prepare(texts),
|
|
282
|
+
priority=self.priority,
|
|
283
|
+
input_type=input_type,
|
|
284
|
+
timeout_secs=self.queue_timeout_secs,
|
|
285
|
+
)
|
|
286
|
+
return self._finish(result.vectors)
|
|
287
|
+
|
|
288
|
+
async def _async(self, texts: list[str], input_type: str) -> list[list[float]]:
|
|
289
|
+
result = await self._queue().embed(
|
|
290
|
+
self.model_alias or self.model_id,
|
|
291
|
+
self._prepare(texts),
|
|
292
|
+
priority=self.priority,
|
|
293
|
+
input_type=input_type,
|
|
294
|
+
timeout_secs=self.queue_timeout_secs,
|
|
295
|
+
)
|
|
296
|
+
return self._finish(result.vectors)
|
|
297
|
+
|
|
298
|
+
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
299
|
+
"""Embed documents on the sync path (`embed_sync`)."""
|
|
300
|
+
return self._sync(texts, "search_document")
|
|
301
|
+
|
|
302
|
+
def embed_query(self, text: str) -> list[float]:
|
|
303
|
+
"""Embed one query on the sync path (`embed_sync`)."""
|
|
304
|
+
return self._sync([text], "search_query")[0]
|
|
305
|
+
|
|
306
|
+
async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
307
|
+
"""Embed documents through the queue as one call (the parent fans out per text)."""
|
|
308
|
+
return await self._async(texts, "search_document")
|
|
309
|
+
|
|
310
|
+
async def aembed_query(self, text: str) -> list[float]:
|
|
311
|
+
"""Embed one query through the queue."""
|
|
312
|
+
return (await self._async([text], "search_query"))[0]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Queued OpenAI-compatible adapters: `ChatOpenAI` / `OpenAIEmbeddings` over `QueuedHTTPTransport`.
|
|
2
|
+
|
|
3
|
+
Both SDK clients get an httpx client whose transport answers from the LLM queue (D88), so every
|
|
4
|
+
LangChain/SDK code path is queued and parsed unchanged. Streaming and SDK retries are forced off:
|
|
5
|
+
the queue's workers own retry and fallback. Chat replies get how the queue served
|
|
6
|
+
them (served model, deployment, attempts, queue wait) in `response_metadata`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import TYPE_CHECKING, Any
|
|
12
|
+
|
|
13
|
+
import httpx
|
|
14
|
+
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
|
15
|
+
from pydantic import Field, model_validator
|
|
16
|
+
|
|
17
|
+
from llm_queue.http_transport import META_KEY, QUEUED_BASE_URL, QueuedHTTPTransport
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from langchain_core.outputs import ChatResult
|
|
21
|
+
|
|
22
|
+
# Placeholder only: the SDK requires a key, but workers hold the real provider credentials.
|
|
23
|
+
_PLACEHOLDER_API_KEY = "llmq-queued"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _queued_client_values(values: Any) -> Any:
|
|
27
|
+
"""Point both SDK clients at a queued transport built from the raw constructor values."""
|
|
28
|
+
if not isinstance(values, dict):
|
|
29
|
+
return values
|
|
30
|
+
transport = QueuedHTTPTransport(
|
|
31
|
+
values.get("llmq"),
|
|
32
|
+
values.get("model_alias"),
|
|
33
|
+
priority=values.get("priority"),
|
|
34
|
+
allow_fallback=values.get("allow_fallback", "all"),
|
|
35
|
+
timeout_secs=values.get("queue_timeout_secs"),
|
|
36
|
+
)
|
|
37
|
+
values["http_async_client"] = httpx.AsyncClient(transport=transport, base_url=QUEUED_BASE_URL)
|
|
38
|
+
values["http_client"] = httpx.Client(transport=transport, base_url=QUEUED_BASE_URL)
|
|
39
|
+
values.pop("openai_api_base", None)
|
|
40
|
+
values["base_url"] = QUEUED_BASE_URL
|
|
41
|
+
values["openai_proxy"] = None # an OPENAI_PROXY env var would conflict with http_client
|
|
42
|
+
if values.get("api_key") is None and values.get("openai_api_key") is None:
|
|
43
|
+
values["api_key"] = _PLACEHOLDER_API_KEY
|
|
44
|
+
values["max_retries"] = 0
|
|
45
|
+
return values
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class QueuedChatOpenAI(ChatOpenAI):
|
|
49
|
+
"""`ChatOpenAI` whose HTTP calls are answered by the LLM queue's `chat` op.
|
|
50
|
+
|
|
51
|
+
`model_alias` is the queue alias; when unset, the request's `model` (this class's `model`)
|
|
52
|
+
is used as the alias. Streaming is always off, so `astream()` yields one full message.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
llmq: Any = Field(default=None, exclude=True)
|
|
56
|
+
model_alias: str | None = None
|
|
57
|
+
allow_fallback: str = "all"
|
|
58
|
+
priority: bool | None = None
|
|
59
|
+
queue_timeout_secs: float | None = None
|
|
60
|
+
|
|
61
|
+
@model_validator(mode="before")
|
|
62
|
+
@classmethod
|
|
63
|
+
def _queue_defaults(cls, values: Any) -> Any:
|
|
64
|
+
values = _queued_client_values(values)
|
|
65
|
+
if isinstance(values, dict):
|
|
66
|
+
values["streaming"] = False
|
|
67
|
+
values["disable_streaming"] = True
|
|
68
|
+
values.setdefault("name", ChatOpenAI.__name__) # trace run name as the parent's
|
|
69
|
+
return values
|
|
70
|
+
|
|
71
|
+
def _create_chat_result(
|
|
72
|
+
self, response: Any, generation_info: dict[str, Any] | None = None
|
|
73
|
+
) -> ChatResult:
|
|
74
|
+
"""Parse like the parent, then add the queue's `llmq_meta` to `response_metadata`."""
|
|
75
|
+
result = super()._create_chat_result(response, generation_info)
|
|
76
|
+
if isinstance(response, dict):
|
|
77
|
+
meta = response.get(META_KEY)
|
|
78
|
+
else:
|
|
79
|
+
meta = (getattr(response, "model_extra", None) or {}).get(META_KEY)
|
|
80
|
+
if isinstance(meta, dict):
|
|
81
|
+
for generation in result.generations:
|
|
82
|
+
generation.message.response_metadata.update(meta)
|
|
83
|
+
return result
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class QueuedOpenAIEmbeddings(OpenAIEmbeddings):
|
|
87
|
+
"""`OpenAIEmbeddings` whose HTTP calls are answered by the LLM queue's `embed` op.
|
|
88
|
+
|
|
89
|
+
`check_embedding_ctx_length` defaults to False because the queue embeds text, not tokens,
|
|
90
|
+
and non-OpenAI servers do not share OpenAI's tokenizer.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
llmq: Any = Field(default=None, exclude=True)
|
|
94
|
+
model_alias: str | None = None
|
|
95
|
+
priority: bool | None = None
|
|
96
|
+
queue_timeout_secs: float | None = None
|
|
97
|
+
check_embedding_ctx_length: bool = False
|
|
98
|
+
|
|
99
|
+
@model_validator(mode="before")
|
|
100
|
+
@classmethod
|
|
101
|
+
def _queue_defaults(cls, values: Any) -> Any:
|
|
102
|
+
return _queued_client_values(values)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Provider helpers shared by the clients: error classification, embedding bodies, host checks.
|
|
2
|
+
|
|
3
|
+
The providers themselves (one attempt on a route) live in `llm_queue.clients`. Importing
|
|
4
|
+
this package never imports aiobotocore or openai.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from llm_queue.providers.classify import (
|
|
10
|
+
classify_bedrock,
|
|
11
|
+
classify_openai,
|
|
12
|
+
error_family,
|
|
13
|
+
is_target_error,
|
|
14
|
+
parse_retry_after,
|
|
15
|
+
)
|
|
16
|
+
from llm_queue.providers.embeddings import (
|
|
17
|
+
COHERE_MAX_BATCH,
|
|
18
|
+
OPENAI_DEFAULT_CHUNK_SIZE,
|
|
19
|
+
TITAN_MAX_BATCH,
|
|
20
|
+
batches,
|
|
21
|
+
cohere_body,
|
|
22
|
+
embedding_body,
|
|
23
|
+
max_batch_size,
|
|
24
|
+
parse_embeddings,
|
|
25
|
+
titan_body,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"COHERE_MAX_BATCH",
|
|
30
|
+
"OPENAI_DEFAULT_CHUNK_SIZE",
|
|
31
|
+
"TITAN_MAX_BATCH",
|
|
32
|
+
"batches",
|
|
33
|
+
"classify_bedrock",
|
|
34
|
+
"classify_openai",
|
|
35
|
+
"cohere_body",
|
|
36
|
+
"embedding_body",
|
|
37
|
+
"error_family",
|
|
38
|
+
"is_target_error",
|
|
39
|
+
"max_batch_size",
|
|
40
|
+
"parse_embeddings",
|
|
41
|
+
"parse_retry_after",
|
|
42
|
+
"titan_body",
|
|
43
|
+
]
|