llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
llm_queue/__init__.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""llm-queue: LLM calls through a queue, balanced across accounts and regions.
|
|
2
|
+
|
|
3
|
+
This is the public front door. `LLMQueue` (loaded on first use, so importing the package has
|
|
4
|
+
no side effects) runs calls through each model's queue under RPM / TPM budgets per deployment
|
|
5
|
+
(model x account x region), with fallback regions and models, health and status. The registry
|
|
6
|
+
(`LLMQ_REGISTRY`) describes accounts, models and settings; the extension points are plain
|
|
7
|
+
functions registered by name.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from importlib import import_module
|
|
13
|
+
from typing import TYPE_CHECKING, Any
|
|
14
|
+
|
|
15
|
+
from llm_queue._version import __version__
|
|
16
|
+
from llm_queue.context import RequestContext, current_context, request_context
|
|
17
|
+
from llm_queue.engine.errors import KINDS, CallFailed
|
|
18
|
+
from llm_queue.engine.hooks import (
|
|
19
|
+
CallContext,
|
|
20
|
+
DeploymentView,
|
|
21
|
+
Reject,
|
|
22
|
+
register_admission,
|
|
23
|
+
register_balancer,
|
|
24
|
+
register_fallback_policy,
|
|
25
|
+
register_probe,
|
|
26
|
+
register_sticky_key,
|
|
27
|
+
)
|
|
28
|
+
from llm_queue.errors import (
|
|
29
|
+
ConfigError,
|
|
30
|
+
LLMQueueError,
|
|
31
|
+
MissingExtra,
|
|
32
|
+
StoreUnavailable,
|
|
33
|
+
UnknownModel,
|
|
34
|
+
)
|
|
35
|
+
from llm_queue.tuning import (
|
|
36
|
+
BedrockOptions,
|
|
37
|
+
EngineTimings,
|
|
38
|
+
RegistryOptions,
|
|
39
|
+
StoreOptions,
|
|
40
|
+
Tuning,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
if TYPE_CHECKING:
|
|
44
|
+
from llm_queue.clients.providers import register_provider
|
|
45
|
+
from llm_queue.core import EmbedResult, LLMQueue
|
|
46
|
+
from llm_queue.engine.envelope import Meta, Reply
|
|
47
|
+
from llm_queue.engine.tokens import register_token_estimator
|
|
48
|
+
from llm_queue.registry import RegistryError, register_account_kind
|
|
49
|
+
|
|
50
|
+
# Loaded on first access: these pull in redis, pydantic models or the SDK layer.
|
|
51
|
+
_LAZY = {
|
|
52
|
+
"LLMQueue": "llm_queue.core",
|
|
53
|
+
"EmbedResult": "llm_queue.core",
|
|
54
|
+
"Reply": "llm_queue.engine.envelope",
|
|
55
|
+
"Meta": "llm_queue.engine.envelope",
|
|
56
|
+
"RegistryError": "llm_queue.registry",
|
|
57
|
+
"register_account_kind": "llm_queue.registry",
|
|
58
|
+
"register_provider": "llm_queue.clients.providers",
|
|
59
|
+
"register_token_estimator": "llm_queue.engine.tokens",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
__all__ = [
|
|
63
|
+
"KINDS",
|
|
64
|
+
"BedrockOptions",
|
|
65
|
+
"CallContext",
|
|
66
|
+
"CallFailed",
|
|
67
|
+
"ConfigError",
|
|
68
|
+
"DeploymentView",
|
|
69
|
+
"EmbedResult",
|
|
70
|
+
"EngineTimings",
|
|
71
|
+
"LLMQueue",
|
|
72
|
+
"LLMQueueError",
|
|
73
|
+
"Meta",
|
|
74
|
+
"MissingExtra",
|
|
75
|
+
"RegistryError",
|
|
76
|
+
"RegistryOptions",
|
|
77
|
+
"Reject",
|
|
78
|
+
"Reply",
|
|
79
|
+
"RequestContext",
|
|
80
|
+
"StoreOptions",
|
|
81
|
+
"StoreUnavailable",
|
|
82
|
+
"Tuning",
|
|
83
|
+
"UnknownModel",
|
|
84
|
+
"__version__",
|
|
85
|
+
"current_context",
|
|
86
|
+
"register_account_kind",
|
|
87
|
+
"register_admission",
|
|
88
|
+
"register_balancer",
|
|
89
|
+
"register_fallback_policy",
|
|
90
|
+
"register_probe",
|
|
91
|
+
"register_provider",
|
|
92
|
+
"register_sticky_key",
|
|
93
|
+
"register_token_estimator",
|
|
94
|
+
"request_context",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def __getattr__(name: str) -> Any:
|
|
99
|
+
"""Import `LLMQueue` and the heavier names on first use."""
|
|
100
|
+
module = _LAZY.get(name)
|
|
101
|
+
if module is None:
|
|
102
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
103
|
+
value = getattr(import_module(module), name)
|
|
104
|
+
globals()[name] = value
|
|
105
|
+
return value
|
llm_queue/_logging.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Structured loggers that always go through stdlib `logging` (no global structlog config).
|
|
2
|
+
|
|
3
|
+
Unconfigured structlog prints to stdout and ignores the app's levels and handlers, so every
|
|
4
|
+
module logs via `get_logger(__name__)`: structlog's key-value API on top of a stdlib logger.
|
|
5
|
+
Apps keep full control (levels, handlers, loguru's InterceptHandler, structlog's
|
|
6
|
+
ProcessorFormatter); the fields ride on `record.llmq` as a dict.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from collections.abc import Mapping, MutableMapping
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import structlog
|
|
16
|
+
|
|
17
|
+
_PASSTHROUGH = ("exc_info", "stack_info")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _to_stdlib(
|
|
21
|
+
_logger: Any, _method: str, event_dict: MutableMapping[str, Any]
|
|
22
|
+
) -> tuple[tuple[str], dict[str, Any]]:
|
|
23
|
+
# Message is readable on its own ("event k=v ..."); the structured copy lives on one
|
|
24
|
+
# `llmq` attribute so field names can never clash with LogRecord's reserved attributes.
|
|
25
|
+
event = str(event_dict.pop("event", ""))
|
|
26
|
+
kwargs: dict[str, Any] = {k: event_dict.pop(k) for k in _PASSTHROUGH if k in event_dict}
|
|
27
|
+
fields: Mapping[str, Any] = dict(event_dict)
|
|
28
|
+
message = " ".join([event, *(f"{k}={v}" for k, v in fields.items())])
|
|
29
|
+
kwargs["extra"] = {"llmq": {"event": event, **fields}}
|
|
30
|
+
kwargs["stacklevel"] = 4 # point %(funcName)s/%(lineno)d at the library call site
|
|
31
|
+
return (message,), kwargs
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_logger(name: str) -> structlog.stdlib.BoundLogger:
|
|
35
|
+
"""Key-value logger bound to `logging.getLogger(name)`; used by every library module."""
|
|
36
|
+
return structlog.wrap_logger( # type: ignore[no-any-return]
|
|
37
|
+
logging.getLogger(name),
|
|
38
|
+
processors=[_to_stdlib],
|
|
39
|
+
wrapper_class=structlog.stdlib.BoundLogger,
|
|
40
|
+
cache_logger_on_first_use=True,
|
|
41
|
+
)
|
llm_queue/_version.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Clients and providers per account and region (plan step A3).
|
|
2
|
+
|
|
3
|
+
`ClientPool` holds one SDK client per (account, region); `Providers` runs one attempt of an
|
|
4
|
+
operation on a `Route` with the attempt timers. Chat always uses the streaming API underneath.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from llm_queue.clients.pool import (
|
|
10
|
+
ClientPool,
|
|
11
|
+
CredentialsUnavailable,
|
|
12
|
+
Route,
|
|
13
|
+
route_for,
|
|
14
|
+
)
|
|
15
|
+
from llm_queue.clients.providers import (
|
|
16
|
+
BedrockProvider,
|
|
17
|
+
OpenAIProvider,
|
|
18
|
+
Provider,
|
|
19
|
+
Providers,
|
|
20
|
+
Result,
|
|
21
|
+
Usage,
|
|
22
|
+
decode_invoke_body,
|
|
23
|
+
register_provider,
|
|
24
|
+
)
|
|
25
|
+
from llm_queue.clients.timing import AttemptTimeout, Timing
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"AttemptTimeout",
|
|
29
|
+
"BedrockProvider",
|
|
30
|
+
"ClientPool",
|
|
31
|
+
"CredentialsUnavailable",
|
|
32
|
+
"OpenAIProvider",
|
|
33
|
+
"Provider",
|
|
34
|
+
"Providers",
|
|
35
|
+
"Result",
|
|
36
|
+
"Route",
|
|
37
|
+
"Timing",
|
|
38
|
+
"Usage",
|
|
39
|
+
"decode_invoke_body",
|
|
40
|
+
"register_provider",
|
|
41
|
+
"route_for",
|
|
42
|
+
]
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""One SDK client per (account, region), built on first use with that account's credentials.
|
|
2
|
+
|
|
3
|
+
A `Route` says where one attempt goes: the account, the region the client calls and the model
|
|
4
|
+
id (an inference profile's id for a profile). `ClientPool` hands out Bedrock and
|
|
5
|
+
OpenAI-compatible clients for routes and closes them all at `aclose()`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import contextlib
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
15
|
+
from contextlib import AsyncExitStack
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import TYPE_CHECKING, Any
|
|
18
|
+
|
|
19
|
+
from llm_queue.errors import ConfigError, LLMQueueError, MissingExtra
|
|
20
|
+
from llm_queue.tuning import BedrockOptions
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from aiobotocore.session import AioSession
|
|
24
|
+
from openai import AsyncOpenAI
|
|
25
|
+
from types_aiobotocore_bedrock_runtime import BedrockRuntimeClient
|
|
26
|
+
|
|
27
|
+
from llm_queue.registry import Account, Deployment, ResolvedRegistry
|
|
28
|
+
|
|
29
|
+
# Unverified starting values. The SDK read timeout only guards a dead socket: the library's own
|
|
30
|
+
# first-response and idle timers decide when an attempt is given up.
|
|
31
|
+
|
|
32
|
+
# Extra `create_client` arguments for an AWS account (None = the pod's own credential chain).
|
|
33
|
+
CredentialSource = Callable[["Account"], Mapping[str, Any] | None]
|
|
34
|
+
# The value of a secret by name (`api_key_ref`, `external_id_ref`); default: the environment.
|
|
35
|
+
SecretSource = Callable[[str], str | None]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True, slots=True)
|
|
39
|
+
class Route:
|
|
40
|
+
"""Where one attempt goes: account, region to call, model id, and the deployment's name."""
|
|
41
|
+
|
|
42
|
+
account: Account
|
|
43
|
+
region: str
|
|
44
|
+
model_id: str
|
|
45
|
+
deployment: str
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def route_for(view: ResolvedRegistry, deployment: Deployment) -> Route:
|
|
49
|
+
"""The route of a deployment: its account and region, its profile id or the model id."""
|
|
50
|
+
model = view.registry.models[deployment.model]
|
|
51
|
+
return Route(
|
|
52
|
+
account=view.registry.accounts[deployment.account],
|
|
53
|
+
region=deployment.region,
|
|
54
|
+
model_id=deployment.profile_id or model.model_id,
|
|
55
|
+
deployment=deployment.key,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _pod_chain(account: Account) -> Mapping[str, Any] | None:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# AWS RoleSessionName allows only [\w+=,.@-]; a worker id like "pod:1a2b" would fail AssumeRole.
|
|
64
|
+
_SESSION_UNSAFE = re.compile(r"[^\w+=,.@-]")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class CredentialsUnavailable(LLMQueueError):
|
|
68
|
+
"""An account's credentials could not be obtained (STS refused or unreachable)."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class ClientPool:
|
|
72
|
+
"""Clients per (account, region) for Bedrock and per account for OpenAI-compatible servers.
|
|
73
|
+
|
|
74
|
+
`credentials` gives an AWS account's client arguments (tests pass static keys per account);
|
|
75
|
+
`secrets` reads `api_key_ref` values. Clients are created once and closed by `aclose()`.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
*,
|
|
81
|
+
bedrock_endpoint_url: str | None = None,
|
|
82
|
+
credentials: CredentialSource = _pod_chain,
|
|
83
|
+
secrets: SecretSource = os.environ.get,
|
|
84
|
+
session: AioSession | None = None,
|
|
85
|
+
options: BedrockOptions | None = None,
|
|
86
|
+
sts_endpoint_url: str | None = None,
|
|
87
|
+
session_name: str = "llmq",
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Keep settings only; no SDK import and no client until the first call.
|
|
90
|
+
|
|
91
|
+
`session_name` names assumed-role sessions (`llmq-<env>-<pod>` in CloudTrail).
|
|
92
|
+
"""
|
|
93
|
+
self._endpoint = bedrock_endpoint_url
|
|
94
|
+
self._credentials = credentials
|
|
95
|
+
self._secrets = secrets
|
|
96
|
+
self._session = session
|
|
97
|
+
self._options = options or BedrockOptions() # timeouts, pool size: `Tuning.bedrock`
|
|
98
|
+
self._read = self._options.read_timeout_secs
|
|
99
|
+
self._connect = self._options.connect_timeout_secs
|
|
100
|
+
self._pool_size = self._options.max_pool_connections
|
|
101
|
+
self._bedrock: dict[tuple[str, str], BedrockRuntimeClient] = {}
|
|
102
|
+
self._sts_endpoint = sts_endpoint_url
|
|
103
|
+
self._session_name = session_name
|
|
104
|
+
self._roles: dict[str, AioSession] = {}
|
|
105
|
+
self._openai: dict[str, AsyncOpenAI] = {}
|
|
106
|
+
self._stack = AsyncExitStack()
|
|
107
|
+
self._closers: dict[tuple[str, str], Callable[[], Awaitable[Any]]] = {}
|
|
108
|
+
self._retiring: dict[asyncio.Task[None], list[Callable[[], Awaitable[Any]]]] = {}
|
|
109
|
+
self._lock = asyncio.Lock()
|
|
110
|
+
|
|
111
|
+
async def bedrock(self, route: Route) -> BedrockRuntimeClient:
|
|
112
|
+
"""The Bedrock runtime client of the route's account and region."""
|
|
113
|
+
key = (route.account.name, route.region)
|
|
114
|
+
if (client := self._bedrock.get(key)) is not None:
|
|
115
|
+
return client
|
|
116
|
+
async with self._lock:
|
|
117
|
+
if (client := self._bedrock.get(key)) is not None:
|
|
118
|
+
return client
|
|
119
|
+
from aiobotocore.config import AioConfig
|
|
120
|
+
from aiobotocore.session import get_session
|
|
121
|
+
|
|
122
|
+
if self._session is None:
|
|
123
|
+
self._session = get_session()
|
|
124
|
+
session = self._session
|
|
125
|
+
extra = dict(self._credentials(route.account) or {})
|
|
126
|
+
if route.account.role_arn:
|
|
127
|
+
session = await self._role_session(route)
|
|
128
|
+
extra = {}
|
|
129
|
+
config = AioConfig(
|
|
130
|
+
read_timeout=self._read,
|
|
131
|
+
connect_timeout=self._connect,
|
|
132
|
+
max_pool_connections=self._pool_size,
|
|
133
|
+
# One attempt per call: throttles reach the library, not botocore's retries.
|
|
134
|
+
retries={"total_max_attempts": 1, "mode": "standard"},
|
|
135
|
+
# botocore re-walks every message block before sending: 8.5 ms vs 3.5 ms CPU
|
|
136
|
+
# per attempt at 218 KB (measured). Bedrock validates anyway; a bad request
|
|
137
|
+
# comes back as ValidationException, which is classified fatal.
|
|
138
|
+
parameter_validation=False,
|
|
139
|
+
)
|
|
140
|
+
context = session.create_client(
|
|
141
|
+
"bedrock-runtime",
|
|
142
|
+
region_name=route.region,
|
|
143
|
+
endpoint_url=self._endpoint,
|
|
144
|
+
config=config,
|
|
145
|
+
**extra,
|
|
146
|
+
)
|
|
147
|
+
client = await context.__aenter__()
|
|
148
|
+
self._closers[key] = lambda: context.__aexit__(None, None, None)
|
|
149
|
+
self._bedrock[key] = client
|
|
150
|
+
return client
|
|
151
|
+
|
|
152
|
+
async def _role_session(self, route: Route) -> AioSession:
|
|
153
|
+
"""A session whose credentials come from STS AssumeRole and refresh before expiry."""
|
|
154
|
+
account = route.account
|
|
155
|
+
if account.name in self._roles:
|
|
156
|
+
return self._roles[account.name]
|
|
157
|
+
from aiobotocore.credentials import AioRefreshableCredentials
|
|
158
|
+
from aiobotocore.session import AioSession, get_session
|
|
159
|
+
|
|
160
|
+
base = self._session or get_session()
|
|
161
|
+
base_kwargs = dict(self._credentials(account) or {})
|
|
162
|
+
external_id = self._secrets(account.external_id_ref) if account.external_id_ref else None
|
|
163
|
+
if account.external_id_ref and not external_id:
|
|
164
|
+
raise CredentialsUnavailable(
|
|
165
|
+
f"account {account.name}: secret {account.external_id_ref} is not set"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
async def assume() -> dict[str, Any]:
|
|
169
|
+
request: dict[str, Any] = {
|
|
170
|
+
"RoleArn": account.role_arn,
|
|
171
|
+
"RoleSessionName": _SESSION_UNSAFE.sub("-", self._session_name)[:64],
|
|
172
|
+
"DurationSeconds": self._options.role_session_secs,
|
|
173
|
+
}
|
|
174
|
+
if external_id:
|
|
175
|
+
request["ExternalId"] = external_id
|
|
176
|
+
try:
|
|
177
|
+
async with base.create_client(
|
|
178
|
+
"sts", region_name=route.region, endpoint_url=self._sts_endpoint, **base_kwargs
|
|
179
|
+
) as sts:
|
|
180
|
+
reply = await sts.assume_role(**request)
|
|
181
|
+
except Exception as exc:
|
|
182
|
+
# The code only: STS text can carry role ARNs and account ids, and this message
|
|
183
|
+
# reaches replies, status and logs (the cause keeps the detail for debugging).
|
|
184
|
+
code = (
|
|
185
|
+
getattr(exc, "response", {}).get("Error", {}).get("Code") or type(exc).__name__
|
|
186
|
+
)
|
|
187
|
+
raise CredentialsUnavailable(
|
|
188
|
+
f"account {account.name}: AssumeRole failed ({code})"
|
|
189
|
+
) from exc
|
|
190
|
+
creds = reply["Credentials"]
|
|
191
|
+
return {
|
|
192
|
+
"access_key": creds["AccessKeyId"],
|
|
193
|
+
"secret_key": creds["SecretAccessKey"],
|
|
194
|
+
"token": creds["SessionToken"],
|
|
195
|
+
"expiry_time": creds["Expiration"].isoformat(),
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
credentials = AioRefreshableCredentials.create_from_metadata(
|
|
199
|
+
metadata=await assume(), refresh_using=assume, method="sts-assume-role"
|
|
200
|
+
)
|
|
201
|
+
session = AioSession()
|
|
202
|
+
# aiobotocore has no public setter for refreshable credentials.
|
|
203
|
+
session._credentials = credentials # type: ignore[attr-defined]
|
|
204
|
+
self._roles[account.name] = session
|
|
205
|
+
return session
|
|
206
|
+
|
|
207
|
+
def openai(self, route: Route) -> AsyncOpenAI:
|
|
208
|
+
"""The OpenAI-compatible client of the route's account (its `base_url` and key)."""
|
|
209
|
+
account = route.account
|
|
210
|
+
if (client := self._openai.get(account.name)) is not None:
|
|
211
|
+
return client
|
|
212
|
+
try:
|
|
213
|
+
from openai import AsyncOpenAI, Timeout
|
|
214
|
+
except ImportError as exc:
|
|
215
|
+
raise MissingExtra("an openai account", "openai") from exc
|
|
216
|
+
key = self._secrets(account.api_key_ref) if account.api_key_ref else None
|
|
217
|
+
if account.api_key_ref and not key:
|
|
218
|
+
raise ConfigError(f"account {account.name}: secret {account.api_key_ref} is not set")
|
|
219
|
+
client = AsyncOpenAI(
|
|
220
|
+
api_key=key or "unused",
|
|
221
|
+
base_url=account.base_url,
|
|
222
|
+
max_retries=0,
|
|
223
|
+
# openai's own Timeout: openai 3.x ships its own HTTP types (not httpx's).
|
|
224
|
+
timeout=Timeout(self._read, connect=self._connect),
|
|
225
|
+
)
|
|
226
|
+
self._openai[account.name] = client
|
|
227
|
+
return client
|
|
228
|
+
|
|
229
|
+
def forget(self, route: Route) -> None:
|
|
230
|
+
"""Drop an account's cached clients so the next call reads its credentials again.
|
|
231
|
+
|
|
232
|
+
The dropped clients are closed after a grace period (calls already using them finish
|
|
233
|
+
first); before, each auth failure leaked a client and its connection pool.
|
|
234
|
+
"""
|
|
235
|
+
closers = []
|
|
236
|
+
for key in [k for k in self._bedrock if k[0] == route.account.name]:
|
|
237
|
+
del self._bedrock[key]
|
|
238
|
+
if (closer := self._closers.pop(key, None)) is not None:
|
|
239
|
+
closers.append(closer)
|
|
240
|
+
self._roles.pop(route.account.name, None)
|
|
241
|
+
if (openai_client := self._openai.pop(route.account.name, None)) is not None:
|
|
242
|
+
closers.append(openai_client.close)
|
|
243
|
+
if closers:
|
|
244
|
+
task = asyncio.get_running_loop().create_task(self._retire(closers))
|
|
245
|
+
self._retiring[task] = closers
|
|
246
|
+
task.add_done_callback(lambda done: self._retiring.pop(done, None))
|
|
247
|
+
|
|
248
|
+
async def _retire(self, closers: list[Callable[[], Awaitable[Any]]]) -> None:
|
|
249
|
+
await asyncio.sleep(self._read + self._connect) # the longest a running call can need
|
|
250
|
+
for close in closers:
|
|
251
|
+
with contextlib.suppress(Exception):
|
|
252
|
+
await close()
|
|
253
|
+
|
|
254
|
+
async def aclose(self) -> None:
|
|
255
|
+
"""Close every client; safe to call more than once."""
|
|
256
|
+
retired = [close for closers in self._retiring.values() for close in closers]
|
|
257
|
+
for task in list(self._retiring): # close retired clients now, not after the grace
|
|
258
|
+
task.cancel()
|
|
259
|
+
self._retiring.clear()
|
|
260
|
+
async with self._lock:
|
|
261
|
+
self._bedrock.clear()
|
|
262
|
+
closers, self._closers = [*self._closers.values(), *retired], {}
|
|
263
|
+
openai_clients = list(self._openai.values())
|
|
264
|
+
self._openai.clear()
|
|
265
|
+
stack, self._stack = self._stack, AsyncExitStack()
|
|
266
|
+
await stack.aclose()
|
|
267
|
+
for close in closers:
|
|
268
|
+
with contextlib.suppress(Exception):
|
|
269
|
+
await close()
|
|
270
|
+
for client in openai_clients:
|
|
271
|
+
await client.close()
|