llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
llm_queue/__init__.py ADDED
@@ -0,0 +1,105 @@
1
+ """llm-queue: LLM calls through a queue, balanced across accounts and regions.
2
+
3
+ This is the public front door. `LLMQueue` (loaded on first use, so importing the package has
4
+ no side effects) runs calls through each model's queue under RPM / TPM budgets per deployment
5
+ (model x account x region), with fallback regions and models, health and status. The registry
6
+ (`LLMQ_REGISTRY`) describes accounts, models and settings; the extension points are plain
7
+ functions registered by name.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from importlib import import_module
13
+ from typing import TYPE_CHECKING, Any
14
+
15
+ from llm_queue._version import __version__
16
+ from llm_queue.context import RequestContext, current_context, request_context
17
+ from llm_queue.engine.errors import KINDS, CallFailed
18
+ from llm_queue.engine.hooks import (
19
+ CallContext,
20
+ DeploymentView,
21
+ Reject,
22
+ register_admission,
23
+ register_balancer,
24
+ register_fallback_policy,
25
+ register_probe,
26
+ register_sticky_key,
27
+ )
28
+ from llm_queue.errors import (
29
+ ConfigError,
30
+ LLMQueueError,
31
+ MissingExtra,
32
+ StoreUnavailable,
33
+ UnknownModel,
34
+ )
35
+ from llm_queue.tuning import (
36
+ BedrockOptions,
37
+ EngineTimings,
38
+ RegistryOptions,
39
+ StoreOptions,
40
+ Tuning,
41
+ )
42
+
43
+ if TYPE_CHECKING:
44
+ from llm_queue.clients.providers import register_provider
45
+ from llm_queue.core import EmbedResult, LLMQueue
46
+ from llm_queue.engine.envelope import Meta, Reply
47
+ from llm_queue.engine.tokens import register_token_estimator
48
+ from llm_queue.registry import RegistryError, register_account_kind
49
+
50
+ # Loaded on first access: these pull in redis, pydantic models or the SDK layer.
51
+ _LAZY = {
52
+ "LLMQueue": "llm_queue.core",
53
+ "EmbedResult": "llm_queue.core",
54
+ "Reply": "llm_queue.engine.envelope",
55
+ "Meta": "llm_queue.engine.envelope",
56
+ "RegistryError": "llm_queue.registry",
57
+ "register_account_kind": "llm_queue.registry",
58
+ "register_provider": "llm_queue.clients.providers",
59
+ "register_token_estimator": "llm_queue.engine.tokens",
60
+ }
61
+
62
+ __all__ = [
63
+ "KINDS",
64
+ "BedrockOptions",
65
+ "CallContext",
66
+ "CallFailed",
67
+ "ConfigError",
68
+ "DeploymentView",
69
+ "EmbedResult",
70
+ "EngineTimings",
71
+ "LLMQueue",
72
+ "LLMQueueError",
73
+ "Meta",
74
+ "MissingExtra",
75
+ "RegistryError",
76
+ "RegistryOptions",
77
+ "Reject",
78
+ "Reply",
79
+ "RequestContext",
80
+ "StoreOptions",
81
+ "StoreUnavailable",
82
+ "Tuning",
83
+ "UnknownModel",
84
+ "__version__",
85
+ "current_context",
86
+ "register_account_kind",
87
+ "register_admission",
88
+ "register_balancer",
89
+ "register_fallback_policy",
90
+ "register_probe",
91
+ "register_provider",
92
+ "register_sticky_key",
93
+ "register_token_estimator",
94
+ "request_context",
95
+ ]
96
+
97
+
98
+ def __getattr__(name: str) -> Any:
99
+ """Import `LLMQueue` and the heavier names on first use."""
100
+ module = _LAZY.get(name)
101
+ if module is None:
102
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
103
+ value = getattr(import_module(module), name)
104
+ globals()[name] = value
105
+ return value
llm_queue/_logging.py ADDED
@@ -0,0 +1,41 @@
1
+ """Structured loggers that always go through stdlib `logging` (no global structlog config).
2
+
3
+ Unconfigured structlog prints to stdout and ignores the app's levels and handlers, so every
4
+ module logs via `get_logger(__name__)`: structlog's key-value API on top of a stdlib logger.
5
+ Apps keep full control (levels, handlers, loguru's InterceptHandler, structlog's
6
+ ProcessorFormatter); the fields ride on `record.llmq` as a dict.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import logging
12
+ from collections.abc import Mapping, MutableMapping
13
+ from typing import Any
14
+
15
+ import structlog
16
+
17
+ _PASSTHROUGH = ("exc_info", "stack_info")
18
+
19
+
20
+ def _to_stdlib(
21
+ _logger: Any, _method: str, event_dict: MutableMapping[str, Any]
22
+ ) -> tuple[tuple[str], dict[str, Any]]:
23
+ # Message is readable on its own ("event k=v ..."); the structured copy lives on one
24
+ # `llmq` attribute so field names can never clash with LogRecord's reserved attributes.
25
+ event = str(event_dict.pop("event", ""))
26
+ kwargs: dict[str, Any] = {k: event_dict.pop(k) for k in _PASSTHROUGH if k in event_dict}
27
+ fields: Mapping[str, Any] = dict(event_dict)
28
+ message = " ".join([event, *(f"{k}={v}" for k, v in fields.items())])
29
+ kwargs["extra"] = {"llmq": {"event": event, **fields}}
30
+ kwargs["stacklevel"] = 4 # point %(funcName)s/%(lineno)d at the library call site
31
+ return (message,), kwargs
32
+
33
+
34
+ def get_logger(name: str) -> structlog.stdlib.BoundLogger:
35
+ """Key-value logger bound to `logging.getLogger(name)`; used by every library module."""
36
+ return structlog.wrap_logger( # type: ignore[no-any-return]
37
+ logging.getLogger(name),
38
+ processors=[_to_stdlib],
39
+ wrapper_class=structlog.stdlib.BoundLogger,
40
+ cache_logger_on_first_use=True,
41
+ )
llm_queue/_version.py ADDED
@@ -0,0 +1,5 @@
1
+ """Package version, read by hatchling at build time and re-exported as `__version__`."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __version__ = "0.1.0b1"
@@ -0,0 +1,42 @@
1
+ """Clients and providers per account and region (plan step A3).
2
+
3
+ `ClientPool` holds one SDK client per (account, region); `Providers` runs one attempt of an
4
+ operation on a `Route` with the attempt timers. Chat always uses the streaming API underneath.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from llm_queue.clients.pool import (
10
+ ClientPool,
11
+ CredentialsUnavailable,
12
+ Route,
13
+ route_for,
14
+ )
15
+ from llm_queue.clients.providers import (
16
+ BedrockProvider,
17
+ OpenAIProvider,
18
+ Provider,
19
+ Providers,
20
+ Result,
21
+ Usage,
22
+ decode_invoke_body,
23
+ register_provider,
24
+ )
25
+ from llm_queue.clients.timing import AttemptTimeout, Timing
26
+
27
+ __all__ = [
28
+ "AttemptTimeout",
29
+ "BedrockProvider",
30
+ "ClientPool",
31
+ "CredentialsUnavailable",
32
+ "OpenAIProvider",
33
+ "Provider",
34
+ "Providers",
35
+ "Result",
36
+ "Route",
37
+ "Timing",
38
+ "Usage",
39
+ "decode_invoke_body",
40
+ "register_provider",
41
+ "route_for",
42
+ ]
@@ -0,0 +1,271 @@
1
+ """One SDK client per (account, region), built on first use with that account's credentials.
2
+
3
+ A `Route` says where one attempt goes: the account, the region the client calls and the model
4
+ id (an inference profile's id for a profile). `ClientPool` hands out Bedrock and
5
+ OpenAI-compatible clients for routes and closes them all at `aclose()`.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ import contextlib
12
+ import os
13
+ import re
14
+ from collections.abc import Awaitable, Callable, Mapping
15
+ from contextlib import AsyncExitStack
16
+ from dataclasses import dataclass
17
+ from typing import TYPE_CHECKING, Any
18
+
19
+ from llm_queue.errors import ConfigError, LLMQueueError, MissingExtra
20
+ from llm_queue.tuning import BedrockOptions
21
+
22
+ if TYPE_CHECKING:
23
+ from aiobotocore.session import AioSession
24
+ from openai import AsyncOpenAI
25
+ from types_aiobotocore_bedrock_runtime import BedrockRuntimeClient
26
+
27
+ from llm_queue.registry import Account, Deployment, ResolvedRegistry
28
+
29
+ # Unverified starting values. The SDK read timeout only guards a dead socket: the library's own
30
+ # first-response and idle timers decide when an attempt is given up.
31
+
32
+ # Extra `create_client` arguments for an AWS account (None = the pod's own credential chain).
33
+ CredentialSource = Callable[["Account"], Mapping[str, Any] | None]
34
+ # The value of a secret by name (`api_key_ref`, `external_id_ref`); default: the environment.
35
+ SecretSource = Callable[[str], str | None]
36
+
37
+
38
+ @dataclass(frozen=True, slots=True)
39
+ class Route:
40
+ """Where one attempt goes: account, region to call, model id, and the deployment's name."""
41
+
42
+ account: Account
43
+ region: str
44
+ model_id: str
45
+ deployment: str
46
+
47
+
48
+ def route_for(view: ResolvedRegistry, deployment: Deployment) -> Route:
49
+ """The route of a deployment: its account and region, its profile id or the model id."""
50
+ model = view.registry.models[deployment.model]
51
+ return Route(
52
+ account=view.registry.accounts[deployment.account],
53
+ region=deployment.region,
54
+ model_id=deployment.profile_id or model.model_id,
55
+ deployment=deployment.key,
56
+ )
57
+
58
+
59
+ def _pod_chain(account: Account) -> Mapping[str, Any] | None:
60
+ return None
61
+
62
+
63
+ # AWS RoleSessionName allows only [\w+=,.@-]; a worker id like "pod:1a2b" would fail AssumeRole.
64
+ _SESSION_UNSAFE = re.compile(r"[^\w+=,.@-]")
65
+
66
+
67
+ class CredentialsUnavailable(LLMQueueError):
68
+ """An account's credentials could not be obtained (STS refused or unreachable)."""
69
+
70
+
71
+ class ClientPool:
72
+ """Clients per (account, region) for Bedrock and per account for OpenAI-compatible servers.
73
+
74
+ `credentials` gives an AWS account's client arguments (tests pass static keys per account);
75
+ `secrets` reads `api_key_ref` values. Clients are created once and closed by `aclose()`.
76
+ """
77
+
78
+ def __init__(
79
+ self,
80
+ *,
81
+ bedrock_endpoint_url: str | None = None,
82
+ credentials: CredentialSource = _pod_chain,
83
+ secrets: SecretSource = os.environ.get,
84
+ session: AioSession | None = None,
85
+ options: BedrockOptions | None = None,
86
+ sts_endpoint_url: str | None = None,
87
+ session_name: str = "llmq",
88
+ ) -> None:
89
+ """Keep settings only; no SDK import and no client until the first call.
90
+
91
+ `session_name` names assumed-role sessions (`llmq-<env>-<pod>` in CloudTrail).
92
+ """
93
+ self._endpoint = bedrock_endpoint_url
94
+ self._credentials = credentials
95
+ self._secrets = secrets
96
+ self._session = session
97
+ self._options = options or BedrockOptions() # timeouts, pool size: `Tuning.bedrock`
98
+ self._read = self._options.read_timeout_secs
99
+ self._connect = self._options.connect_timeout_secs
100
+ self._pool_size = self._options.max_pool_connections
101
+ self._bedrock: dict[tuple[str, str], BedrockRuntimeClient] = {}
102
+ self._sts_endpoint = sts_endpoint_url
103
+ self._session_name = session_name
104
+ self._roles: dict[str, AioSession] = {}
105
+ self._openai: dict[str, AsyncOpenAI] = {}
106
+ self._stack = AsyncExitStack()
107
+ self._closers: dict[tuple[str, str], Callable[[], Awaitable[Any]]] = {}
108
+ self._retiring: dict[asyncio.Task[None], list[Callable[[], Awaitable[Any]]]] = {}
109
+ self._lock = asyncio.Lock()
110
+
111
+ async def bedrock(self, route: Route) -> BedrockRuntimeClient:
112
+ """The Bedrock runtime client of the route's account and region."""
113
+ key = (route.account.name, route.region)
114
+ if (client := self._bedrock.get(key)) is not None:
115
+ return client
116
+ async with self._lock:
117
+ if (client := self._bedrock.get(key)) is not None:
118
+ return client
119
+ from aiobotocore.config import AioConfig
120
+ from aiobotocore.session import get_session
121
+
122
+ if self._session is None:
123
+ self._session = get_session()
124
+ session = self._session
125
+ extra = dict(self._credentials(route.account) or {})
126
+ if route.account.role_arn:
127
+ session = await self._role_session(route)
128
+ extra = {}
129
+ config = AioConfig(
130
+ read_timeout=self._read,
131
+ connect_timeout=self._connect,
132
+ max_pool_connections=self._pool_size,
133
+ # One attempt per call: throttles reach the library, not botocore's retries.
134
+ retries={"total_max_attempts": 1, "mode": "standard"},
135
+ # botocore re-walks every message block before sending: 8.5 ms vs 3.5 ms CPU
136
+ # per attempt at 218 KB (measured). Bedrock validates anyway; a bad request
137
+ # comes back as ValidationException, which is classified fatal.
138
+ parameter_validation=False,
139
+ )
140
+ context = session.create_client(
141
+ "bedrock-runtime",
142
+ region_name=route.region,
143
+ endpoint_url=self._endpoint,
144
+ config=config,
145
+ **extra,
146
+ )
147
+ client = await context.__aenter__()
148
+ self._closers[key] = lambda: context.__aexit__(None, None, None)
149
+ self._bedrock[key] = client
150
+ return client
151
+
152
+ async def _role_session(self, route: Route) -> AioSession:
153
+ """A session whose credentials come from STS AssumeRole and refresh before expiry."""
154
+ account = route.account
155
+ if account.name in self._roles:
156
+ return self._roles[account.name]
157
+ from aiobotocore.credentials import AioRefreshableCredentials
158
+ from aiobotocore.session import AioSession, get_session
159
+
160
+ base = self._session or get_session()
161
+ base_kwargs = dict(self._credentials(account) or {})
162
+ external_id = self._secrets(account.external_id_ref) if account.external_id_ref else None
163
+ if account.external_id_ref and not external_id:
164
+ raise CredentialsUnavailable(
165
+ f"account {account.name}: secret {account.external_id_ref} is not set"
166
+ )
167
+
168
+ async def assume() -> dict[str, Any]:
169
+ request: dict[str, Any] = {
170
+ "RoleArn": account.role_arn,
171
+ "RoleSessionName": _SESSION_UNSAFE.sub("-", self._session_name)[:64],
172
+ "DurationSeconds": self._options.role_session_secs,
173
+ }
174
+ if external_id:
175
+ request["ExternalId"] = external_id
176
+ try:
177
+ async with base.create_client(
178
+ "sts", region_name=route.region, endpoint_url=self._sts_endpoint, **base_kwargs
179
+ ) as sts:
180
+ reply = await sts.assume_role(**request)
181
+ except Exception as exc:
182
+ # The code only: STS text can carry role ARNs and account ids, and this message
183
+ # reaches replies, status and logs (the cause keeps the detail for debugging).
184
+ code = (
185
+ getattr(exc, "response", {}).get("Error", {}).get("Code") or type(exc).__name__
186
+ )
187
+ raise CredentialsUnavailable(
188
+ f"account {account.name}: AssumeRole failed ({code})"
189
+ ) from exc
190
+ creds = reply["Credentials"]
191
+ return {
192
+ "access_key": creds["AccessKeyId"],
193
+ "secret_key": creds["SecretAccessKey"],
194
+ "token": creds["SessionToken"],
195
+ "expiry_time": creds["Expiration"].isoformat(),
196
+ }
197
+
198
+ credentials = AioRefreshableCredentials.create_from_metadata(
199
+ metadata=await assume(), refresh_using=assume, method="sts-assume-role"
200
+ )
201
+ session = AioSession()
202
+ # aiobotocore has no public setter for refreshable credentials.
203
+ session._credentials = credentials # type: ignore[attr-defined]
204
+ self._roles[account.name] = session
205
+ return session
206
+
207
+ def openai(self, route: Route) -> AsyncOpenAI:
208
+ """The OpenAI-compatible client of the route's account (its `base_url` and key)."""
209
+ account = route.account
210
+ if (client := self._openai.get(account.name)) is not None:
211
+ return client
212
+ try:
213
+ from openai import AsyncOpenAI, Timeout
214
+ except ImportError as exc:
215
+ raise MissingExtra("an openai account", "openai") from exc
216
+ key = self._secrets(account.api_key_ref) if account.api_key_ref else None
217
+ if account.api_key_ref and not key:
218
+ raise ConfigError(f"account {account.name}: secret {account.api_key_ref} is not set")
219
+ client = AsyncOpenAI(
220
+ api_key=key or "unused",
221
+ base_url=account.base_url,
222
+ max_retries=0,
223
+ # openai's own Timeout: openai 3.x ships its own HTTP types (not httpx's).
224
+ timeout=Timeout(self._read, connect=self._connect),
225
+ )
226
+ self._openai[account.name] = client
227
+ return client
228
+
229
+ def forget(self, route: Route) -> None:
230
+ """Drop an account's cached clients so the next call reads its credentials again.
231
+
232
+ The dropped clients are closed after a grace period (calls already using them finish
233
+ first); before, each auth failure leaked a client and its connection pool.
234
+ """
235
+ closers = []
236
+ for key in [k for k in self._bedrock if k[0] == route.account.name]:
237
+ del self._bedrock[key]
238
+ if (closer := self._closers.pop(key, None)) is not None:
239
+ closers.append(closer)
240
+ self._roles.pop(route.account.name, None)
241
+ if (openai_client := self._openai.pop(route.account.name, None)) is not None:
242
+ closers.append(openai_client.close)
243
+ if closers:
244
+ task = asyncio.get_running_loop().create_task(self._retire(closers))
245
+ self._retiring[task] = closers
246
+ task.add_done_callback(lambda done: self._retiring.pop(done, None))
247
+
248
+ async def _retire(self, closers: list[Callable[[], Awaitable[Any]]]) -> None:
249
+ await asyncio.sleep(self._read + self._connect) # the longest a running call can need
250
+ for close in closers:
251
+ with contextlib.suppress(Exception):
252
+ await close()
253
+
254
+ async def aclose(self) -> None:
255
+ """Close every client; safe to call more than once."""
256
+ retired = [close for closers in self._retiring.values() for close in closers]
257
+ for task in list(self._retiring): # close retired clients now, not after the grace
258
+ task.cancel()
259
+ self._retiring.clear()
260
+ async with self._lock:
261
+ self._bedrock.clear()
262
+ closers, self._closers = [*self._closers.values(), *retired], {}
263
+ openai_clients = list(self._openai.values())
264
+ self._openai.clear()
265
+ stack, self._stack = self._stack, AsyncExitStack()
266
+ await stack.aclose()
267
+ for close in closers:
268
+ with contextlib.suppress(Exception):
269
+ await close()
270
+ for client in openai_clients:
271
+ await client.close()