llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
"""Every registry setting: where it may be set, its type and range, and its library default.
|
|
2
|
+
|
|
3
|
+
One table drives validation (unknown names, levels, types, ranges) and resolution (defaults), so
|
|
4
|
+
a new setting is one row here. Used by `registry.schema` (checks) and `registry.resolve`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from collections.abc import Callable, Mapping
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
# Levels, most general first: `defaults`, an account's `settings`, the model, a deployment.
|
|
15
|
+
D, A, M, R = "D", "A", "M", "R"
|
|
16
|
+
LEVEL_NAMES = {D: "defaults", A: "account", M: "model", R: "deployment"}
|
|
17
|
+
|
|
18
|
+
Check = Callable[[Any], "str | None"]
|
|
19
|
+
|
|
20
|
+
STRATEGIES = ("ordered", "round_robin", "weighted", "headroom")
|
|
21
|
+
_CUSTOM_RE = re.compile(r"^custom:[A-Za-z0-9_.-]+$")
|
|
22
|
+
_HOOK_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _is_number(value: Any) -> bool:
|
|
26
|
+
return isinstance(value, int | float) and not isinstance(value, bool)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def number(
|
|
30
|
+
lo: float | None = None,
|
|
31
|
+
hi: float | None = None,
|
|
32
|
+
*,
|
|
33
|
+
integer: bool = False,
|
|
34
|
+
lo_open: bool = False,
|
|
35
|
+
) -> Check:
|
|
36
|
+
"""A number (an integer when `integer`) inside [lo, hi]; `lo_open` makes the low end strict."""
|
|
37
|
+
|
|
38
|
+
def check(value: Any) -> str | None:
|
|
39
|
+
if integer and not (isinstance(value, int) and not isinstance(value, bool)):
|
|
40
|
+
return "must be an integer"
|
|
41
|
+
if not _is_number(value):
|
|
42
|
+
return "must be a number"
|
|
43
|
+
if lo is not None and (value <= lo if lo_open else value < lo):
|
|
44
|
+
return f"must be {'above' if lo_open else 'at least'} {lo:g}"
|
|
45
|
+
if hi is not None and value > hi:
|
|
46
|
+
return f"must be at most {hi:g}"
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
return check
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def one_of(*values: str) -> Check:
|
|
53
|
+
"""One of the given words."""
|
|
54
|
+
|
|
55
|
+
def check(value: Any) -> str | None:
|
|
56
|
+
return None if value in values else f"must be one of {', '.join(values)}"
|
|
57
|
+
|
|
58
|
+
return check
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def boolean(value: Any) -> str | None:
|
|
62
|
+
"""True or false."""
|
|
63
|
+
return None if isinstance(value, bool) else "must be true or false"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def text(value: Any) -> str | None:
|
|
67
|
+
"""A non-empty string."""
|
|
68
|
+
return None if isinstance(value, str) and value else "must be a non-empty string"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def strategy(value: Any) -> str | None:
|
|
72
|
+
"""A built-in balancing strategy or `custom:<name>`."""
|
|
73
|
+
if value in STRATEGIES or (isinstance(value, str) and _CUSTOM_RE.match(value)):
|
|
74
|
+
return None
|
|
75
|
+
return f"must be one of {', '.join(STRATEGIES)} or custom:<name>"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def hook_name(value: Any) -> str | None:
|
|
79
|
+
"""The name of a registered hook."""
|
|
80
|
+
return None if isinstance(value, str) and _HOOK_RE.match(value) else "must be a hook name"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def text_list(value: Any) -> str | None:
|
|
84
|
+
"""A list of non-empty strings."""
|
|
85
|
+
if isinstance(value, list) and all(isinstance(v, str) and v for v in value):
|
|
86
|
+
return None
|
|
87
|
+
return "must be a list of non-empty strings"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def any_object(value: Any) -> str | None:
|
|
91
|
+
"""Any JSON object (passed through unchecked)."""
|
|
92
|
+
return None if isinstance(value, dict) else "must be an object"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def obj(fields: Mapping[str, Check], *, required: tuple[str, ...] = ()) -> Check:
|
|
96
|
+
"""An object whose keys are among `fields`, each checked; `required` keys must be present."""
|
|
97
|
+
|
|
98
|
+
def check(value: Any) -> str | None:
|
|
99
|
+
if not isinstance(value, dict):
|
|
100
|
+
return "must be an object"
|
|
101
|
+
problems = [f"needs `{key}`" for key in required if key not in value]
|
|
102
|
+
for key, item in value.items():
|
|
103
|
+
sub = fields.get(key)
|
|
104
|
+
if sub is None:
|
|
105
|
+
problems.append(f"unknown field `{key}` (known: {', '.join(sorted(fields))})")
|
|
106
|
+
continue
|
|
107
|
+
problem = sub(item)
|
|
108
|
+
if problem:
|
|
109
|
+
problems.append(f"`{key}` {problem}")
|
|
110
|
+
return "; ".join(problems) or None
|
|
111
|
+
|
|
112
|
+
return check
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _tags(value: Any) -> str | None:
|
|
116
|
+
if isinstance(value, dict) and all(isinstance(v, str) or _is_number(v) for v in value.values()):
|
|
117
|
+
return None
|
|
118
|
+
return "must be an object of text or number values"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _causes(value: Any) -> str | None:
|
|
122
|
+
allowed = ("exhausted", "broken", "error")
|
|
123
|
+
if isinstance(value, list) and value and all(v in allowed for v in value):
|
|
124
|
+
return None
|
|
125
|
+
return f"must be a non-empty list of {', '.join(allowed)}"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _fallback_models(value: Any) -> str | None:
|
|
129
|
+
if not isinstance(value, list) or not value:
|
|
130
|
+
return "must be a non-empty list"
|
|
131
|
+
for item in value:
|
|
132
|
+
if isinstance(item, str) and item:
|
|
133
|
+
continue
|
|
134
|
+
entry = obj({"model": text, "require_headroom": boolean}, required=("model",))(item)
|
|
135
|
+
if entry:
|
|
136
|
+
return f"each entry is an alias or {{model, require_headroom}}: {entry}"
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _sticky_key(value: Any) -> str | None:
|
|
141
|
+
if value == "session" or (isinstance(value, str) and _CUSTOM_RE.match(value)):
|
|
142
|
+
return None
|
|
143
|
+
return "must be session or custom:<name>"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _stop(value: Any) -> str | None:
|
|
147
|
+
return None if isinstance(value, str) or text_list(value) is None else "must be text or a list"
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# Inference parameters and the model type they belong to (rules of §3.9).
|
|
151
|
+
CHAT_INFERENCE: Mapping[str, Check] = {
|
|
152
|
+
"max_tokens": number(1, integer=True),
|
|
153
|
+
"temperature": number(0, 2),
|
|
154
|
+
"top_p": number(0, 1),
|
|
155
|
+
"stop": _stop,
|
|
156
|
+
"top_k": number(1, integer=True),
|
|
157
|
+
"seed": number(integer=True),
|
|
158
|
+
"frequency_penalty": number(-2, 2),
|
|
159
|
+
"presence_penalty": number(-2, 2),
|
|
160
|
+
"extra": any_object,
|
|
161
|
+
}
|
|
162
|
+
EMBED_INFERENCE: Mapping[str, Check] = {
|
|
163
|
+
"dimensions": number(1, integer=True),
|
|
164
|
+
"normalize": boolean,
|
|
165
|
+
"input_type": text,
|
|
166
|
+
"truncate": text,
|
|
167
|
+
"embedding_types": text_list,
|
|
168
|
+
}
|
|
169
|
+
INFERENCE_NAMES = frozenset(CHAT_INFERENCE) | frozenset(EMBED_INFERENCE)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _locked(value: Any) -> str | None:
|
|
173
|
+
if text_list(value) is None and all(v in INFERENCE_NAMES for v in value):
|
|
174
|
+
return None
|
|
175
|
+
return "must be a list of inference parameter names"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
HEALTH_DEFAULTS = {
|
|
179
|
+
"disable_errors": 3,
|
|
180
|
+
"disable_window_secs": 300,
|
|
181
|
+
"probe_after_secs": 900,
|
|
182
|
+
"probe_max_secs": 7200,
|
|
183
|
+
"slow_ratio": 3.0,
|
|
184
|
+
"slow_min_secs": 5,
|
|
185
|
+
"slow_min_samples": 20,
|
|
186
|
+
}
|
|
187
|
+
ALERT_DEFAULTS = {
|
|
188
|
+
"quota_used_pct": 90,
|
|
189
|
+
"quota_used_mins": 5,
|
|
190
|
+
"spill_pct": 20,
|
|
191
|
+
"spill_window_secs": 300,
|
|
192
|
+
"waiting_age_secs": 60,
|
|
193
|
+
"dlq_messages": 1,
|
|
194
|
+
}
|
|
195
|
+
QUEUE_DEFAULTS = {
|
|
196
|
+
"visibility_secs": 60,
|
|
197
|
+
"max_receives": 10,
|
|
198
|
+
"retention_secs": 345_600,
|
|
199
|
+
"dlq_retention_secs": 86_400,
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
_PCT = number(0, 100, integer=True)
|
|
203
|
+
_POS = number(0, lo_open=True)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclass(frozen=True, slots=True)
|
|
207
|
+
class Spec:
|
|
208
|
+
"""One setting: the levels it may be set at, its check, its default and how levels combine.
|
|
209
|
+
|
|
210
|
+
`default` may be a callable taking the model type (`chat` / `embedding`); `merge` objects
|
|
211
|
+
combine key by key across levels instead of the most specific one replacing the rest.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
levels: frozenset[str]
|
|
215
|
+
check: Check
|
|
216
|
+
default: Any = None
|
|
217
|
+
merge: bool = False
|
|
218
|
+
deploy_only: bool = False
|
|
219
|
+
protected: bool = False
|
|
220
|
+
by_type: Callable[[str], Any] | None = field(default=None)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _spec(levels: str, check: Check, default: Any = None, **kw: Any) -> Spec:
|
|
224
|
+
return Spec(frozenset(levels.split(",")), check, default, **kw)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
SETTINGS: Mapping[str, Spec] = {
|
|
228
|
+
"enabled": _spec("A,M,R", boolean, True),
|
|
229
|
+
"rpm": _spec("D,M,R", number(1, integer=True)),
|
|
230
|
+
"tpm": _spec("D,M,R", number(1, integer=True)),
|
|
231
|
+
"rate_limits": _spec(
|
|
232
|
+
"D,A,M,R",
|
|
233
|
+
one_of("both", "rpm", "tpm", "off"),
|
|
234
|
+
by_type=lambda kind: "both" if kind == "chat" else "rpm",
|
|
235
|
+
),
|
|
236
|
+
"tpm_output_weight": _spec("D,A,M,R", number(1), 1),
|
|
237
|
+
"tpm_count_cached": _spec("D,A,M,R", boolean, False),
|
|
238
|
+
"default_output_tokens": _spec("D,A,M,R", number(1, integer=True), 1000),
|
|
239
|
+
"shared_rpm": _spec("A", number(1, integer=True)),
|
|
240
|
+
"shared_tpm": _spec("A", number(1, integer=True)),
|
|
241
|
+
"rate_window": _spec("D,A,M,R", one_of("smooth", "fixed"), "smooth"),
|
|
242
|
+
"rate_burst": _spec("D,A,M,R", number(0, 1), 0.1),
|
|
243
|
+
"stream_flush_ms": _spec("M,R", number(0, 1000, integer=True), 30),
|
|
244
|
+
"max_concurrency": _spec("D,A,M,R", number(1, integer=True)),
|
|
245
|
+
"priority_reserve": _spec(
|
|
246
|
+
"D,A,M", number(0, 1), by_type=lambda kind: 0.2 if kind == "chat" else 0.0
|
|
247
|
+
),
|
|
248
|
+
"normal_min_share": _spec("D,M", number(0, 1), 0.1),
|
|
249
|
+
"timeout_secs": _spec("D,A,M", _POS, 120),
|
|
250
|
+
"timeout_max_secs": _spec("D,M", _POS, 900),
|
|
251
|
+
"max_wait_secs": _spec("D,M", _POS),
|
|
252
|
+
"attempt_timeout_secs": _spec("D,A,M,R", _POS, 60),
|
|
253
|
+
"throttle_retry_attempts": _spec("D,A,M,R", number(1, 20, integer=True), 6),
|
|
254
|
+
"retry_base_secs": _spec("D,A,M,R", number(0), 1.0),
|
|
255
|
+
"retry_multiplier": _spec("D,A,M,R", number(1), 4.0),
|
|
256
|
+
"retry_spread": _spec("D,A,M,R", number(1), 2.5),
|
|
257
|
+
"retry_max_secs": _spec("D,A,M,R", _POS, 30.0),
|
|
258
|
+
"breaker_threshold": _spec("D,A,M,R", number(1, integer=True), 5),
|
|
259
|
+
"breaker_cooldown_secs": _spec("D,A,M,R", _POS, 15.0),
|
|
260
|
+
# Failures count toward the threshold only within this window (one after it restarts it).
|
|
261
|
+
"breaker_window_secs": _spec("D,A,M,R", _POS, 60.0),
|
|
262
|
+
# A probe call that never reported back stops blocking others after this.
|
|
263
|
+
"breaker_probe_secs": _spec("D,A,M,R", _POS, 60.0),
|
|
264
|
+
"balance": _spec("D,M", strategy, "ordered"),
|
|
265
|
+
"balance_opts": _spec(
|
|
266
|
+
"D,M", obj({"pick": one_of("percent", "max", "spread")}), {"pick": "percent"}, merge=True
|
|
267
|
+
),
|
|
268
|
+
"balance_priority": _spec("M", strategy),
|
|
269
|
+
"max_attempts": _spec("D,M", number(1, integer=True), 4),
|
|
270
|
+
"retry_budget_pct": _spec("D,M", _PCT, 20),
|
|
271
|
+
"retry_budget_min": _spec("D,M", number(0, integer=True), 10),
|
|
272
|
+
"stream_idle_secs": _spec("D,M", _POS, 30),
|
|
273
|
+
"stream_max_bytes": _spec("M", number(1024, integer=True), 8_388_608),
|
|
274
|
+
"max_waiting": _spec("D,M", number(0, integer=True), 50_000),
|
|
275
|
+
"sticky_move_after_secs": _spec("D,M", _POS, 1),
|
|
276
|
+
"require_headroom_pct": _spec("D,M", _PCT, 20),
|
|
277
|
+
"tags": _spec("R", _tags),
|
|
278
|
+
"weight": _spec("R", number(0), 1),
|
|
279
|
+
"sticky": _spec(
|
|
280
|
+
"D,M",
|
|
281
|
+
obj({"key": _sticky_key, "idle_ttl_secs": _POS}, required=("key",)),
|
|
282
|
+
),
|
|
283
|
+
"fallback": _spec(
|
|
284
|
+
"D,M",
|
|
285
|
+
obj(
|
|
286
|
+
{"on": _causes, "after_secs": number(0), "regions": one_of("account", "any")},
|
|
287
|
+
required=("on",),
|
|
288
|
+
),
|
|
289
|
+
),
|
|
290
|
+
"model_fallback": _spec(
|
|
291
|
+
"M",
|
|
292
|
+
obj(
|
|
293
|
+
{"on": _causes, "after_secs": number(0), "models": _fallback_models},
|
|
294
|
+
required=("on", "models"),
|
|
295
|
+
),
|
|
296
|
+
),
|
|
297
|
+
"store_down": _spec("D,M", one_of("pause", "static_cap", "fail_fast"), "pause"),
|
|
298
|
+
"queue_pct": _spec("D,M", _PCT, 100),
|
|
299
|
+
"worker_slots": _spec("D,M", number(1, 1000, integer=True), 20, deploy_only=True),
|
|
300
|
+
"queue": _spec(
|
|
301
|
+
"D,M",
|
|
302
|
+
obj(
|
|
303
|
+
{
|
|
304
|
+
"name": text,
|
|
305
|
+
"visibility_secs": number(1, 43_200, integer=True),
|
|
306
|
+
"max_receives": number(1, integer=True),
|
|
307
|
+
"retention_secs": number(60, integer=True),
|
|
308
|
+
"dlq_retention_secs": number(60, integer=True),
|
|
309
|
+
}
|
|
310
|
+
),
|
|
311
|
+
QUEUE_DEFAULTS,
|
|
312
|
+
merge=True,
|
|
313
|
+
),
|
|
314
|
+
"embed_batch_size": _spec("M", number(1, integer=True), 1),
|
|
315
|
+
"inference": _spec("D,A,M,R", obj({**CHAT_INFERENCE, **EMBED_INFERENCE}), merge=True),
|
|
316
|
+
"inference_limits": _spec(
|
|
317
|
+
"D,A,M,R",
|
|
318
|
+
obj(
|
|
319
|
+
{
|
|
320
|
+
"max_tokens": number(1, integer=True),
|
|
321
|
+
"locked": _locked,
|
|
322
|
+
"on_exceed": one_of("clamp", "reject"),
|
|
323
|
+
}
|
|
324
|
+
),
|
|
325
|
+
merge=True,
|
|
326
|
+
),
|
|
327
|
+
"params": _spec("R", any_object, protected=True),
|
|
328
|
+
"health": _spec(
|
|
329
|
+
"D,A,M,R",
|
|
330
|
+
obj(
|
|
331
|
+
{
|
|
332
|
+
"disable_errors": number(1, integer=True),
|
|
333
|
+
"disable_window_secs": _POS,
|
|
334
|
+
"probe_after_secs": _POS,
|
|
335
|
+
"probe_max_secs": _POS,
|
|
336
|
+
"slow_ratio": number(1, lo_open=True),
|
|
337
|
+
"slow_min_secs": number(0),
|
|
338
|
+
"slow_min_samples": number(1, integer=True),
|
|
339
|
+
}
|
|
340
|
+
),
|
|
341
|
+
HEALTH_DEFAULTS,
|
|
342
|
+
merge=True,
|
|
343
|
+
),
|
|
344
|
+
"alerts": _spec(
|
|
345
|
+
"D,M",
|
|
346
|
+
obj({key: number(0) for key in ALERT_DEFAULTS}),
|
|
347
|
+
ALERT_DEFAULTS,
|
|
348
|
+
merge=True,
|
|
349
|
+
),
|
|
350
|
+
"admission": _spec("D,M", hook_name),
|
|
351
|
+
"fallback_policy": _spec("D,M", hook_name),
|
|
352
|
+
"token_estimator": _spec("D,M", hook_name),
|
|
353
|
+
"probe": _spec("D,M", hook_name),
|
|
354
|
+
"debug_capture": _spec("M", boolean, False),
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
# Defaults of the optional fields inside the two fallback rules.
|
|
358
|
+
FALLBACK_AFTER_SECS = 2.0
|
|
359
|
+
MODEL_FALLBACK_AFTER_SECS = 5.0
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def settings_at(level: str) -> frozenset[str]:
|
|
363
|
+
"""Names of the settings that may be set at `level`."""
|
|
364
|
+
return frozenset(name for name, spec in SETTINGS.items() if level in spec.levels)
|