llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,364 @@
1
+ """Every registry setting: where it may be set, its type and range, and its library default.
2
+
3
+ One table drives validation (unknown names, levels, types, ranges) and resolution (defaults), so
4
+ a new setting is one row here. Used by `registry.schema` (checks) and `registry.resolve`.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+ from collections.abc import Callable, Mapping
11
+ from dataclasses import dataclass, field
12
+ from typing import Any
13
+
14
+ # Levels, most general first: `defaults`, an account's `settings`, the model, a deployment.
15
+ D, A, M, R = "D", "A", "M", "R"
16
+ LEVEL_NAMES = {D: "defaults", A: "account", M: "model", R: "deployment"}
17
+
18
+ Check = Callable[[Any], "str | None"]
19
+
20
+ STRATEGIES = ("ordered", "round_robin", "weighted", "headroom")
21
+ _CUSTOM_RE = re.compile(r"^custom:[A-Za-z0-9_.-]+$")
22
+ _HOOK_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
23
+
24
+
25
+ def _is_number(value: Any) -> bool:
26
+ return isinstance(value, int | float) and not isinstance(value, bool)
27
+
28
+
29
+ def number(
30
+ lo: float | None = None,
31
+ hi: float | None = None,
32
+ *,
33
+ integer: bool = False,
34
+ lo_open: bool = False,
35
+ ) -> Check:
36
+ """A number (an integer when `integer`) inside [lo, hi]; `lo_open` makes the low end strict."""
37
+
38
+ def check(value: Any) -> str | None:
39
+ if integer and not (isinstance(value, int) and not isinstance(value, bool)):
40
+ return "must be an integer"
41
+ if not _is_number(value):
42
+ return "must be a number"
43
+ if lo is not None and (value <= lo if lo_open else value < lo):
44
+ return f"must be {'above' if lo_open else 'at least'} {lo:g}"
45
+ if hi is not None and value > hi:
46
+ return f"must be at most {hi:g}"
47
+ return None
48
+
49
+ return check
50
+
51
+
52
+ def one_of(*values: str) -> Check:
53
+ """One of the given words."""
54
+
55
+ def check(value: Any) -> str | None:
56
+ return None if value in values else f"must be one of {', '.join(values)}"
57
+
58
+ return check
59
+
60
+
61
+ def boolean(value: Any) -> str | None:
62
+ """True or false."""
63
+ return None if isinstance(value, bool) else "must be true or false"
64
+
65
+
66
+ def text(value: Any) -> str | None:
67
+ """A non-empty string."""
68
+ return None if isinstance(value, str) and value else "must be a non-empty string"
69
+
70
+
71
+ def strategy(value: Any) -> str | None:
72
+ """A built-in balancing strategy or `custom:<name>`."""
73
+ if value in STRATEGIES or (isinstance(value, str) and _CUSTOM_RE.match(value)):
74
+ return None
75
+ return f"must be one of {', '.join(STRATEGIES)} or custom:<name>"
76
+
77
+
78
+ def hook_name(value: Any) -> str | None:
79
+ """The name of a registered hook."""
80
+ return None if isinstance(value, str) and _HOOK_RE.match(value) else "must be a hook name"
81
+
82
+
83
+ def text_list(value: Any) -> str | None:
84
+ """A list of non-empty strings."""
85
+ if isinstance(value, list) and all(isinstance(v, str) and v for v in value):
86
+ return None
87
+ return "must be a list of non-empty strings"
88
+
89
+
90
+ def any_object(value: Any) -> str | None:
91
+ """Any JSON object (passed through unchecked)."""
92
+ return None if isinstance(value, dict) else "must be an object"
93
+
94
+
95
+ def obj(fields: Mapping[str, Check], *, required: tuple[str, ...] = ()) -> Check:
96
+ """An object whose keys are among `fields`, each checked; `required` keys must be present."""
97
+
98
+ def check(value: Any) -> str | None:
99
+ if not isinstance(value, dict):
100
+ return "must be an object"
101
+ problems = [f"needs `{key}`" for key in required if key not in value]
102
+ for key, item in value.items():
103
+ sub = fields.get(key)
104
+ if sub is None:
105
+ problems.append(f"unknown field `{key}` (known: {', '.join(sorted(fields))})")
106
+ continue
107
+ problem = sub(item)
108
+ if problem:
109
+ problems.append(f"`{key}` {problem}")
110
+ return "; ".join(problems) or None
111
+
112
+ return check
113
+
114
+
115
+ def _tags(value: Any) -> str | None:
116
+ if isinstance(value, dict) and all(isinstance(v, str) or _is_number(v) for v in value.values()):
117
+ return None
118
+ return "must be an object of text or number values"
119
+
120
+
121
+ def _causes(value: Any) -> str | None:
122
+ allowed = ("exhausted", "broken", "error")
123
+ if isinstance(value, list) and value and all(v in allowed for v in value):
124
+ return None
125
+ return f"must be a non-empty list of {', '.join(allowed)}"
126
+
127
+
128
+ def _fallback_models(value: Any) -> str | None:
129
+ if not isinstance(value, list) or not value:
130
+ return "must be a non-empty list"
131
+ for item in value:
132
+ if isinstance(item, str) and item:
133
+ continue
134
+ entry = obj({"model": text, "require_headroom": boolean}, required=("model",))(item)
135
+ if entry:
136
+ return f"each entry is an alias or {{model, require_headroom}}: {entry}"
137
+ return None
138
+
139
+
140
+ def _sticky_key(value: Any) -> str | None:
141
+ if value == "session" or (isinstance(value, str) and _CUSTOM_RE.match(value)):
142
+ return None
143
+ return "must be session or custom:<name>"
144
+
145
+
146
+ def _stop(value: Any) -> str | None:
147
+ return None if isinstance(value, str) or text_list(value) is None else "must be text or a list"
148
+
149
+
150
+ # Inference parameters and the model type they belong to (rules of §3.9).
151
+ CHAT_INFERENCE: Mapping[str, Check] = {
152
+ "max_tokens": number(1, integer=True),
153
+ "temperature": number(0, 2),
154
+ "top_p": number(0, 1),
155
+ "stop": _stop,
156
+ "top_k": number(1, integer=True),
157
+ "seed": number(integer=True),
158
+ "frequency_penalty": number(-2, 2),
159
+ "presence_penalty": number(-2, 2),
160
+ "extra": any_object,
161
+ }
162
+ EMBED_INFERENCE: Mapping[str, Check] = {
163
+ "dimensions": number(1, integer=True),
164
+ "normalize": boolean,
165
+ "input_type": text,
166
+ "truncate": text,
167
+ "embedding_types": text_list,
168
+ }
169
+ INFERENCE_NAMES = frozenset(CHAT_INFERENCE) | frozenset(EMBED_INFERENCE)
170
+
171
+
172
+ def _locked(value: Any) -> str | None:
173
+ if text_list(value) is None and all(v in INFERENCE_NAMES for v in value):
174
+ return None
175
+ return "must be a list of inference parameter names"
176
+
177
+
178
+ HEALTH_DEFAULTS = {
179
+ "disable_errors": 3,
180
+ "disable_window_secs": 300,
181
+ "probe_after_secs": 900,
182
+ "probe_max_secs": 7200,
183
+ "slow_ratio": 3.0,
184
+ "slow_min_secs": 5,
185
+ "slow_min_samples": 20,
186
+ }
187
+ ALERT_DEFAULTS = {
188
+ "quota_used_pct": 90,
189
+ "quota_used_mins": 5,
190
+ "spill_pct": 20,
191
+ "spill_window_secs": 300,
192
+ "waiting_age_secs": 60,
193
+ "dlq_messages": 1,
194
+ }
195
+ QUEUE_DEFAULTS = {
196
+ "visibility_secs": 60,
197
+ "max_receives": 10,
198
+ "retention_secs": 345_600,
199
+ "dlq_retention_secs": 86_400,
200
+ }
201
+
202
+ _PCT = number(0, 100, integer=True)
203
+ _POS = number(0, lo_open=True)
204
+
205
+
206
+ @dataclass(frozen=True, slots=True)
207
+ class Spec:
208
+ """One setting: the levels it may be set at, its check, its default and how levels combine.
209
+
210
+ `default` may be a callable taking the model type (`chat` / `embedding`); `merge` objects
211
+ combine key by key across levels instead of the most specific one replacing the rest.
212
+ """
213
+
214
+ levels: frozenset[str]
215
+ check: Check
216
+ default: Any = None
217
+ merge: bool = False
218
+ deploy_only: bool = False
219
+ protected: bool = False
220
+ by_type: Callable[[str], Any] | None = field(default=None)
221
+
222
+
223
+ def _spec(levels: str, check: Check, default: Any = None, **kw: Any) -> Spec:
224
+ return Spec(frozenset(levels.split(",")), check, default, **kw)
225
+
226
+
227
+ SETTINGS: Mapping[str, Spec] = {
228
+ "enabled": _spec("A,M,R", boolean, True),
229
+ "rpm": _spec("D,M,R", number(1, integer=True)),
230
+ "tpm": _spec("D,M,R", number(1, integer=True)),
231
+ "rate_limits": _spec(
232
+ "D,A,M,R",
233
+ one_of("both", "rpm", "tpm", "off"),
234
+ by_type=lambda kind: "both" if kind == "chat" else "rpm",
235
+ ),
236
+ "tpm_output_weight": _spec("D,A,M,R", number(1), 1),
237
+ "tpm_count_cached": _spec("D,A,M,R", boolean, False),
238
+ "default_output_tokens": _spec("D,A,M,R", number(1, integer=True), 1000),
239
+ "shared_rpm": _spec("A", number(1, integer=True)),
240
+ "shared_tpm": _spec("A", number(1, integer=True)),
241
+ "rate_window": _spec("D,A,M,R", one_of("smooth", "fixed"), "smooth"),
242
+ "rate_burst": _spec("D,A,M,R", number(0, 1), 0.1),
243
+ "stream_flush_ms": _spec("M,R", number(0, 1000, integer=True), 30),
244
+ "max_concurrency": _spec("D,A,M,R", number(1, integer=True)),
245
+ "priority_reserve": _spec(
246
+ "D,A,M", number(0, 1), by_type=lambda kind: 0.2 if kind == "chat" else 0.0
247
+ ),
248
+ "normal_min_share": _spec("D,M", number(0, 1), 0.1),
249
+ "timeout_secs": _spec("D,A,M", _POS, 120),
250
+ "timeout_max_secs": _spec("D,M", _POS, 900),
251
+ "max_wait_secs": _spec("D,M", _POS),
252
+ "attempt_timeout_secs": _spec("D,A,M,R", _POS, 60),
253
+ "throttle_retry_attempts": _spec("D,A,M,R", number(1, 20, integer=True), 6),
254
+ "retry_base_secs": _spec("D,A,M,R", number(0), 1.0),
255
+ "retry_multiplier": _spec("D,A,M,R", number(1), 4.0),
256
+ "retry_spread": _spec("D,A,M,R", number(1), 2.5),
257
+ "retry_max_secs": _spec("D,A,M,R", _POS, 30.0),
258
+ "breaker_threshold": _spec("D,A,M,R", number(1, integer=True), 5),
259
+ "breaker_cooldown_secs": _spec("D,A,M,R", _POS, 15.0),
260
+ # Failures count toward the threshold only within this window (one after it restarts it).
261
+ "breaker_window_secs": _spec("D,A,M,R", _POS, 60.0),
262
+ # A probe call that never reported back stops blocking others after this.
263
+ "breaker_probe_secs": _spec("D,A,M,R", _POS, 60.0),
264
+ "balance": _spec("D,M", strategy, "ordered"),
265
+ "balance_opts": _spec(
266
+ "D,M", obj({"pick": one_of("percent", "max", "spread")}), {"pick": "percent"}, merge=True
267
+ ),
268
+ "balance_priority": _spec("M", strategy),
269
+ "max_attempts": _spec("D,M", number(1, integer=True), 4),
270
+ "retry_budget_pct": _spec("D,M", _PCT, 20),
271
+ "retry_budget_min": _spec("D,M", number(0, integer=True), 10),
272
+ "stream_idle_secs": _spec("D,M", _POS, 30),
273
+ "stream_max_bytes": _spec("M", number(1024, integer=True), 8_388_608),
274
+ "max_waiting": _spec("D,M", number(0, integer=True), 50_000),
275
+ "sticky_move_after_secs": _spec("D,M", _POS, 1),
276
+ "require_headroom_pct": _spec("D,M", _PCT, 20),
277
+ "tags": _spec("R", _tags),
278
+ "weight": _spec("R", number(0), 1),
279
+ "sticky": _spec(
280
+ "D,M",
281
+ obj({"key": _sticky_key, "idle_ttl_secs": _POS}, required=("key",)),
282
+ ),
283
+ "fallback": _spec(
284
+ "D,M",
285
+ obj(
286
+ {"on": _causes, "after_secs": number(0), "regions": one_of("account", "any")},
287
+ required=("on",),
288
+ ),
289
+ ),
290
+ "model_fallback": _spec(
291
+ "M",
292
+ obj(
293
+ {"on": _causes, "after_secs": number(0), "models": _fallback_models},
294
+ required=("on", "models"),
295
+ ),
296
+ ),
297
+ "store_down": _spec("D,M", one_of("pause", "static_cap", "fail_fast"), "pause"),
298
+ "queue_pct": _spec("D,M", _PCT, 100),
299
+ "worker_slots": _spec("D,M", number(1, 1000, integer=True), 20, deploy_only=True),
300
+ "queue": _spec(
301
+ "D,M",
302
+ obj(
303
+ {
304
+ "name": text,
305
+ "visibility_secs": number(1, 43_200, integer=True),
306
+ "max_receives": number(1, integer=True),
307
+ "retention_secs": number(60, integer=True),
308
+ "dlq_retention_secs": number(60, integer=True),
309
+ }
310
+ ),
311
+ QUEUE_DEFAULTS,
312
+ merge=True,
313
+ ),
314
+ "embed_batch_size": _spec("M", number(1, integer=True), 1),
315
+ "inference": _spec("D,A,M,R", obj({**CHAT_INFERENCE, **EMBED_INFERENCE}), merge=True),
316
+ "inference_limits": _spec(
317
+ "D,A,M,R",
318
+ obj(
319
+ {
320
+ "max_tokens": number(1, integer=True),
321
+ "locked": _locked,
322
+ "on_exceed": one_of("clamp", "reject"),
323
+ }
324
+ ),
325
+ merge=True,
326
+ ),
327
+ "params": _spec("R", any_object, protected=True),
328
+ "health": _spec(
329
+ "D,A,M,R",
330
+ obj(
331
+ {
332
+ "disable_errors": number(1, integer=True),
333
+ "disable_window_secs": _POS,
334
+ "probe_after_secs": _POS,
335
+ "probe_max_secs": _POS,
336
+ "slow_ratio": number(1, lo_open=True),
337
+ "slow_min_secs": number(0),
338
+ "slow_min_samples": number(1, integer=True),
339
+ }
340
+ ),
341
+ HEALTH_DEFAULTS,
342
+ merge=True,
343
+ ),
344
+ "alerts": _spec(
345
+ "D,M",
346
+ obj({key: number(0) for key in ALERT_DEFAULTS}),
347
+ ALERT_DEFAULTS,
348
+ merge=True,
349
+ ),
350
+ "admission": _spec("D,M", hook_name),
351
+ "fallback_policy": _spec("D,M", hook_name),
352
+ "token_estimator": _spec("D,M", hook_name),
353
+ "probe": _spec("D,M", hook_name),
354
+ "debug_capture": _spec("M", boolean, False),
355
+ }
356
+
357
+ # Defaults of the optional fields inside the two fallback rules.
358
+ FALLBACK_AFTER_SECS = 2.0
359
+ MODEL_FALLBACK_AFTER_SECS = 5.0
360
+
361
+
362
+ def settings_at(level: str) -> frozenset[str]:
363
+ """Names of the settings that may be set at `level`."""
364
+ return frozenset(name for name, spec in SETTINGS.items() if level in spec.levels)