llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Message format version 2 (plan E80) and the reply stored for the caller.
|
|
2
|
+
|
|
3
|
+
A call's deadline is not an absolute pod time: the message carries `timeout_secs` and the
|
|
4
|
+
worker measures it from the queue position (the store's clock), so pod clock skew does not
|
|
5
|
+
matter (plan §3.18). Version 2 only: a newer message is returned to the queue, never dropped.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
from typing import Any, Literal
|
|
12
|
+
|
|
13
|
+
import orjson
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
15
|
+
|
|
16
|
+
from llm_queue.errors import UnsupportedEnvelopeError
|
|
17
|
+
|
|
18
|
+
VERSION = 2
|
|
19
|
+
Op = Literal["converse", "invoke_model", "chat", "embeddings"]
|
|
20
|
+
AllowFallback = Literal["all", "regions", "none"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Envelope(BaseModel):
|
|
24
|
+
"""One LLM call on the queue. `params` are provider-native and travel inside the message."""
|
|
25
|
+
|
|
26
|
+
model_config = ConfigDict(frozen=True, extra="ignore")
|
|
27
|
+
|
|
28
|
+
v: int = VERSION
|
|
29
|
+
request_id: str
|
|
30
|
+
op: Op
|
|
31
|
+
model: str
|
|
32
|
+
params: dict[str, Any]
|
|
33
|
+
priority: bool = False
|
|
34
|
+
caller: str
|
|
35
|
+
timeout_secs: float = Field(gt=0)
|
|
36
|
+
max_wait_secs: float | None = Field(default=None, gt=0)
|
|
37
|
+
allow_fallback: AllowFallback = "all"
|
|
38
|
+
allowed_regions: tuple[str, ...] | None = None
|
|
39
|
+
allowed_accounts: tuple[str, ...] | None = None
|
|
40
|
+
session_id: str | None = None
|
|
41
|
+
user_id: str | None = None
|
|
42
|
+
tenant_id: str | None = None
|
|
43
|
+
app_request_id: bool = False
|
|
44
|
+
stream: bool = False
|
|
45
|
+
|
|
46
|
+
def dumps(self) -> bytes:
|
|
47
|
+
"""JSON bytes for the queue (fields directly: `model_dump` re-walked the params)."""
|
|
48
|
+
return orjson.dumps({k: v for k, v in self.__dict__.items() if v is not None})
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse(data: bytes | str) -> Envelope:
|
|
52
|
+
"""Decode a message; `UnsupportedEnvelopeError` for a newer version (return it, do not drop)."""
|
|
53
|
+
raw = orjson.loads(data)
|
|
54
|
+
if not isinstance(raw, dict) or not isinstance(raw.get("v"), int):
|
|
55
|
+
raise ValueError("not an envelope")
|
|
56
|
+
if raw["v"] > VERSION:
|
|
57
|
+
raise UnsupportedEnvelopeError(f"message version {raw['v']} is newer than {VERSION}")
|
|
58
|
+
if raw["v"] < VERSION:
|
|
59
|
+
raise ValueError(f"message version {raw['v']} is not supported (0.3 reads version 2)")
|
|
60
|
+
return Envelope.model_validate(raw)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def params_hash(params: Any) -> str:
|
|
64
|
+
"""Stable hash of a call's params (duplicate request ids must carry the same params)."""
|
|
65
|
+
return hashlib.sha256(orjson.dumps(params, option=orjson.OPT_SORT_KEYS)).hexdigest()[:32]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Meta(BaseModel):
|
|
69
|
+
"""How a call was served: deployment, fallback, attempts, timings, tokens."""
|
|
70
|
+
|
|
71
|
+
model_config = ConfigDict(frozen=True)
|
|
72
|
+
|
|
73
|
+
served_model: str = ""
|
|
74
|
+
deployment: str = ""
|
|
75
|
+
fallback: Literal["region", "model"] | None = None
|
|
76
|
+
attempts: int = 0
|
|
77
|
+
queue_wait_ms: int = 0
|
|
78
|
+
provider_ms: int = 0
|
|
79
|
+
tokens_in: int | None = None
|
|
80
|
+
tokens_out: int | None = None
|
|
81
|
+
worker: str = ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Reply(BaseModel):
|
|
85
|
+
"""The stored result of a call: the provider reply, or an error kind with its details."""
|
|
86
|
+
|
|
87
|
+
model_config = ConfigDict(frozen=True)
|
|
88
|
+
|
|
89
|
+
request_id: str
|
|
90
|
+
ok: bool
|
|
91
|
+
response: dict[str, Any] | None = None
|
|
92
|
+
error: str | None = None
|
|
93
|
+
message: str = ""
|
|
94
|
+
retry_after_secs: float | None = None
|
|
95
|
+
partial_text: str | None = None
|
|
96
|
+
meta: Meta = Field(default_factory=Meta)
|
|
97
|
+
|
|
98
|
+
def dumps(self) -> bytes:
|
|
99
|
+
"""JSON bytes for the store (fields directly: `model_dump` re-walked the response)."""
|
|
100
|
+
out = {k: v for k, v in self.__dict__.items() if v is not None}
|
|
101
|
+
out["meta"] = {k: v for k, v in self.meta.__dict__.items() if v is not None}
|
|
102
|
+
return orjson.dumps(out)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Error kinds a caller can receive (plan §3.19), as one exception type with a `kind`.
|
|
2
|
+
|
|
3
|
+
Callers catch `CallFailed` (an `LLMQueueError`) and branch on `kind`; `retry_after_secs` is set
|
|
4
|
+
where the plan says a later retry can work, `partial_text` for a stream cut mid-answer.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from llm_queue.errors import LLMQueueError
|
|
10
|
+
|
|
11
|
+
KINDS = frozenset(
|
|
12
|
+
{
|
|
13
|
+
"rate_limited",
|
|
14
|
+
"deadline_exceeded",
|
|
15
|
+
"request_too_large",
|
|
16
|
+
"no_deployment",
|
|
17
|
+
"model_unavailable",
|
|
18
|
+
"model_disabled",
|
|
19
|
+
"queue_full",
|
|
20
|
+
"queue_unavailable",
|
|
21
|
+
"credentials_unavailable",
|
|
22
|
+
"retry_budget_exhausted",
|
|
23
|
+
"request_id_conflict",
|
|
24
|
+
"result_expired",
|
|
25
|
+
"dead_lettered",
|
|
26
|
+
"worker_lost",
|
|
27
|
+
"cancelled",
|
|
28
|
+
"provider_error",
|
|
29
|
+
"rejected",
|
|
30
|
+
}
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CallFailed(LLMQueueError):
|
|
35
|
+
"""A call ended without a reply; `kind` is one of `KINDS`."""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
kind: str,
|
|
40
|
+
message: str = "",
|
|
41
|
+
*,
|
|
42
|
+
retry_after_secs: float | None = None,
|
|
43
|
+
partial_text: str | None = None,
|
|
44
|
+
request_id: str | None = None,
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Keep the kind and its details."""
|
|
47
|
+
super().__init__(f"{kind}: {message}" if message else kind)
|
|
48
|
+
self.kind = kind
|
|
49
|
+
self.retry_after_secs = retry_after_secs
|
|
50
|
+
self.partial_text = partial_text
|
|
51
|
+
self.request_id = request_id
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Deployment health (plan §3.5, §3.18): auto-disable with safeguards, probes, recovery.
|
|
2
|
+
|
|
3
|
+
Target errors (model not in region, access denied) are counted per deployment and request in a
|
|
4
|
+
window; at `disable_errors` the deployment is disabled, unless it is the model's last usable one
|
|
5
|
+
(then only an alert). Errors are counted once the call's outcome is known: a request that hits
|
|
6
|
+
target errors on two deployments is the cause and is never counted. A disabled deployment stays
|
|
7
|
+
out until a probe succeeds; probes start after `probe_after_secs` and back off up to
|
|
8
|
+
`probe_max_secs`. Health lives apart from the registry, so a publish never re-enables one.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
import orjson
|
|
17
|
+
|
|
18
|
+
from llm_queue.engine.keys import Keys
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
import redis.asyncio
|
|
22
|
+
|
|
23
|
+
_NOW = """
|
|
24
|
+
local t = redis.call('TIME')
|
|
25
|
+
local now = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
|
|
26
|
+
local req = cjson.decode(ARGV[1])
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
# KEYS: health hash, the deployment's target-error set. req: dep, request_id, window_ms, needed,
|
|
30
|
+
# probe_after_ms, others (keys of the model's other deployments).
|
|
31
|
+
# Reply: counted | disabled | last.
|
|
32
|
+
RECORD = (
|
|
33
|
+
_NOW
|
|
34
|
+
+ """
|
|
35
|
+
redis.call('ZADD', KEYS[2], now, req.request_id)
|
|
36
|
+
redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now - req.window_ms)
|
|
37
|
+
redis.call('PEXPIRE', KEYS[2], req.window_ms)
|
|
38
|
+
if redis.call('ZCARD', KEYS[2]) < req.needed then return 'counted' end
|
|
39
|
+
local usable_left = 0
|
|
40
|
+
for _, other in ipairs(req.others) do
|
|
41
|
+
if not redis.call('HGET', KEYS[1], other .. '|until') then usable_left = usable_left + 1 end
|
|
42
|
+
end
|
|
43
|
+
if usable_left == 0 then return 'last' end
|
|
44
|
+
redis.call('HSET', KEYS[1], req.dep .. '|until', now + req.probe_after_ms,
|
|
45
|
+
req.dep .. '|wait', req.probe_after_ms, req.dep .. '|since', now)
|
|
46
|
+
redis.call('DEL', KEYS[2])
|
|
47
|
+
return 'disabled'
|
|
48
|
+
"""
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# KEYS: health hash. req: dep, ok (bool), probe_max_ms. Reply: recovered | waiting.
|
|
52
|
+
PROBED = (
|
|
53
|
+
_NOW
|
|
54
|
+
+ """
|
|
55
|
+
if req.ok then
|
|
56
|
+
redis.call('HDEL', KEYS[1], req.dep .. '|until', req.dep .. '|wait', req.dep .. '|since')
|
|
57
|
+
return 'recovered'
|
|
58
|
+
end
|
|
59
|
+
local last = tonumber(redis.call('HGET', KEYS[1], req.dep .. '|wait') or '0')
|
|
60
|
+
local wait = math.min(math.max(last * 2, 60000), req.probe_max_ms)
|
|
61
|
+
redis.call('HSET', KEYS[1], req.dep .. '|until', now + wait, req.dep .. '|wait', wait)
|
|
62
|
+
return 'waiting'
|
|
63
|
+
"""
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Health:
|
|
68
|
+
"""Health records of every model of one environment."""
|
|
69
|
+
|
|
70
|
+
def __init__(self, client: redis.asyncio.Redis, keys: Keys) -> None:
|
|
71
|
+
"""Register the scripts; no I/O until the first call."""
|
|
72
|
+
self._client = client
|
|
73
|
+
self.keys = keys
|
|
74
|
+
self._record = client.register_script(RECORD)
|
|
75
|
+
self._probed = client.register_script(PROBED)
|
|
76
|
+
|
|
77
|
+
def _terr(self, alias: str, dep: str) -> str:
|
|
78
|
+
return self.keys.model(alias, f"terr:{dep}")
|
|
79
|
+
|
|
80
|
+
async def record_target_error(
|
|
81
|
+
self,
|
|
82
|
+
alias: str,
|
|
83
|
+
dep: str,
|
|
84
|
+
request_id: str,
|
|
85
|
+
*,
|
|
86
|
+
others: Sequence[str],
|
|
87
|
+
window_secs: float,
|
|
88
|
+
needed: int,
|
|
89
|
+
probe_after_secs: float,
|
|
90
|
+
) -> str:
|
|
91
|
+
"""Count a target error; `disabled`, `last` (would be the last usable) or `counted`."""
|
|
92
|
+
body = {
|
|
93
|
+
"dep": dep,
|
|
94
|
+
"request_id": request_id,
|
|
95
|
+
"window_ms": int(window_secs * 1000),
|
|
96
|
+
"needed": needed,
|
|
97
|
+
"probe_after_ms": int(probe_after_secs * 1000),
|
|
98
|
+
"others": list(others),
|
|
99
|
+
}
|
|
100
|
+
reply = await self._record(
|
|
101
|
+
[self.keys.model(alias, "health"), self._terr(alias, dep)], [orjson.dumps(body)]
|
|
102
|
+
)
|
|
103
|
+
return reply.decode() if isinstance(reply, bytes) else str(reply)
|
|
104
|
+
|
|
105
|
+
async def probed(self, alias: str, dep: str, *, ok: bool, probe_max_secs: float) -> str:
|
|
106
|
+
"""A probe's outcome: `recovered` or `waiting` (the wait doubles up to the maximum)."""
|
|
107
|
+
body = {"dep": dep, "ok": ok, "probe_max_ms": int(probe_max_secs * 1000)}
|
|
108
|
+
reply = await self._probed([self.keys.model(alias, "health")], [orjson.dumps(body)])
|
|
109
|
+
return reply.decode() if isinstance(reply, bytes) else str(reply)
|
|
110
|
+
|
|
111
|
+
async def disabled(self, alias: str) -> dict[str, float]:
|
|
112
|
+
"""Disabled deployments of a model and when each is next probed (store time, secs)."""
|
|
113
|
+
raw = await self._client.hgetall(self.keys.model(alias, "health"))
|
|
114
|
+
out: dict[str, float] = {}
|
|
115
|
+
for field, value in (raw or {}).items():
|
|
116
|
+
name = field.decode() if isinstance(field, bytes) else str(field)
|
|
117
|
+
if name.endswith("|until"):
|
|
118
|
+
out[name[: -len("|until")]] = float(value) / 1000.0
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
async def disable(self, alias: str, dep: str, *, probe_after_secs: float) -> None:
|
|
122
|
+
"""Disable a deployment now (tests and operators)."""
|
|
123
|
+
secs, micros = await self._client.time()
|
|
124
|
+
now_ms = int(secs * 1000 + micros / 1000)
|
|
125
|
+
wait = int(probe_after_secs * 1000)
|
|
126
|
+
await self._client.hset(
|
|
127
|
+
self.keys.model(alias, "health"),
|
|
128
|
+
mapping={f"{dep}|until": now_ms + wait, f"{dep}|wait": wait, f"{dep}|since": now_ms},
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
async def snapshot(self, alias: str) -> dict[str, Any]:
|
|
132
|
+
"""The raw health hash of a model (status view)."""
|
|
133
|
+
raw = await self._client.hgetall(self.keys.model(alias, "health"))
|
|
134
|
+
return {
|
|
135
|
+
(k.decode() if isinstance(k, bytes) else str(k)): (
|
|
136
|
+
v.decode() if isinstance(v, bytes) else str(v)
|
|
137
|
+
)
|
|
138
|
+
for k, v in (raw or {}).items()
|
|
139
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Named hooks chosen by name in the registry: balancer, fallback policy, admission, sticky key.
|
|
2
|
+
|
|
3
|
+
Plan §3.10. Hooks are plain functions registered in code; the registry names them (`balance:
|
|
4
|
+
"custom:cheapest"`, `fallback_policy: "office_hours"`). A hook that raises is logged once per
|
|
5
|
+
name and the built-in behaviour is used, so a bug in a hook never fails a call.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from llm_queue._logging import get_logger
|
|
15
|
+
from llm_queue.errors import ConfigError
|
|
16
|
+
|
|
17
|
+
logger = get_logger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True)
|
|
21
|
+
class DeploymentView:
|
|
22
|
+
"""What a custom balancer sees of one candidate deployment."""
|
|
23
|
+
|
|
24
|
+
key: str
|
|
25
|
+
account: str
|
|
26
|
+
region: str
|
|
27
|
+
group: str
|
|
28
|
+
rpm: int | None
|
|
29
|
+
tpm: int | None
|
|
30
|
+
weight: float
|
|
31
|
+
tags: Mapping[str, Any] = field(default_factory=dict)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True, slots=True)
|
|
35
|
+
class CallContext:
|
|
36
|
+
"""What hooks see of a call (no prompt text)."""
|
|
37
|
+
|
|
38
|
+
model: str
|
|
39
|
+
request_id: str
|
|
40
|
+
priority: bool
|
|
41
|
+
session_id: str | None
|
|
42
|
+
user_id: str | None
|
|
43
|
+
tenant_id: str | None
|
|
44
|
+
waited_secs: float = 0.0
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class Reject:
|
|
49
|
+
"""Returned by an admission hook to refuse a call before it is sent."""
|
|
50
|
+
|
|
51
|
+
reason: str
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
Balancer = Callable[[Sequence[DeploymentView], CallContext], Sequence[str]]
|
|
55
|
+
FallbackPolicy = Callable[[str, float, Mapping[str, Any], CallContext], bool]
|
|
56
|
+
AdmissionHook = Callable[[CallContext], "Reject | None"]
|
|
57
|
+
StickyKey = Callable[[CallContext], "str | None"]
|
|
58
|
+
|
|
59
|
+
_HOOKS: dict[str, dict[str, Callable[..., Any]]] = {
|
|
60
|
+
"balancer": {},
|
|
61
|
+
"fallback_policy": {},
|
|
62
|
+
"admission": {},
|
|
63
|
+
"sticky_key": {},
|
|
64
|
+
"probe": {},
|
|
65
|
+
}
|
|
66
|
+
_WARNED: set[tuple[str, str, str]] = set() # (event, kind, name): each event warns once
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _register(kind: str, name: str, fn: Callable[..., Any]) -> None:
|
|
70
|
+
if not name or not callable(fn):
|
|
71
|
+
raise ConfigError(f"{kind} hook needs a name and a callable")
|
|
72
|
+
_HOOKS[kind][name] = fn
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def register_balancer(name: str, fn: Balancer) -> None:
|
|
76
|
+
"""`balance: "custom:<name>"`: fn(candidates, ctx) -> deployment keys, most preferred first."""
|
|
77
|
+
_register("balancer", name, fn)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def register_fallback_policy(name: str, fn: FallbackPolicy) -> None:
|
|
81
|
+
"""`fallback_policy: <name>`: fn(cause, waited_secs, rule, ctx) -> may this call spill now."""
|
|
82
|
+
_register("fallback_policy", name, fn)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def register_admission(name: str, fn: AdmissionHook) -> None:
|
|
86
|
+
"""`admission: <name>`: fn(ctx) -> Reject(reason) to refuse the call, None to admit it."""
|
|
87
|
+
_register("admission", name, fn)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def register_sticky_key(name: str, fn: StickyKey) -> None:
|
|
91
|
+
"""`sticky.key: "custom:<name>"`: fn(ctx) -> the key a call sticks by (None: no sticking)."""
|
|
92
|
+
_register("sticky_key", name, fn)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def register_probe(name: str, fn: Callable[..., Any]) -> None:
|
|
96
|
+
"""`probe: <name>`: the request a health probe sends (step A11)."""
|
|
97
|
+
_register("probe", name, fn)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def get(kind: str, name: str | None) -> Callable[..., Any] | None:
|
|
101
|
+
"""A registered hook, or None (unknown names fall back to the built-in behaviour)."""
|
|
102
|
+
if not name:
|
|
103
|
+
return None
|
|
104
|
+
fn = _HOOKS[kind].get(name)
|
|
105
|
+
if fn is None and ("missing", kind, name) not in _WARNED:
|
|
106
|
+
_WARNED.add(("missing", kind, name))
|
|
107
|
+
logger.warning("hook_not_registered", kind=kind, name=name)
|
|
108
|
+
return fn
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def call_safely(kind: str, name: str, fn: Callable[..., Any], *args: Any, default: Any) -> Any:
|
|
112
|
+
"""Run a hook; on an exception log once per (kind, name) and return `default`."""
|
|
113
|
+
try:
|
|
114
|
+
return fn(*args)
|
|
115
|
+
except Exception as exc: # a hook bug must never fail a call
|
|
116
|
+
if ("failed", kind, name) not in _WARNED:
|
|
117
|
+
_WARNED.add(("failed", kind, name))
|
|
118
|
+
logger.warning("hook_failed", kind=kind, name=name, error=str(exc))
|
|
119
|
+
return default
|
llm_queue/engine/http.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""HTTP views of the engine (plan §3.12, E132, E138): status, requests, registry, health.
|
|
2
|
+
|
|
3
|
+
`Handlers` are plain async functions returning (status code, JSON body), so they are tested
|
|
4
|
+
without a web framework; `status_router()` mounts them on FastAPI (extra `http`). Health routes
|
|
5
|
+
take no auth; the app puts its admin dependency on the rest. Writes (safe fields only) are off
|
|
6
|
+
unless the app passes `allow_writes=True`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
12
|
+
from typing import TYPE_CHECKING, Any
|
|
13
|
+
|
|
14
|
+
from llm_queue.errors import ConfigError, MissingExtra
|
|
15
|
+
from llm_queue.registry.store import LiveEditRefused, audit_diff
|
|
16
|
+
from llm_queue.registry.validate import RegistryError
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from llm_queue.engine.runtime import Engine
|
|
20
|
+
|
|
21
|
+
_REDACT = frozenset({"role_arn", "external_id_ref", "api_key_ref"})
|
|
22
|
+
Response = tuple[int, dict[str, Any]]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def redacted(doc: Any) -> Any:
|
|
26
|
+
"""The registry with role ARNs and secret references hidden."""
|
|
27
|
+
if isinstance(doc, Mapping):
|
|
28
|
+
return {k: "[redacted]" if k in _REDACT else redacted(v) for k, v in doc.items()}
|
|
29
|
+
if isinstance(doc, list):
|
|
30
|
+
return [redacted(v) for v in doc]
|
|
31
|
+
return doc
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Handlers:
|
|
35
|
+
"""The HTTP handlers of one engine."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, engine: Engine, *, allow_writes: bool = False) -> None:
|
|
38
|
+
"""Writes stay refused unless `allow_writes`."""
|
|
39
|
+
self.engine = engine
|
|
40
|
+
self.allow_writes = allow_writes
|
|
41
|
+
|
|
42
|
+
async def live(self) -> Response:
|
|
43
|
+
"""200 while the process runs."""
|
|
44
|
+
return 200, {"live": True}
|
|
45
|
+
|
|
46
|
+
async def ready(self) -> Response:
|
|
47
|
+
"""200 when the registry is loaded and the process is not draining (stores in the body)."""
|
|
48
|
+
health = await self.engine.health_check()
|
|
49
|
+
return (200 if health["ready"] else 503), health
|
|
50
|
+
|
|
51
|
+
async def status(self, model: str | None = None) -> Response:
|
|
52
|
+
"""Status of a model or of everything."""
|
|
53
|
+
try:
|
|
54
|
+
return 200, await self.engine.status(model)
|
|
55
|
+
except KeyError:
|
|
56
|
+
return 404, {"error": f"unknown model {model}"}
|
|
57
|
+
|
|
58
|
+
async def request(self, request_id: str) -> Response:
|
|
59
|
+
"""Status of one call."""
|
|
60
|
+
found = await self.engine.request_status(request_id)
|
|
61
|
+
return (200, found) if found else (404, {"error": "unknown or expired request"})
|
|
62
|
+
|
|
63
|
+
async def registry(self) -> Response:
|
|
64
|
+
"""The live registry, secrets hidden, with what is live but not in config."""
|
|
65
|
+
store = self.engine.registry
|
|
66
|
+
return 200, {
|
|
67
|
+
"registry": redacted(dict(store.current.registry.raw)),
|
|
68
|
+
"changes": store.changes,
|
|
69
|
+
"live_edits": redacted(await store.diff_with_config()),
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async def set(
|
|
73
|
+
self, values: Mapping[str, Any], *, actor: str, model: str | None = None
|
|
74
|
+
) -> Response:
|
|
75
|
+
"""A live edit of safe fields (only with `allow_writes`); audited with `actor`."""
|
|
76
|
+
if not self.allow_writes:
|
|
77
|
+
return 403, {"error": "writes are off"}
|
|
78
|
+
before = dict(self.engine.registry.current.registry.raw)
|
|
79
|
+
try:
|
|
80
|
+
view = await self.engine.registry.set(values, actor=f"http:{actor}", model=model)
|
|
81
|
+
except (LiveEditRefused, RegistryError, ConfigError) as exc:
|
|
82
|
+
return 400, {"error": str(exc)}
|
|
83
|
+
return 200, {"diff": redacted(audit_diff(before, dict(view.registry.raw)))}
|
|
84
|
+
|
|
85
|
+
async def block(
|
|
86
|
+
self, *, actor: str, region: str | None = None, account: str | None = None, on: bool = True
|
|
87
|
+
) -> Response:
|
|
88
|
+
"""Block or unblock a region or account (only with `allow_writes`)."""
|
|
89
|
+
if not self.allow_writes:
|
|
90
|
+
return 403, {"error": "writes are off"}
|
|
91
|
+
try:
|
|
92
|
+
await self.engine.registry.block(
|
|
93
|
+
region=region, account=account, on=on, actor=f"http:{actor}"
|
|
94
|
+
)
|
|
95
|
+
except ConfigError as exc:
|
|
96
|
+
return 400, {"error": str(exc)}
|
|
97
|
+
routing = self.engine.registry.current.registry.routing
|
|
98
|
+
return 200, {
|
|
99
|
+
"blocked_regions": list(routing.blocked_regions),
|
|
100
|
+
"blocked_accounts": list(routing.blocked_accounts),
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def status_router(
|
|
105
|
+
engine: Engine,
|
|
106
|
+
*,
|
|
107
|
+
dependencies: Sequence[Any] = (),
|
|
108
|
+
allow_writes: bool = False,
|
|
109
|
+
actor: Callable[[Any], str] | None = None,
|
|
110
|
+
) -> Any:
|
|
111
|
+
"""A FastAPI `APIRouter` with the handlers (`pip install 'llm-queue[http]'`).
|
|
112
|
+
|
|
113
|
+
`dependencies` (the app's admin check) guard every route except `/health/live` and
|
|
114
|
+
`/health/ready`. Writes need `actor`: it gets the request and returns the authenticated
|
|
115
|
+
name the audit records (never a client-supplied value, which anyone could spoof).
|
|
116
|
+
"""
|
|
117
|
+
try:
|
|
118
|
+
from fastapi import APIRouter, Body, Depends, Request
|
|
119
|
+
from fastapi.responses import JSONResponse
|
|
120
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
121
|
+
raise MissingExtra("http_router", "http") from exc
|
|
122
|
+
# Annotations are strings here (`from __future__ import annotations`); FastAPI resolves them
|
|
123
|
+
# in this module's globals, where the optional import is not, so a `Request` parameter
|
|
124
|
+
# would be read as a query field. Publish the name once it is imported.
|
|
125
|
+
globals()["Request"] = Request
|
|
126
|
+
if allow_writes and actor is None:
|
|
127
|
+
raise ConfigError("allow_writes=True needs actor= (who made the edit, for the audit)")
|
|
128
|
+
|
|
129
|
+
handlers = Handlers(engine, allow_writes=allow_writes)
|
|
130
|
+
|
|
131
|
+
def who(request: Request) -> str:
|
|
132
|
+
return actor(request) if actor is not None else "unknown"
|
|
133
|
+
|
|
134
|
+
router = APIRouter()
|
|
135
|
+
guarded = [Depends(d) if callable(d) else d for d in dependencies]
|
|
136
|
+
|
|
137
|
+
def reply(result: Response) -> Any:
|
|
138
|
+
code, body = result
|
|
139
|
+
return JSONResponse(body, status_code=code)
|
|
140
|
+
|
|
141
|
+
@router.get("/health/live")
|
|
142
|
+
async def live() -> Any:
|
|
143
|
+
return reply(await handlers.live())
|
|
144
|
+
|
|
145
|
+
@router.get("/health/ready")
|
|
146
|
+
async def ready() -> Any:
|
|
147
|
+
return reply(await handlers.ready())
|
|
148
|
+
|
|
149
|
+
@router.get("/status", dependencies=guarded)
|
|
150
|
+
async def status(model: str | None = None) -> Any:
|
|
151
|
+
return reply(await handlers.status(model))
|
|
152
|
+
|
|
153
|
+
@router.get("/requests/{request_id}", dependencies=guarded)
|
|
154
|
+
async def request(request_id: str) -> Any:
|
|
155
|
+
return reply(await handlers.request(request_id))
|
|
156
|
+
|
|
157
|
+
@router.get("/registry", dependencies=guarded)
|
|
158
|
+
async def registry() -> Any:
|
|
159
|
+
return reply(await handlers.registry())
|
|
160
|
+
|
|
161
|
+
@router.post("/registry", dependencies=guarded)
|
|
162
|
+
async def set_values(
|
|
163
|
+
request: Request,
|
|
164
|
+
values: dict[str, Any] = Body(...), # noqa: B008 # FastAPI's dependency style
|
|
165
|
+
model: str | None = None,
|
|
166
|
+
) -> Any:
|
|
167
|
+
return reply(await handlers.set(values, actor=who(request), model=model))
|
|
168
|
+
|
|
169
|
+
@router.post("/registry/block", dependencies=guarded)
|
|
170
|
+
async def block(
|
|
171
|
+
request: Request,
|
|
172
|
+
region: str | None = None,
|
|
173
|
+
account: str | None = None,
|
|
174
|
+
on: bool = True,
|
|
175
|
+
) -> Any:
|
|
176
|
+
return reply(
|
|
177
|
+
await handlers.block(actor=who(request), region=region, account=account, on=on)
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
return router
|