llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,102 @@
1
+ """Message format version 2 (plan E80) and the reply stored for the caller.
2
+
3
+ A call's deadline is not an absolute pod time: the message carries `timeout_secs` and the
4
+ worker measures it from the queue position (the store's clock), so pod clock skew does not
5
+ matter (plan §3.18). Version 2 only: a newer message is returned to the queue, never dropped.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ from typing import Any, Literal
12
+
13
+ import orjson
14
+ from pydantic import BaseModel, ConfigDict, Field
15
+
16
+ from llm_queue.errors import UnsupportedEnvelopeError
17
+
18
+ VERSION = 2
19
+ Op = Literal["converse", "invoke_model", "chat", "embeddings"]
20
+ AllowFallback = Literal["all", "regions", "none"]
21
+
22
+
23
+ class Envelope(BaseModel):
24
+ """One LLM call on the queue. `params` are provider-native and travel inside the message."""
25
+
26
+ model_config = ConfigDict(frozen=True, extra="ignore")
27
+
28
+ v: int = VERSION
29
+ request_id: str
30
+ op: Op
31
+ model: str
32
+ params: dict[str, Any]
33
+ priority: bool = False
34
+ caller: str
35
+ timeout_secs: float = Field(gt=0)
36
+ max_wait_secs: float | None = Field(default=None, gt=0)
37
+ allow_fallback: AllowFallback = "all"
38
+ allowed_regions: tuple[str, ...] | None = None
39
+ allowed_accounts: tuple[str, ...] | None = None
40
+ session_id: str | None = None
41
+ user_id: str | None = None
42
+ tenant_id: str | None = None
43
+ app_request_id: bool = False
44
+ stream: bool = False
45
+
46
+ def dumps(self) -> bytes:
47
+ """JSON bytes for the queue (fields directly: `model_dump` re-walked the params)."""
48
+ return orjson.dumps({k: v for k, v in self.__dict__.items() if v is not None})
49
+
50
+
51
+ def parse(data: bytes | str) -> Envelope:
52
+ """Decode a message; `UnsupportedEnvelopeError` for a newer version (return it, do not drop)."""
53
+ raw = orjson.loads(data)
54
+ if not isinstance(raw, dict) or not isinstance(raw.get("v"), int):
55
+ raise ValueError("not an envelope")
56
+ if raw["v"] > VERSION:
57
+ raise UnsupportedEnvelopeError(f"message version {raw['v']} is newer than {VERSION}")
58
+ if raw["v"] < VERSION:
59
+ raise ValueError(f"message version {raw['v']} is not supported (0.3 reads version 2)")
60
+ return Envelope.model_validate(raw)
61
+
62
+
63
+ def params_hash(params: Any) -> str:
64
+ """Stable hash of a call's params (duplicate request ids must carry the same params)."""
65
+ return hashlib.sha256(orjson.dumps(params, option=orjson.OPT_SORT_KEYS)).hexdigest()[:32]
66
+
67
+
68
+ class Meta(BaseModel):
69
+ """How a call was served: deployment, fallback, attempts, timings, tokens."""
70
+
71
+ model_config = ConfigDict(frozen=True)
72
+
73
+ served_model: str = ""
74
+ deployment: str = ""
75
+ fallback: Literal["region", "model"] | None = None
76
+ attempts: int = 0
77
+ queue_wait_ms: int = 0
78
+ provider_ms: int = 0
79
+ tokens_in: int | None = None
80
+ tokens_out: int | None = None
81
+ worker: str = ""
82
+
83
+
84
+ class Reply(BaseModel):
85
+ """The stored result of a call: the provider reply, or an error kind with its details."""
86
+
87
+ model_config = ConfigDict(frozen=True)
88
+
89
+ request_id: str
90
+ ok: bool
91
+ response: dict[str, Any] | None = None
92
+ error: str | None = None
93
+ message: str = ""
94
+ retry_after_secs: float | None = None
95
+ partial_text: str | None = None
96
+ meta: Meta = Field(default_factory=Meta)
97
+
98
+ def dumps(self) -> bytes:
99
+ """JSON bytes for the store (fields directly: `model_dump` re-walked the response)."""
100
+ out = {k: v for k, v in self.__dict__.items() if v is not None}
101
+ out["meta"] = {k: v for k, v in self.meta.__dict__.items() if v is not None}
102
+ return orjson.dumps(out)
@@ -0,0 +1,51 @@
1
+ """Error kinds a caller can receive (plan §3.19), as one exception type with a `kind`.
2
+
3
+ Callers catch `CallFailed` (an `LLMQueueError`) and branch on `kind`; `retry_after_secs` is set
4
+ where the plan says a later retry can work, `partial_text` for a stream cut mid-answer.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from llm_queue.errors import LLMQueueError
10
+
11
+ KINDS = frozenset(
12
+ {
13
+ "rate_limited",
14
+ "deadline_exceeded",
15
+ "request_too_large",
16
+ "no_deployment",
17
+ "model_unavailable",
18
+ "model_disabled",
19
+ "queue_full",
20
+ "queue_unavailable",
21
+ "credentials_unavailable",
22
+ "retry_budget_exhausted",
23
+ "request_id_conflict",
24
+ "result_expired",
25
+ "dead_lettered",
26
+ "worker_lost",
27
+ "cancelled",
28
+ "provider_error",
29
+ "rejected",
30
+ }
31
+ )
32
+
33
+
34
+ class CallFailed(LLMQueueError):
35
+ """A call ended without a reply; `kind` is one of `KINDS`."""
36
+
37
+ def __init__(
38
+ self,
39
+ kind: str,
40
+ message: str = "",
41
+ *,
42
+ retry_after_secs: float | None = None,
43
+ partial_text: str | None = None,
44
+ request_id: str | None = None,
45
+ ) -> None:
46
+ """Keep the kind and its details."""
47
+ super().__init__(f"{kind}: {message}" if message else kind)
48
+ self.kind = kind
49
+ self.retry_after_secs = retry_after_secs
50
+ self.partial_text = partial_text
51
+ self.request_id = request_id
@@ -0,0 +1,139 @@
1
+ """Deployment health (plan §3.5, §3.18): auto-disable with safeguards, probes, recovery.
2
+
3
+ Target errors (model not in region, access denied) are counted per deployment and request in a
4
+ window; at `disable_errors` the deployment is disabled, unless it is the model's last usable one
5
+ (then only an alert). Errors are counted once the call's outcome is known: a request that hits
6
+ target errors on two deployments is the cause and is never counted. A disabled deployment stays
7
+ out until a probe succeeds; probes start after `probe_after_secs` and back off up to
8
+ `probe_max_secs`. Health lives apart from the registry, so a publish never re-enables one.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from collections.abc import Sequence
14
+ from typing import TYPE_CHECKING, Any
15
+
16
+ import orjson
17
+
18
+ from llm_queue.engine.keys import Keys
19
+
20
+ if TYPE_CHECKING:
21
+ import redis.asyncio
22
+
23
+ _NOW = """
24
+ local t = redis.call('TIME')
25
+ local now = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
26
+ local req = cjson.decode(ARGV[1])
27
+ """
28
+
29
+ # KEYS: health hash, the deployment's target-error set. req: dep, request_id, window_ms, needed,
30
+ # probe_after_ms, others (keys of the model's other deployments).
31
+ # Reply: counted | disabled | last.
32
+ RECORD = (
33
+ _NOW
34
+ + """
35
+ redis.call('ZADD', KEYS[2], now, req.request_id)
36
+ redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now - req.window_ms)
37
+ redis.call('PEXPIRE', KEYS[2], req.window_ms)
38
+ if redis.call('ZCARD', KEYS[2]) < req.needed then return 'counted' end
39
+ local usable_left = 0
40
+ for _, other in ipairs(req.others) do
41
+ if not redis.call('HGET', KEYS[1], other .. '|until') then usable_left = usable_left + 1 end
42
+ end
43
+ if usable_left == 0 then return 'last' end
44
+ redis.call('HSET', KEYS[1], req.dep .. '|until', now + req.probe_after_ms,
45
+ req.dep .. '|wait', req.probe_after_ms, req.dep .. '|since', now)
46
+ redis.call('DEL', KEYS[2])
47
+ return 'disabled'
48
+ """
49
+ )
50
+
51
+ # KEYS: health hash. req: dep, ok (bool), probe_max_ms. Reply: recovered | waiting.
52
+ PROBED = (
53
+ _NOW
54
+ + """
55
+ if req.ok then
56
+ redis.call('HDEL', KEYS[1], req.dep .. '|until', req.dep .. '|wait', req.dep .. '|since')
57
+ return 'recovered'
58
+ end
59
+ local last = tonumber(redis.call('HGET', KEYS[1], req.dep .. '|wait') or '0')
60
+ local wait = math.min(math.max(last * 2, 60000), req.probe_max_ms)
61
+ redis.call('HSET', KEYS[1], req.dep .. '|until', now + wait, req.dep .. '|wait', wait)
62
+ return 'waiting'
63
+ """
64
+ )
65
+
66
+
67
+ class Health:
68
+ """Health records of every model of one environment."""
69
+
70
+ def __init__(self, client: redis.asyncio.Redis, keys: Keys) -> None:
71
+ """Register the scripts; no I/O until the first call."""
72
+ self._client = client
73
+ self.keys = keys
74
+ self._record = client.register_script(RECORD)
75
+ self._probed = client.register_script(PROBED)
76
+
77
+ def _terr(self, alias: str, dep: str) -> str:
78
+ return self.keys.model(alias, f"terr:{dep}")
79
+
80
+ async def record_target_error(
81
+ self,
82
+ alias: str,
83
+ dep: str,
84
+ request_id: str,
85
+ *,
86
+ others: Sequence[str],
87
+ window_secs: float,
88
+ needed: int,
89
+ probe_after_secs: float,
90
+ ) -> str:
91
+ """Count a target error; `disabled`, `last` (would be the last usable) or `counted`."""
92
+ body = {
93
+ "dep": dep,
94
+ "request_id": request_id,
95
+ "window_ms": int(window_secs * 1000),
96
+ "needed": needed,
97
+ "probe_after_ms": int(probe_after_secs * 1000),
98
+ "others": list(others),
99
+ }
100
+ reply = await self._record(
101
+ [self.keys.model(alias, "health"), self._terr(alias, dep)], [orjson.dumps(body)]
102
+ )
103
+ return reply.decode() if isinstance(reply, bytes) else str(reply)
104
+
105
+ async def probed(self, alias: str, dep: str, *, ok: bool, probe_max_secs: float) -> str:
106
+ """A probe's outcome: `recovered` or `waiting` (the wait doubles up to the maximum)."""
107
+ body = {"dep": dep, "ok": ok, "probe_max_ms": int(probe_max_secs * 1000)}
108
+ reply = await self._probed([self.keys.model(alias, "health")], [orjson.dumps(body)])
109
+ return reply.decode() if isinstance(reply, bytes) else str(reply)
110
+
111
+ async def disabled(self, alias: str) -> dict[str, float]:
112
+ """Disabled deployments of a model and when each is next probed (store time, secs)."""
113
+ raw = await self._client.hgetall(self.keys.model(alias, "health"))
114
+ out: dict[str, float] = {}
115
+ for field, value in (raw or {}).items():
116
+ name = field.decode() if isinstance(field, bytes) else str(field)
117
+ if name.endswith("|until"):
118
+ out[name[: -len("|until")]] = float(value) / 1000.0
119
+ return out
120
+
121
+ async def disable(self, alias: str, dep: str, *, probe_after_secs: float) -> None:
122
+ """Disable a deployment now (tests and operators)."""
123
+ secs, micros = await self._client.time()
124
+ now_ms = int(secs * 1000 + micros / 1000)
125
+ wait = int(probe_after_secs * 1000)
126
+ await self._client.hset(
127
+ self.keys.model(alias, "health"),
128
+ mapping={f"{dep}|until": now_ms + wait, f"{dep}|wait": wait, f"{dep}|since": now_ms},
129
+ )
130
+
131
+ async def snapshot(self, alias: str) -> dict[str, Any]:
132
+ """The raw health hash of a model (status view)."""
133
+ raw = await self._client.hgetall(self.keys.model(alias, "health"))
134
+ return {
135
+ (k.decode() if isinstance(k, bytes) else str(k)): (
136
+ v.decode() if isinstance(v, bytes) else str(v)
137
+ )
138
+ for k, v in (raw or {}).items()
139
+ }
@@ -0,0 +1,119 @@
1
+ """Named hooks chosen by name in the registry: balancer, fallback policy, admission, sticky key.
2
+
3
+ Plan §3.10. Hooks are plain functions registered in code; the registry names them (`balance:
4
+ "custom:cheapest"`, `fallback_policy: "office_hours"`). A hook that raises is logged once per
5
+ name and the built-in behaviour is used, so a bug in a hook never fails a call.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Callable, Mapping, Sequence
11
+ from dataclasses import dataclass, field
12
+ from typing import Any
13
+
14
+ from llm_queue._logging import get_logger
15
+ from llm_queue.errors import ConfigError
16
+
17
+ logger = get_logger(__name__)
18
+
19
+
20
+ @dataclass(frozen=True, slots=True)
21
+ class DeploymentView:
22
+ """What a custom balancer sees of one candidate deployment."""
23
+
24
+ key: str
25
+ account: str
26
+ region: str
27
+ group: str
28
+ rpm: int | None
29
+ tpm: int | None
30
+ weight: float
31
+ tags: Mapping[str, Any] = field(default_factory=dict)
32
+
33
+
34
+ @dataclass(frozen=True, slots=True)
35
+ class CallContext:
36
+ """What hooks see of a call (no prompt text)."""
37
+
38
+ model: str
39
+ request_id: str
40
+ priority: bool
41
+ session_id: str | None
42
+ user_id: str | None
43
+ tenant_id: str | None
44
+ waited_secs: float = 0.0
45
+
46
+
47
+ @dataclass(frozen=True, slots=True)
48
+ class Reject:
49
+ """Returned by an admission hook to refuse a call before it is sent."""
50
+
51
+ reason: str
52
+
53
+
54
+ Balancer = Callable[[Sequence[DeploymentView], CallContext], Sequence[str]]
55
+ FallbackPolicy = Callable[[str, float, Mapping[str, Any], CallContext], bool]
56
+ AdmissionHook = Callable[[CallContext], "Reject | None"]
57
+ StickyKey = Callable[[CallContext], "str | None"]
58
+
59
+ _HOOKS: dict[str, dict[str, Callable[..., Any]]] = {
60
+ "balancer": {},
61
+ "fallback_policy": {},
62
+ "admission": {},
63
+ "sticky_key": {},
64
+ "probe": {},
65
+ }
66
+ _WARNED: set[tuple[str, str, str]] = set() # (event, kind, name): each event warns once
67
+
68
+
69
+ def _register(kind: str, name: str, fn: Callable[..., Any]) -> None:
70
+ if not name or not callable(fn):
71
+ raise ConfigError(f"{kind} hook needs a name and a callable")
72
+ _HOOKS[kind][name] = fn
73
+
74
+
75
+ def register_balancer(name: str, fn: Balancer) -> None:
76
+ """`balance: "custom:<name>"`: fn(candidates, ctx) -> deployment keys, most preferred first."""
77
+ _register("balancer", name, fn)
78
+
79
+
80
+ def register_fallback_policy(name: str, fn: FallbackPolicy) -> None:
81
+ """`fallback_policy: <name>`: fn(cause, waited_secs, rule, ctx) -> may this call spill now."""
82
+ _register("fallback_policy", name, fn)
83
+
84
+
85
+ def register_admission(name: str, fn: AdmissionHook) -> None:
86
+ """`admission: <name>`: fn(ctx) -> Reject(reason) to refuse the call, None to admit it."""
87
+ _register("admission", name, fn)
88
+
89
+
90
+ def register_sticky_key(name: str, fn: StickyKey) -> None:
91
+ """`sticky.key: "custom:<name>"`: fn(ctx) -> the key a call sticks by (None: no sticking)."""
92
+ _register("sticky_key", name, fn)
93
+
94
+
95
+ def register_probe(name: str, fn: Callable[..., Any]) -> None:
96
+ """`probe: <name>`: the request a health probe sends (step A11)."""
97
+ _register("probe", name, fn)
98
+
99
+
100
+ def get(kind: str, name: str | None) -> Callable[..., Any] | None:
101
+ """A registered hook, or None (unknown names fall back to the built-in behaviour)."""
102
+ if not name:
103
+ return None
104
+ fn = _HOOKS[kind].get(name)
105
+ if fn is None and ("missing", kind, name) not in _WARNED:
106
+ _WARNED.add(("missing", kind, name))
107
+ logger.warning("hook_not_registered", kind=kind, name=name)
108
+ return fn
109
+
110
+
111
+ def call_safely(kind: str, name: str, fn: Callable[..., Any], *args: Any, default: Any) -> Any:
112
+ """Run a hook; on an exception log once per (kind, name) and return `default`."""
113
+ try:
114
+ return fn(*args)
115
+ except Exception as exc: # a hook bug must never fail a call
116
+ if ("failed", kind, name) not in _WARNED:
117
+ _WARNED.add(("failed", kind, name))
118
+ logger.warning("hook_failed", kind=kind, name=name, error=str(exc))
119
+ return default
@@ -0,0 +1,180 @@
1
+ """HTTP views of the engine (plan §3.12, E132, E138): status, requests, registry, health.
2
+
3
+ `Handlers` are plain async functions returning (status code, JSON body), so they are tested
4
+ without a web framework; `status_router()` mounts them on FastAPI (extra `http`). Health routes
5
+ take no auth; the app puts its admin dependency on the rest. Writes (safe fields only) are off
6
+ unless the app passes `allow_writes=True`.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Callable, Mapping, Sequence
12
+ from typing import TYPE_CHECKING, Any
13
+
14
+ from llm_queue.errors import ConfigError, MissingExtra
15
+ from llm_queue.registry.store import LiveEditRefused, audit_diff
16
+ from llm_queue.registry.validate import RegistryError
17
+
18
+ if TYPE_CHECKING:
19
+ from llm_queue.engine.runtime import Engine
20
+
21
+ _REDACT = frozenset({"role_arn", "external_id_ref", "api_key_ref"})
22
+ Response = tuple[int, dict[str, Any]]
23
+
24
+
25
+ def redacted(doc: Any) -> Any:
26
+ """The registry with role ARNs and secret references hidden."""
27
+ if isinstance(doc, Mapping):
28
+ return {k: "[redacted]" if k in _REDACT else redacted(v) for k, v in doc.items()}
29
+ if isinstance(doc, list):
30
+ return [redacted(v) for v in doc]
31
+ return doc
32
+
33
+
34
+ class Handlers:
35
+ """The HTTP handlers of one engine."""
36
+
37
+ def __init__(self, engine: Engine, *, allow_writes: bool = False) -> None:
38
+ """Writes stay refused unless `allow_writes`."""
39
+ self.engine = engine
40
+ self.allow_writes = allow_writes
41
+
42
+ async def live(self) -> Response:
43
+ """200 while the process runs."""
44
+ return 200, {"live": True}
45
+
46
+ async def ready(self) -> Response:
47
+ """200 when the registry is loaded and the process is not draining (stores in the body)."""
48
+ health = await self.engine.health_check()
49
+ return (200 if health["ready"] else 503), health
50
+
51
+ async def status(self, model: str | None = None) -> Response:
52
+ """Status of a model or of everything."""
53
+ try:
54
+ return 200, await self.engine.status(model)
55
+ except KeyError:
56
+ return 404, {"error": f"unknown model {model}"}
57
+
58
+ async def request(self, request_id: str) -> Response:
59
+ """Status of one call."""
60
+ found = await self.engine.request_status(request_id)
61
+ return (200, found) if found else (404, {"error": "unknown or expired request"})
62
+
63
+ async def registry(self) -> Response:
64
+ """The live registry, secrets hidden, with what is live but not in config."""
65
+ store = self.engine.registry
66
+ return 200, {
67
+ "registry": redacted(dict(store.current.registry.raw)),
68
+ "changes": store.changes,
69
+ "live_edits": redacted(await store.diff_with_config()),
70
+ }
71
+
72
+ async def set(
73
+ self, values: Mapping[str, Any], *, actor: str, model: str | None = None
74
+ ) -> Response:
75
+ """A live edit of safe fields (only with `allow_writes`); audited with `actor`."""
76
+ if not self.allow_writes:
77
+ return 403, {"error": "writes are off"}
78
+ before = dict(self.engine.registry.current.registry.raw)
79
+ try:
80
+ view = await self.engine.registry.set(values, actor=f"http:{actor}", model=model)
81
+ except (LiveEditRefused, RegistryError, ConfigError) as exc:
82
+ return 400, {"error": str(exc)}
83
+ return 200, {"diff": redacted(audit_diff(before, dict(view.registry.raw)))}
84
+
85
+ async def block(
86
+ self, *, actor: str, region: str | None = None, account: str | None = None, on: bool = True
87
+ ) -> Response:
88
+ """Block or unblock a region or account (only with `allow_writes`)."""
89
+ if not self.allow_writes:
90
+ return 403, {"error": "writes are off"}
91
+ try:
92
+ await self.engine.registry.block(
93
+ region=region, account=account, on=on, actor=f"http:{actor}"
94
+ )
95
+ except ConfigError as exc:
96
+ return 400, {"error": str(exc)}
97
+ routing = self.engine.registry.current.registry.routing
98
+ return 200, {
99
+ "blocked_regions": list(routing.blocked_regions),
100
+ "blocked_accounts": list(routing.blocked_accounts),
101
+ }
102
+
103
+
104
+ def status_router(
105
+ engine: Engine,
106
+ *,
107
+ dependencies: Sequence[Any] = (),
108
+ allow_writes: bool = False,
109
+ actor: Callable[[Any], str] | None = None,
110
+ ) -> Any:
111
+ """A FastAPI `APIRouter` with the handlers (`pip install 'llm-queue[http]'`).
112
+
113
+ `dependencies` (the app's admin check) guard every route except `/health/live` and
114
+ `/health/ready`. Writes need `actor`: it gets the request and returns the authenticated
115
+ name the audit records (never a client-supplied value, which anyone could spoof).
116
+ """
117
+ try:
118
+ from fastapi import APIRouter, Body, Depends, Request
119
+ from fastapi.responses import JSONResponse
120
+ except ImportError as exc: # pragma: no cover - exercised only without the extra
121
+ raise MissingExtra("http_router", "http") from exc
122
+ # Annotations are strings here (`from __future__ import annotations`); FastAPI resolves them
123
+ # in this module's globals, where the optional import is not, so a `Request` parameter
124
+ # would be read as a query field. Publish the name once it is imported.
125
+ globals()["Request"] = Request
126
+ if allow_writes and actor is None:
127
+ raise ConfigError("allow_writes=True needs actor= (who made the edit, for the audit)")
128
+
129
+ handlers = Handlers(engine, allow_writes=allow_writes)
130
+
131
+ def who(request: Request) -> str:
132
+ return actor(request) if actor is not None else "unknown"
133
+
134
+ router = APIRouter()
135
+ guarded = [Depends(d) if callable(d) else d for d in dependencies]
136
+
137
+ def reply(result: Response) -> Any:
138
+ code, body = result
139
+ return JSONResponse(body, status_code=code)
140
+
141
+ @router.get("/health/live")
142
+ async def live() -> Any:
143
+ return reply(await handlers.live())
144
+
145
+ @router.get("/health/ready")
146
+ async def ready() -> Any:
147
+ return reply(await handlers.ready())
148
+
149
+ @router.get("/status", dependencies=guarded)
150
+ async def status(model: str | None = None) -> Any:
151
+ return reply(await handlers.status(model))
152
+
153
+ @router.get("/requests/{request_id}", dependencies=guarded)
154
+ async def request(request_id: str) -> Any:
155
+ return reply(await handlers.request(request_id))
156
+
157
+ @router.get("/registry", dependencies=guarded)
158
+ async def registry() -> Any:
159
+ return reply(await handlers.registry())
160
+
161
+ @router.post("/registry", dependencies=guarded)
162
+ async def set_values(
163
+ request: Request,
164
+ values: dict[str, Any] = Body(...), # noqa: B008 # FastAPI's dependency style
165
+ model: str | None = None,
166
+ ) -> Any:
167
+ return reply(await handlers.set(values, actor=who(request), model=model))
168
+
169
+ @router.post("/registry/block", dependencies=guarded)
170
+ async def block(
171
+ request: Request,
172
+ region: str | None = None,
173
+ account: str | None = None,
174
+ on: bool = True,
175
+ ) -> Any:
176
+ return reply(
177
+ await handlers.block(actor=who(request), region=region, account=account, on=on)
178
+ )
179
+
180
+ return router