llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
llm_queue/core.py
ADDED
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
"""`LLMQueue`: the library's front door (plan §3.15).
|
|
2
|
+
|
|
3
|
+
Built from the environment (`from_env`: stores, registry, clients) or from parts. `start()`
|
|
4
|
+
publishes / loads the registry and runs the worker side in this process; `converse`, `chat` and
|
|
5
|
+
`embed` send calls through the model's queue; `*_sync` run on the direct path in a private
|
|
6
|
+
thread (E135); jobs, cancel, status, health and the registry are here too.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import contextlib
|
|
13
|
+
import socket
|
|
14
|
+
import threading
|
|
15
|
+
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping, Sequence
|
|
16
|
+
from concurrent.futures import Future
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Any, TypeVar
|
|
19
|
+
|
|
20
|
+
import orjson
|
|
21
|
+
import redis.asyncio
|
|
22
|
+
from redis.credentials import CredentialProvider
|
|
23
|
+
|
|
24
|
+
from llm_queue._logging import get_logger
|
|
25
|
+
from llm_queue.clients import ClientPool, Providers
|
|
26
|
+
from llm_queue.context import current_context
|
|
27
|
+
from llm_queue.engine.envelope import Meta, Reply
|
|
28
|
+
from llm_queue.engine.jobs import JobFn, JobRunner
|
|
29
|
+
from llm_queue.engine.metrics import Metrics, sink_for
|
|
30
|
+
from llm_queue.engine.offload import ResultOffload
|
|
31
|
+
from llm_queue.engine.runtime import Engine, EngineSettings
|
|
32
|
+
from llm_queue.engine.settings import InfraSettings
|
|
33
|
+
from llm_queue.engine.store_client import store_client
|
|
34
|
+
from llm_queue.engine.store_guard import store_op
|
|
35
|
+
from llm_queue.errors import ConfigError, LLMQueueError
|
|
36
|
+
from llm_queue.registry.store import RegistryStore
|
|
37
|
+
from llm_queue.tuning import StoreOptions, Tuning
|
|
38
|
+
|
|
39
|
+
logger = get_logger(__name__)
|
|
40
|
+
_T = TypeVar("_T")
|
|
41
|
+
Secrets = Callable[[str], "str | None"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True, slots=True)
|
|
45
|
+
class EmbedResult:
|
|
46
|
+
"""Vectors (one per text, in order) and how the batches were served."""
|
|
47
|
+
|
|
48
|
+
vectors: list[list[float]]
|
|
49
|
+
meta: Meta
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _FreshPassword(CredentialProvider):
|
|
53
|
+
"""Reads the store password from the secret on every new connection (rotation, R42)."""
|
|
54
|
+
|
|
55
|
+
def __init__(self, secrets: Secrets, name: str, username: str | None = None) -> None:
|
|
56
|
+
self._secrets = secrets
|
|
57
|
+
self._name = name
|
|
58
|
+
self._username = username
|
|
59
|
+
|
|
60
|
+
def get_credentials(self) -> tuple[str] | tuple[str, str]:
|
|
61
|
+
password = self._secrets(self._name) or ""
|
|
62
|
+
return (self._username, password) if self._username else (password,)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _client(
|
|
66
|
+
url: str, secrets: Secrets | None, name: str, max_connections: int, options: StoreOptions
|
|
67
|
+
) -> redis.asyncio.Redis:
|
|
68
|
+
"""An async client (bytes replies) for a store URL; a password secret wins when present."""
|
|
69
|
+
provider = _FreshPassword(secrets, name) if secrets is not None and secrets(name) else None
|
|
70
|
+
return store_client(url, provider, max_connections=max_connections, options=options)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class LLMQueue:
|
|
74
|
+
"""One process's LLM queue: caller, worker, jobs and admin over the stores."""
|
|
75
|
+
|
|
76
|
+
def __init__(
|
|
77
|
+
self,
|
|
78
|
+
engine: Engine,
|
|
79
|
+
*,
|
|
80
|
+
settings: InfraSettings | None = None,
|
|
81
|
+
builder: Callable[[], Engine] | None = None,
|
|
82
|
+
) -> None:
|
|
83
|
+
"""Wrap a built `Engine`; `builder` makes a second, caller-only engine for sync calls."""
|
|
84
|
+
self.engine = engine
|
|
85
|
+
self.settings = settings
|
|
86
|
+
self.jobs = JobRunner(engine.queues, queue_prefix=engine.settings.queue_prefix)
|
|
87
|
+
self._builder = builder
|
|
88
|
+
self._sync: _SyncRunner | None = None
|
|
89
|
+
self._sync_lock = threading.Lock()
|
|
90
|
+
self.started = False
|
|
91
|
+
self._stopped = False
|
|
92
|
+
|
|
93
|
+
# ---------------------------------------------------------------- construction
|
|
94
|
+
|
|
95
|
+
@classmethod
|
|
96
|
+
def from_env(
|
|
97
|
+
cls,
|
|
98
|
+
environ: Mapping[str, str] | None = None,
|
|
99
|
+
*,
|
|
100
|
+
secrets: Secrets | None = None,
|
|
101
|
+
metrics: Metrics | None = None,
|
|
102
|
+
worker_id: str | None = None,
|
|
103
|
+
credentials: Callable[[Any], Mapping[str, Any] | None] | None = None,
|
|
104
|
+
offload: ResultOffload | None = None,
|
|
105
|
+
offload_bytes: int | None = None,
|
|
106
|
+
tuning: Tuning | None = None,
|
|
107
|
+
) -> LLMQueue:
|
|
108
|
+
"""Build from `LLMQ_*` (secret first when `secrets` is given, then the environment).
|
|
109
|
+
|
|
110
|
+
`tuning` sets timeouts, pool sizes and loop pacing in code (`Tuning()` is the
|
|
111
|
+
defaults); a bad value or combination raises `ConfigError` listing every problem.
|
|
112
|
+
"""
|
|
113
|
+
settings = InfraSettings.from_env(environ, secrets=secrets)
|
|
114
|
+
tuning = tuning or Tuning()
|
|
115
|
+
if problems := tuning.problems(queue_wait_ms=settings.queue_wait_ms):
|
|
116
|
+
raise ConfigError("invalid tuning:\n " + "\n ".join(problems))
|
|
117
|
+
|
|
118
|
+
def build() -> Engine:
|
|
119
|
+
cap = settings.store_max_connections
|
|
120
|
+
state = _client(settings.store_url, secrets, "LLMQ_STORE_PASSWORD", cap, tuning.store)
|
|
121
|
+
queue_store = (
|
|
122
|
+
_client(
|
|
123
|
+
settings.queue_store_url,
|
|
124
|
+
secrets,
|
|
125
|
+
"LLMQ_QUEUE_STORE_PASSWORD",
|
|
126
|
+
cap,
|
|
127
|
+
tuning.store,
|
|
128
|
+
)
|
|
129
|
+
if settings.queue_store_url
|
|
130
|
+
else None
|
|
131
|
+
)
|
|
132
|
+
registry = RegistryStore(
|
|
133
|
+
state,
|
|
134
|
+
settings.key_prefix,
|
|
135
|
+
config=orjson.loads(settings.registry_document()),
|
|
136
|
+
live_edit=settings.registry_live_edit,
|
|
137
|
+
allowed_hosts=settings.base_url_allowed_hosts,
|
|
138
|
+
audit_retention_days=settings.audit_retention_days,
|
|
139
|
+
history_size=tuning.registry.history_size,
|
|
140
|
+
edit_retries=tuning.registry.edit_retries,
|
|
141
|
+
)
|
|
142
|
+
pool_kwargs: dict[str, Any] = {
|
|
143
|
+
"bedrock_endpoint_url": settings.bedrock_endpoint_url,
|
|
144
|
+
"sts_endpoint_url": settings.sts_endpoint_url,
|
|
145
|
+
"session_name": f"llmq-{settings.environment}-{worker_id or socket.gethostname()}",
|
|
146
|
+
"options": tuning.bedrock,
|
|
147
|
+
}
|
|
148
|
+
if credentials is not None:
|
|
149
|
+
pool_kwargs["credentials"] = credentials
|
|
150
|
+
if secrets is not None:
|
|
151
|
+
pool_kwargs["secrets"] = secrets
|
|
152
|
+
engine_settings = EngineSettings(
|
|
153
|
+
queue_prefix=settings.queue_prefix,
|
|
154
|
+
lease_secs=settings.lease_secs,
|
|
155
|
+
result_ttl_secs=settings.result_ttl_secs,
|
|
156
|
+
result_dedupe_secs=settings.result_dedupe_secs,
|
|
157
|
+
status_ttl_secs=settings.status_ttl_secs,
|
|
158
|
+
caller_ttl_secs=settings.caller_ttl_secs,
|
|
159
|
+
drain_secs=settings.drain_secs,
|
|
160
|
+
local_limit_cap=settings.local_limit_cap,
|
|
161
|
+
id_hash_salt=settings.id_hash_salt or "",
|
|
162
|
+
cfg_reload_secs=settings.cfg_reload_secs,
|
|
163
|
+
queue_failover_secs=settings.queue_failover_secs,
|
|
164
|
+
queue_wait_replicas=settings.queue_wait_replicas,
|
|
165
|
+
queue_wait_ms=settings.queue_wait_ms,
|
|
166
|
+
debug_capture_allowed=settings.debug_capture_from_config,
|
|
167
|
+
timings=tuning.engine,
|
|
168
|
+
)
|
|
169
|
+
return Engine(
|
|
170
|
+
state,
|
|
171
|
+
registry,
|
|
172
|
+
Providers(ClientPool(**pool_kwargs)),
|
|
173
|
+
queue_store=queue_store,
|
|
174
|
+
settings=engine_settings,
|
|
175
|
+
worker_id=worker_id,
|
|
176
|
+
metrics=metrics or sink_for(settings.metrics_sink),
|
|
177
|
+
offload=offload,
|
|
178
|
+
offload_bytes=offload_bytes,
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
return cls(build(), settings=settings, builder=build)
|
|
182
|
+
|
|
183
|
+
# ---------------------------------------------------------------- lifecycle
|
|
184
|
+
|
|
185
|
+
async def start(self) -> None:
|
|
186
|
+
"""Publish / load the registry; run the worker side unless `mode=caller`.
|
|
187
|
+
|
|
188
|
+
A second `start()` does nothing; a stopped queue cannot start again (build a new one).
|
|
189
|
+
"""
|
|
190
|
+
if self.started:
|
|
191
|
+
return
|
|
192
|
+
if self._stopped:
|
|
193
|
+
raise LLMQueueError("this LLMQueue was stopped; build a new one to start again")
|
|
194
|
+
mode = self.settings.mode if self.settings else "both"
|
|
195
|
+
await self.engine.start(
|
|
196
|
+
worker=mode in ("both", "worker"), caller=mode in ("both", "caller")
|
|
197
|
+
)
|
|
198
|
+
if mode in ("both", "worker"):
|
|
199
|
+
await self.jobs.start()
|
|
200
|
+
from llm_queue.langchain import set_default_queue
|
|
201
|
+
|
|
202
|
+
set_default_queue(self)
|
|
203
|
+
self.started = True
|
|
204
|
+
|
|
205
|
+
def pause_receiving(self) -> None:
|
|
206
|
+
"""Fail readiness and take no new calls or jobs; running work goes on (preStop hook)."""
|
|
207
|
+
self.engine.pause_receiving()
|
|
208
|
+
self.jobs.pause()
|
|
209
|
+
|
|
210
|
+
async def stop(self, drain_secs: float | None = None) -> None:
|
|
211
|
+
"""Shut down in order (plan §3.18) and close every client; a second call does nothing.
|
|
212
|
+
|
|
213
|
+
`drain_secs` (default `LLMQ_DRAIN_SECS`) bounds the whole wait: jobs first (they make
|
|
214
|
+
calls), then running calls get what is left of it.
|
|
215
|
+
"""
|
|
216
|
+
from llm_queue.langchain import get_default_queue, set_default_queue
|
|
217
|
+
|
|
218
|
+
if self._stopped:
|
|
219
|
+
return
|
|
220
|
+
self._stopped = True
|
|
221
|
+
with contextlib.suppress(LLMQueueError):
|
|
222
|
+
if get_default_queue() is self:
|
|
223
|
+
set_default_queue(None)
|
|
224
|
+
self.started = False
|
|
225
|
+
loop = asyncio.get_running_loop()
|
|
226
|
+
drain = self.engine.settings.drain_secs if drain_secs is None else drain_secs
|
|
227
|
+
until = loop.time() + drain
|
|
228
|
+
await self.jobs.stop(drain_secs=drain)
|
|
229
|
+
await self.engine.stop(drain_secs=max(0.0, until - loop.time()))
|
|
230
|
+
await asyncio.to_thread(self.close_sync) # blocking: off the event loop
|
|
231
|
+
await self.engine.aclose()
|
|
232
|
+
|
|
233
|
+
# ---------------------------------------------------------------- calls
|
|
234
|
+
|
|
235
|
+
def _call_kwargs(self, priority: bool | None, **kw: Any) -> dict[str, Any]:
|
|
236
|
+
ctx = current_context()
|
|
237
|
+
out = {
|
|
238
|
+
"priority": ctx.priority if priority is None else priority,
|
|
239
|
+
"session_id": kw.pop("session_id", None) or ctx.session_id,
|
|
240
|
+
"user_id": kw.pop("user_id", None) or ctx.user_id,
|
|
241
|
+
"tenant_id": kw.pop("tenant_id", None) or ctx.tenant_id,
|
|
242
|
+
}
|
|
243
|
+
out.update({k: v for k, v in kw.items() if v is not None})
|
|
244
|
+
return out
|
|
245
|
+
|
|
246
|
+
async def call(
|
|
247
|
+
self, model: str, params: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
248
|
+
) -> Reply:
|
|
249
|
+
"""Any operation of a model (`converse`, `chat`, `invoke_model`, `embeddings`)."""
|
|
250
|
+
return await self.engine.call(model, params, **self._call_kwargs(priority, **kw))
|
|
251
|
+
|
|
252
|
+
async def converse(
|
|
253
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
254
|
+
) -> Reply:
|
|
255
|
+
"""One Bedrock Converse call (the reply's `response` is the Converse body)."""
|
|
256
|
+
return await self.call(model, request, priority=priority, **kw)
|
|
257
|
+
|
|
258
|
+
async def chat(
|
|
259
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
260
|
+
) -> Reply:
|
|
261
|
+
"""One OpenAI-compatible chat completion."""
|
|
262
|
+
return await self.call(model, request, priority=priority, **kw)
|
|
263
|
+
|
|
264
|
+
async def converse_stream(
|
|
265
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
266
|
+
) -> AsyncIterator[dict[str, Any]]:
|
|
267
|
+
"""Stream a chat call's events as they arrive (plan §3.13).
|
|
268
|
+
|
|
269
|
+
Bedrock models yield ConverseStream events, OpenAI-compatible ones chat completion
|
|
270
|
+
chunks. A failure after events were yielded raises `CallFailed` with `partial_text`.
|
|
271
|
+
"""
|
|
272
|
+
kwargs = self._call_kwargs(priority, **kw)
|
|
273
|
+
async for event in self.engine.stream(model, request, **kwargs):
|
|
274
|
+
yield event
|
|
275
|
+
|
|
276
|
+
chat_stream = converse_stream
|
|
277
|
+
|
|
278
|
+
async def embed(
|
|
279
|
+
self,
|
|
280
|
+
model: str,
|
|
281
|
+
texts: Sequence[str],
|
|
282
|
+
*,
|
|
283
|
+
priority: bool | None = None,
|
|
284
|
+
input_type: str | None = None,
|
|
285
|
+
**kw: Any,
|
|
286
|
+
) -> EmbedResult:
|
|
287
|
+
"""One vector per text, in order (batched by the model's `embed_batch_size`)."""
|
|
288
|
+
kwargs = self._call_kwargs(priority, **kw)
|
|
289
|
+
vectors = await self.engine.embed(model, texts, input_type=input_type, **kwargs)
|
|
290
|
+
alias = self.engine.registry.current.registry.alias_of(model)
|
|
291
|
+
return EmbedResult(vectors, Meta(served_model=alias))
|
|
292
|
+
|
|
293
|
+
# ---------------------------------------------------------------- sync (direct path)
|
|
294
|
+
|
|
295
|
+
def _sync_runner(self) -> _SyncRunner:
|
|
296
|
+
if threading.current_thread().name == _SyncRunner.THREAD_NAME:
|
|
297
|
+
raise LLMQueueError("a sync call from inside the sync runner would deadlock")
|
|
298
|
+
_warn_if_loop_running()
|
|
299
|
+
runner = self._sync
|
|
300
|
+
if runner is None:
|
|
301
|
+
if self._builder is None:
|
|
302
|
+
raise LLMQueueError("sync calls need LLMQueue.from_env (a second, private engine)")
|
|
303
|
+
with self._sync_lock: # threads making their first sync call together build one
|
|
304
|
+
if self._sync is None:
|
|
305
|
+
self._sync = _SyncRunner(self._builder)
|
|
306
|
+
runner = self._sync
|
|
307
|
+
return runner
|
|
308
|
+
|
|
309
|
+
def converse_sync(
|
|
310
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
311
|
+
) -> Reply:
|
|
312
|
+
"""`converse` on the direct path from sync code (same budgets, no queue)."""
|
|
313
|
+
kwargs = self._call_kwargs(priority, **kw)
|
|
314
|
+
return self._sync_runner().run(
|
|
315
|
+
lambda e: e.call(model, request, force_direct=True, **kwargs)
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
def chat_sync(
|
|
319
|
+
self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
|
|
320
|
+
) -> Reply:
|
|
321
|
+
"""`chat` on the direct path from sync code."""
|
|
322
|
+
return self.converse_sync(model, request, priority=priority, **kw)
|
|
323
|
+
|
|
324
|
+
def embed_sync(
|
|
325
|
+
self,
|
|
326
|
+
model: str,
|
|
327
|
+
texts: Sequence[str],
|
|
328
|
+
*,
|
|
329
|
+
priority: bool | None = None,
|
|
330
|
+
input_type: str | None = None,
|
|
331
|
+
**kw: Any,
|
|
332
|
+
) -> EmbedResult:
|
|
333
|
+
"""`embed` on the direct path from sync code."""
|
|
334
|
+
kwargs = self._call_kwargs(priority, **kw)
|
|
335
|
+
|
|
336
|
+
async def run(engine: Engine) -> EmbedResult:
|
|
337
|
+
vectors = await engine.embed(
|
|
338
|
+
model, texts, input_type=input_type, force_direct=True, **kwargs
|
|
339
|
+
)
|
|
340
|
+
alias = engine.registry.current.registry.alias_of(model)
|
|
341
|
+
return EmbedResult(vectors, Meta(served_model=alias))
|
|
342
|
+
|
|
343
|
+
return self._sync_runner().run(run)
|
|
344
|
+
|
|
345
|
+
def close_sync(self) -> None:
|
|
346
|
+
"""Stop the private engine of the sync calls (sync-only processes; `stop()` does too)."""
|
|
347
|
+
with self._sync_lock:
|
|
348
|
+
runner, self._sync = self._sync, None
|
|
349
|
+
if runner is not None:
|
|
350
|
+
runner.close()
|
|
351
|
+
|
|
352
|
+
# ---------------------------------------------------------------- jobs and admin
|
|
353
|
+
|
|
354
|
+
def register_job(self, name: str, fn: JobFn) -> None:
|
|
355
|
+
"""Allow this process to run job `name` (call before `start`)."""
|
|
356
|
+
self.jobs.register(name, fn)
|
|
357
|
+
|
|
358
|
+
@store_op
|
|
359
|
+
async def send_job(
|
|
360
|
+
self, name: str, payload: Mapping[str, Any], *, job_id: str | None = None
|
|
361
|
+
) -> str:
|
|
362
|
+
"""Queue a job; returns its id (jobs run at least once)."""
|
|
363
|
+
return await self.jobs.send(name, payload, job_id=job_id)
|
|
364
|
+
|
|
365
|
+
@store_op
|
|
366
|
+
async def cancel(self, request_id: str) -> None:
|
|
367
|
+
"""Cancel a call (waiting or running)."""
|
|
368
|
+
await self.engine.cancel(request_id)
|
|
369
|
+
|
|
370
|
+
@store_op
|
|
371
|
+
async def status(self, model: str | None = None) -> dict[str, Any]:
|
|
372
|
+
"""Status of a model or of everything."""
|
|
373
|
+
return await self.engine.status(model)
|
|
374
|
+
|
|
375
|
+
@store_op
|
|
376
|
+
async def request_status(self, request_id: str) -> dict[str, Any]:
|
|
377
|
+
"""Status of one call."""
|
|
378
|
+
return await self.engine.request_status(request_id)
|
|
379
|
+
|
|
380
|
+
async def health(self) -> dict[str, Any]:
|
|
381
|
+
"""Liveness, readiness and the stores' state."""
|
|
382
|
+
return await self.engine.health_check()
|
|
383
|
+
|
|
384
|
+
@property
|
|
385
|
+
def registry(self) -> RegistryStore:
|
|
386
|
+
"""The registry store (live edits, audit, rollback)."""
|
|
387
|
+
return self.engine.registry
|
|
388
|
+
|
|
389
|
+
def http_router(
|
|
390
|
+
self,
|
|
391
|
+
*,
|
|
392
|
+
dependencies: Sequence[Any] = (),
|
|
393
|
+
allow_writes: bool = False,
|
|
394
|
+
actor: Callable[[Any], str] | None = None,
|
|
395
|
+
) -> Any:
|
|
396
|
+
"""A FastAPI router with status, requests, registry and health (extra `http`).
|
|
397
|
+
|
|
398
|
+
Writes need `actor(request) -> str`, the authenticated name recorded in the audit.
|
|
399
|
+
"""
|
|
400
|
+
from llm_queue.engine.http import status_router
|
|
401
|
+
|
|
402
|
+
return status_router(
|
|
403
|
+
self.engine, dependencies=dependencies, allow_writes=allow_writes, actor=actor
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
class _SyncRunner:
|
|
408
|
+
"""A private event loop in a thread with its own caller-only engine, for sync calls."""
|
|
409
|
+
|
|
410
|
+
THREAD_NAME = "llmq-sync"
|
|
411
|
+
|
|
412
|
+
def __init__(self, build: Callable[[], Engine]) -> None:
|
|
413
|
+
self._loop = asyncio.new_event_loop()
|
|
414
|
+
self._thread = threading.Thread(
|
|
415
|
+
target=self._loop.run_forever, name=self.THREAD_NAME, daemon=True
|
|
416
|
+
)
|
|
417
|
+
self._thread.start()
|
|
418
|
+
self._closing = False
|
|
419
|
+
|
|
420
|
+
async def boot() -> Engine:
|
|
421
|
+
engine = build()
|
|
422
|
+
await engine.start(worker=False, caller=True)
|
|
423
|
+
return engine
|
|
424
|
+
|
|
425
|
+
try:
|
|
426
|
+
self._engine: Engine = self._submit(boot()).result()
|
|
427
|
+
except BaseException:
|
|
428
|
+
self._loop.call_soon_threadsafe(self._loop.stop) # no thread left behind
|
|
429
|
+
self._thread.join(timeout=5)
|
|
430
|
+
self._loop.close()
|
|
431
|
+
raise
|
|
432
|
+
|
|
433
|
+
def _submit(self, coro: Coroutine[Any, Any, _T]) -> Future[_T]:
|
|
434
|
+
return asyncio.run_coroutine_threadsafe(coro, self._loop)
|
|
435
|
+
|
|
436
|
+
def run(self, fn: Callable[[Engine], Coroutine[Any, Any, _T]]) -> _T:
|
|
437
|
+
"""Run one call on the private loop and wait for it."""
|
|
438
|
+
if self._closing:
|
|
439
|
+
raise LLMQueueError("the sync runner is closed")
|
|
440
|
+
return self._submit(fn(self._engine)).result()
|
|
441
|
+
|
|
442
|
+
def close(self) -> None:
|
|
443
|
+
"""Stop the private engine and loop; calls still running are cancelled.
|
|
444
|
+
|
|
445
|
+
Their callers get the error: before, a call running at close never returned.
|
|
446
|
+
"""
|
|
447
|
+
self._closing = True
|
|
448
|
+
|
|
449
|
+
async def stop() -> None:
|
|
450
|
+
await self._engine.stop(drain_secs=1)
|
|
451
|
+
current = asyncio.current_task()
|
|
452
|
+
rest = [t for t in asyncio.all_tasks() if t is not current]
|
|
453
|
+
for task in rest:
|
|
454
|
+
task.cancel()
|
|
455
|
+
await asyncio.gather(*rest, return_exceptions=True)
|
|
456
|
+
await self._engine.aclose()
|
|
457
|
+
|
|
458
|
+
with contextlib.suppress(Exception):
|
|
459
|
+
self._submit(stop()).result(timeout=10)
|
|
460
|
+
self._loop.call_soon_threadsafe(self._loop.stop)
|
|
461
|
+
self._thread.join(timeout=5)
|
|
462
|
+
if not self._thread.is_alive():
|
|
463
|
+
self._loop.close()
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
_warned_loop = False
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _warn_if_loop_running() -> None:
|
|
470
|
+
"""Warn once when a sync call blocks a running event loop (use the async methods there)."""
|
|
471
|
+
global _warned_loop
|
|
472
|
+
if _warned_loop:
|
|
473
|
+
return
|
|
474
|
+
try:
|
|
475
|
+
asyncio.get_running_loop()
|
|
476
|
+
except RuntimeError:
|
|
477
|
+
return
|
|
478
|
+
_warned_loop = True
|
|
479
|
+
logger.warning(
|
|
480
|
+
"sync_call_on_event_loop",
|
|
481
|
+
hint="converse_sync / embed_sync block this event loop; await converse / embed instead",
|
|
482
|
+
)
|