llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
llm_queue/core.py ADDED
@@ -0,0 +1,482 @@
1
+ """`LLMQueue`: the library's front door (plan §3.15).
2
+
3
+ Built from the environment (`from_env`: stores, registry, clients) or from parts. `start()`
4
+ publishes / loads the registry and runs the worker side in this process; `converse`, `chat` and
5
+ `embed` send calls through the model's queue; `*_sync` run on the direct path in a private
6
+ thread (E135); jobs, cancel, status, health and the registry are here too.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import contextlib
13
+ import socket
14
+ import threading
15
+ from collections.abc import AsyncIterator, Callable, Coroutine, Mapping, Sequence
16
+ from concurrent.futures import Future
17
+ from dataclasses import dataclass
18
+ from typing import Any, TypeVar
19
+
20
+ import orjson
21
+ import redis.asyncio
22
+ from redis.credentials import CredentialProvider
23
+
24
+ from llm_queue._logging import get_logger
25
+ from llm_queue.clients import ClientPool, Providers
26
+ from llm_queue.context import current_context
27
+ from llm_queue.engine.envelope import Meta, Reply
28
+ from llm_queue.engine.jobs import JobFn, JobRunner
29
+ from llm_queue.engine.metrics import Metrics, sink_for
30
+ from llm_queue.engine.offload import ResultOffload
31
+ from llm_queue.engine.runtime import Engine, EngineSettings
32
+ from llm_queue.engine.settings import InfraSettings
33
+ from llm_queue.engine.store_client import store_client
34
+ from llm_queue.engine.store_guard import store_op
35
+ from llm_queue.errors import ConfigError, LLMQueueError
36
+ from llm_queue.registry.store import RegistryStore
37
+ from llm_queue.tuning import StoreOptions, Tuning
38
+
39
+ logger = get_logger(__name__)
40
+ _T = TypeVar("_T")
41
+ Secrets = Callable[[str], "str | None"]
42
+
43
+
44
+ @dataclass(frozen=True, slots=True)
45
+ class EmbedResult:
46
+ """Vectors (one per text, in order) and how the batches were served."""
47
+
48
+ vectors: list[list[float]]
49
+ meta: Meta
50
+
51
+
52
+ class _FreshPassword(CredentialProvider):
53
+ """Reads the store password from the secret on every new connection (rotation, R42)."""
54
+
55
+ def __init__(self, secrets: Secrets, name: str, username: str | None = None) -> None:
56
+ self._secrets = secrets
57
+ self._name = name
58
+ self._username = username
59
+
60
+ def get_credentials(self) -> tuple[str] | tuple[str, str]:
61
+ password = self._secrets(self._name) or ""
62
+ return (self._username, password) if self._username else (password,)
63
+
64
+
65
+ def _client(
66
+ url: str, secrets: Secrets | None, name: str, max_connections: int, options: StoreOptions
67
+ ) -> redis.asyncio.Redis:
68
+ """An async client (bytes replies) for a store URL; a password secret wins when present."""
69
+ provider = _FreshPassword(secrets, name) if secrets is not None and secrets(name) else None
70
+ return store_client(url, provider, max_connections=max_connections, options=options)
71
+
72
+
73
+ class LLMQueue:
74
+ """One process's LLM queue: caller, worker, jobs and admin over the stores."""
75
+
76
+ def __init__(
77
+ self,
78
+ engine: Engine,
79
+ *,
80
+ settings: InfraSettings | None = None,
81
+ builder: Callable[[], Engine] | None = None,
82
+ ) -> None:
83
+ """Wrap a built `Engine`; `builder` makes a second, caller-only engine for sync calls."""
84
+ self.engine = engine
85
+ self.settings = settings
86
+ self.jobs = JobRunner(engine.queues, queue_prefix=engine.settings.queue_prefix)
87
+ self._builder = builder
88
+ self._sync: _SyncRunner | None = None
89
+ self._sync_lock = threading.Lock()
90
+ self.started = False
91
+ self._stopped = False
92
+
93
+ # ---------------------------------------------------------------- construction
94
+
95
+ @classmethod
96
+ def from_env(
97
+ cls,
98
+ environ: Mapping[str, str] | None = None,
99
+ *,
100
+ secrets: Secrets | None = None,
101
+ metrics: Metrics | None = None,
102
+ worker_id: str | None = None,
103
+ credentials: Callable[[Any], Mapping[str, Any] | None] | None = None,
104
+ offload: ResultOffload | None = None,
105
+ offload_bytes: int | None = None,
106
+ tuning: Tuning | None = None,
107
+ ) -> LLMQueue:
108
+ """Build from `LLMQ_*` (secret first when `secrets` is given, then the environment).
109
+
110
+ `tuning` sets timeouts, pool sizes and loop pacing in code (`Tuning()` is the
111
+ defaults); a bad value or combination raises `ConfigError` listing every problem.
112
+ """
113
+ settings = InfraSettings.from_env(environ, secrets=secrets)
114
+ tuning = tuning or Tuning()
115
+ if problems := tuning.problems(queue_wait_ms=settings.queue_wait_ms):
116
+ raise ConfigError("invalid tuning:\n " + "\n ".join(problems))
117
+
118
+ def build() -> Engine:
119
+ cap = settings.store_max_connections
120
+ state = _client(settings.store_url, secrets, "LLMQ_STORE_PASSWORD", cap, tuning.store)
121
+ queue_store = (
122
+ _client(
123
+ settings.queue_store_url,
124
+ secrets,
125
+ "LLMQ_QUEUE_STORE_PASSWORD",
126
+ cap,
127
+ tuning.store,
128
+ )
129
+ if settings.queue_store_url
130
+ else None
131
+ )
132
+ registry = RegistryStore(
133
+ state,
134
+ settings.key_prefix,
135
+ config=orjson.loads(settings.registry_document()),
136
+ live_edit=settings.registry_live_edit,
137
+ allowed_hosts=settings.base_url_allowed_hosts,
138
+ audit_retention_days=settings.audit_retention_days,
139
+ history_size=tuning.registry.history_size,
140
+ edit_retries=tuning.registry.edit_retries,
141
+ )
142
+ pool_kwargs: dict[str, Any] = {
143
+ "bedrock_endpoint_url": settings.bedrock_endpoint_url,
144
+ "sts_endpoint_url": settings.sts_endpoint_url,
145
+ "session_name": f"llmq-{settings.environment}-{worker_id or socket.gethostname()}",
146
+ "options": tuning.bedrock,
147
+ }
148
+ if credentials is not None:
149
+ pool_kwargs["credentials"] = credentials
150
+ if secrets is not None:
151
+ pool_kwargs["secrets"] = secrets
152
+ engine_settings = EngineSettings(
153
+ queue_prefix=settings.queue_prefix,
154
+ lease_secs=settings.lease_secs,
155
+ result_ttl_secs=settings.result_ttl_secs,
156
+ result_dedupe_secs=settings.result_dedupe_secs,
157
+ status_ttl_secs=settings.status_ttl_secs,
158
+ caller_ttl_secs=settings.caller_ttl_secs,
159
+ drain_secs=settings.drain_secs,
160
+ local_limit_cap=settings.local_limit_cap,
161
+ id_hash_salt=settings.id_hash_salt or "",
162
+ cfg_reload_secs=settings.cfg_reload_secs,
163
+ queue_failover_secs=settings.queue_failover_secs,
164
+ queue_wait_replicas=settings.queue_wait_replicas,
165
+ queue_wait_ms=settings.queue_wait_ms,
166
+ debug_capture_allowed=settings.debug_capture_from_config,
167
+ timings=tuning.engine,
168
+ )
169
+ return Engine(
170
+ state,
171
+ registry,
172
+ Providers(ClientPool(**pool_kwargs)),
173
+ queue_store=queue_store,
174
+ settings=engine_settings,
175
+ worker_id=worker_id,
176
+ metrics=metrics or sink_for(settings.metrics_sink),
177
+ offload=offload,
178
+ offload_bytes=offload_bytes,
179
+ )
180
+
181
+ return cls(build(), settings=settings, builder=build)
182
+
183
+ # ---------------------------------------------------------------- lifecycle
184
+
185
+ async def start(self) -> None:
186
+ """Publish / load the registry; run the worker side unless `mode=caller`.
187
+
188
+ A second `start()` does nothing; a stopped queue cannot start again (build a new one).
189
+ """
190
+ if self.started:
191
+ return
192
+ if self._stopped:
193
+ raise LLMQueueError("this LLMQueue was stopped; build a new one to start again")
194
+ mode = self.settings.mode if self.settings else "both"
195
+ await self.engine.start(
196
+ worker=mode in ("both", "worker"), caller=mode in ("both", "caller")
197
+ )
198
+ if mode in ("both", "worker"):
199
+ await self.jobs.start()
200
+ from llm_queue.langchain import set_default_queue
201
+
202
+ set_default_queue(self)
203
+ self.started = True
204
+
205
+ def pause_receiving(self) -> None:
206
+ """Fail readiness and take no new calls or jobs; running work goes on (preStop hook)."""
207
+ self.engine.pause_receiving()
208
+ self.jobs.pause()
209
+
210
+ async def stop(self, drain_secs: float | None = None) -> None:
211
+ """Shut down in order (plan §3.18) and close every client; a second call does nothing.
212
+
213
+ `drain_secs` (default `LLMQ_DRAIN_SECS`) bounds the whole wait: jobs first (they make
214
+ calls), then running calls get what is left of it.
215
+ """
216
+ from llm_queue.langchain import get_default_queue, set_default_queue
217
+
218
+ if self._stopped:
219
+ return
220
+ self._stopped = True
221
+ with contextlib.suppress(LLMQueueError):
222
+ if get_default_queue() is self:
223
+ set_default_queue(None)
224
+ self.started = False
225
+ loop = asyncio.get_running_loop()
226
+ drain = self.engine.settings.drain_secs if drain_secs is None else drain_secs
227
+ until = loop.time() + drain
228
+ await self.jobs.stop(drain_secs=drain)
229
+ await self.engine.stop(drain_secs=max(0.0, until - loop.time()))
230
+ await asyncio.to_thread(self.close_sync) # blocking: off the event loop
231
+ await self.engine.aclose()
232
+
233
+ # ---------------------------------------------------------------- calls
234
+
235
+ def _call_kwargs(self, priority: bool | None, **kw: Any) -> dict[str, Any]:
236
+ ctx = current_context()
237
+ out = {
238
+ "priority": ctx.priority if priority is None else priority,
239
+ "session_id": kw.pop("session_id", None) or ctx.session_id,
240
+ "user_id": kw.pop("user_id", None) or ctx.user_id,
241
+ "tenant_id": kw.pop("tenant_id", None) or ctx.tenant_id,
242
+ }
243
+ out.update({k: v for k, v in kw.items() if v is not None})
244
+ return out
245
+
246
+ async def call(
247
+ self, model: str, params: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
248
+ ) -> Reply:
249
+ """Any operation of a model (`converse`, `chat`, `invoke_model`, `embeddings`)."""
250
+ return await self.engine.call(model, params, **self._call_kwargs(priority, **kw))
251
+
252
+ async def converse(
253
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
254
+ ) -> Reply:
255
+ """One Bedrock Converse call (the reply's `response` is the Converse body)."""
256
+ return await self.call(model, request, priority=priority, **kw)
257
+
258
+ async def chat(
259
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
260
+ ) -> Reply:
261
+ """One OpenAI-compatible chat completion."""
262
+ return await self.call(model, request, priority=priority, **kw)
263
+
264
+ async def converse_stream(
265
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
266
+ ) -> AsyncIterator[dict[str, Any]]:
267
+ """Stream a chat call's events as they arrive (plan §3.13).
268
+
269
+ Bedrock models yield ConverseStream events, OpenAI-compatible ones chat completion
270
+ chunks. A failure after events were yielded raises `CallFailed` with `partial_text`.
271
+ """
272
+ kwargs = self._call_kwargs(priority, **kw)
273
+ async for event in self.engine.stream(model, request, **kwargs):
274
+ yield event
275
+
276
+ chat_stream = converse_stream
277
+
278
+ async def embed(
279
+ self,
280
+ model: str,
281
+ texts: Sequence[str],
282
+ *,
283
+ priority: bool | None = None,
284
+ input_type: str | None = None,
285
+ **kw: Any,
286
+ ) -> EmbedResult:
287
+ """One vector per text, in order (batched by the model's `embed_batch_size`)."""
288
+ kwargs = self._call_kwargs(priority, **kw)
289
+ vectors = await self.engine.embed(model, texts, input_type=input_type, **kwargs)
290
+ alias = self.engine.registry.current.registry.alias_of(model)
291
+ return EmbedResult(vectors, Meta(served_model=alias))
292
+
293
+ # ---------------------------------------------------------------- sync (direct path)
294
+
295
+ def _sync_runner(self) -> _SyncRunner:
296
+ if threading.current_thread().name == _SyncRunner.THREAD_NAME:
297
+ raise LLMQueueError("a sync call from inside the sync runner would deadlock")
298
+ _warn_if_loop_running()
299
+ runner = self._sync
300
+ if runner is None:
301
+ if self._builder is None:
302
+ raise LLMQueueError("sync calls need LLMQueue.from_env (a second, private engine)")
303
+ with self._sync_lock: # threads making their first sync call together build one
304
+ if self._sync is None:
305
+ self._sync = _SyncRunner(self._builder)
306
+ runner = self._sync
307
+ return runner
308
+
309
+ def converse_sync(
310
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
311
+ ) -> Reply:
312
+ """`converse` on the direct path from sync code (same budgets, no queue)."""
313
+ kwargs = self._call_kwargs(priority, **kw)
314
+ return self._sync_runner().run(
315
+ lambda e: e.call(model, request, force_direct=True, **kwargs)
316
+ )
317
+
318
+ def chat_sync(
319
+ self, model: str, request: Mapping[str, Any], *, priority: bool | None = None, **kw: Any
320
+ ) -> Reply:
321
+ """`chat` on the direct path from sync code."""
322
+ return self.converse_sync(model, request, priority=priority, **kw)
323
+
324
+ def embed_sync(
325
+ self,
326
+ model: str,
327
+ texts: Sequence[str],
328
+ *,
329
+ priority: bool | None = None,
330
+ input_type: str | None = None,
331
+ **kw: Any,
332
+ ) -> EmbedResult:
333
+ """`embed` on the direct path from sync code."""
334
+ kwargs = self._call_kwargs(priority, **kw)
335
+
336
+ async def run(engine: Engine) -> EmbedResult:
337
+ vectors = await engine.embed(
338
+ model, texts, input_type=input_type, force_direct=True, **kwargs
339
+ )
340
+ alias = engine.registry.current.registry.alias_of(model)
341
+ return EmbedResult(vectors, Meta(served_model=alias))
342
+
343
+ return self._sync_runner().run(run)
344
+
345
+ def close_sync(self) -> None:
346
+ """Stop the private engine of the sync calls (sync-only processes; `stop()` does too)."""
347
+ with self._sync_lock:
348
+ runner, self._sync = self._sync, None
349
+ if runner is not None:
350
+ runner.close()
351
+
352
+ # ---------------------------------------------------------------- jobs and admin
353
+
354
+ def register_job(self, name: str, fn: JobFn) -> None:
355
+ """Allow this process to run job `name` (call before `start`)."""
356
+ self.jobs.register(name, fn)
357
+
358
+ @store_op
359
+ async def send_job(
360
+ self, name: str, payload: Mapping[str, Any], *, job_id: str | None = None
361
+ ) -> str:
362
+ """Queue a job; returns its id (jobs run at least once)."""
363
+ return await self.jobs.send(name, payload, job_id=job_id)
364
+
365
+ @store_op
366
+ async def cancel(self, request_id: str) -> None:
367
+ """Cancel a call (waiting or running)."""
368
+ await self.engine.cancel(request_id)
369
+
370
+ @store_op
371
+ async def status(self, model: str | None = None) -> dict[str, Any]:
372
+ """Status of a model or of everything."""
373
+ return await self.engine.status(model)
374
+
375
+ @store_op
376
+ async def request_status(self, request_id: str) -> dict[str, Any]:
377
+ """Status of one call."""
378
+ return await self.engine.request_status(request_id)
379
+
380
+ async def health(self) -> dict[str, Any]:
381
+ """Liveness, readiness and the stores' state."""
382
+ return await self.engine.health_check()
383
+
384
+ @property
385
+ def registry(self) -> RegistryStore:
386
+ """The registry store (live edits, audit, rollback)."""
387
+ return self.engine.registry
388
+
389
+ def http_router(
390
+ self,
391
+ *,
392
+ dependencies: Sequence[Any] = (),
393
+ allow_writes: bool = False,
394
+ actor: Callable[[Any], str] | None = None,
395
+ ) -> Any:
396
+ """A FastAPI router with status, requests, registry and health (extra `http`).
397
+
398
+ Writes need `actor(request) -> str`, the authenticated name recorded in the audit.
399
+ """
400
+ from llm_queue.engine.http import status_router
401
+
402
+ return status_router(
403
+ self.engine, dependencies=dependencies, allow_writes=allow_writes, actor=actor
404
+ )
405
+
406
+
407
+ class _SyncRunner:
408
+ """A private event loop in a thread with its own caller-only engine, for sync calls."""
409
+
410
+ THREAD_NAME = "llmq-sync"
411
+
412
+ def __init__(self, build: Callable[[], Engine]) -> None:
413
+ self._loop = asyncio.new_event_loop()
414
+ self._thread = threading.Thread(
415
+ target=self._loop.run_forever, name=self.THREAD_NAME, daemon=True
416
+ )
417
+ self._thread.start()
418
+ self._closing = False
419
+
420
+ async def boot() -> Engine:
421
+ engine = build()
422
+ await engine.start(worker=False, caller=True)
423
+ return engine
424
+
425
+ try:
426
+ self._engine: Engine = self._submit(boot()).result()
427
+ except BaseException:
428
+ self._loop.call_soon_threadsafe(self._loop.stop) # no thread left behind
429
+ self._thread.join(timeout=5)
430
+ self._loop.close()
431
+ raise
432
+
433
+ def _submit(self, coro: Coroutine[Any, Any, _T]) -> Future[_T]:
434
+ return asyncio.run_coroutine_threadsafe(coro, self._loop)
435
+
436
+ def run(self, fn: Callable[[Engine], Coroutine[Any, Any, _T]]) -> _T:
437
+ """Run one call on the private loop and wait for it."""
438
+ if self._closing:
439
+ raise LLMQueueError("the sync runner is closed")
440
+ return self._submit(fn(self._engine)).result()
441
+
442
+ def close(self) -> None:
443
+ """Stop the private engine and loop; calls still running are cancelled.
444
+
445
+ Their callers get the error: before, a call running at close never returned.
446
+ """
447
+ self._closing = True
448
+
449
+ async def stop() -> None:
450
+ await self._engine.stop(drain_secs=1)
451
+ current = asyncio.current_task()
452
+ rest = [t for t in asyncio.all_tasks() if t is not current]
453
+ for task in rest:
454
+ task.cancel()
455
+ await asyncio.gather(*rest, return_exceptions=True)
456
+ await self._engine.aclose()
457
+
458
+ with contextlib.suppress(Exception):
459
+ self._submit(stop()).result(timeout=10)
460
+ self._loop.call_soon_threadsafe(self._loop.stop)
461
+ self._thread.join(timeout=5)
462
+ if not self._thread.is_alive():
463
+ self._loop.close()
464
+
465
+
466
+ _warned_loop = False
467
+
468
+
469
+ def _warn_if_loop_running() -> None:
470
+ """Warn once when a sync call blocks a running event loop (use the async methods there)."""
471
+ global _warned_loop
472
+ if _warned_loop:
473
+ return
474
+ try:
475
+ asyncio.get_running_loop()
476
+ except RuntimeError:
477
+ return
478
+ _warned_loop = True
479
+ logger.warning(
480
+ "sync_call_on_event_loop",
481
+ hint="converse_sync / embed_sync block this event loop; await converse / embed instead",
482
+ )
@@ -0,0 +1,3 @@
1
+ """The engine: keys, message format, request records, queues, admission and the worker."""
2
+
3
+ from __future__ import annotations