llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""Admission: pick a deployment and take everything a call needs in one store call.
|
|
2
|
+
|
|
3
|
+
`Admission.pick_take` asks the store script to choose a deployment of a model by its strategy
|
|
4
|
+
and take the RPM / TPM budgets, the account budget, a concurrency slot of the call's class and an
|
|
5
|
+
in-progress entry, all or nothing, honouring waiter order and the claim. `finish` corrects
|
|
6
|
+
tokens and learns; `undo` gives a grant back; `renew` is the heartbeat; `sweep` clears the dead.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import random
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
15
|
+
|
|
16
|
+
import orjson
|
|
17
|
+
from redis.exceptions import ConnectionError as StoreConnectionError
|
|
18
|
+
from redis.exceptions import ReadOnlyError
|
|
19
|
+
from redis.exceptions import TimeoutError as StoreTimeoutError
|
|
20
|
+
|
|
21
|
+
from llm_queue._logging import get_logger
|
|
22
|
+
from llm_queue.engine import _admission_lua as lua
|
|
23
|
+
from llm_queue.engine.keys import Keys
|
|
24
|
+
from llm_queue.tuning import EngineTimings
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
import redis.asyncio
|
|
28
|
+
|
|
29
|
+
Class = Literal["p", "n"]
|
|
30
|
+
Outcome = Literal["ok", "throttle", "fail", "none"]
|
|
31
|
+
|
|
32
|
+
logger = get_logger(__name__)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def owner_token(request_id: str, worker: str, attempt: int) -> str:
|
|
36
|
+
"""`request|worker|attempt`: identifies one grant (lease, entry) of one attempt."""
|
|
37
|
+
return f"{request_id}|{worker}|{attempt}"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def position_of(message_id: str) -> float:
|
|
41
|
+
"""Queue position as a number (stream id `ms-seq` -> ms * 1000 + seq)."""
|
|
42
|
+
ms, _, seq = message_id.partition("-")
|
|
43
|
+
return float(int(ms) * 1000 + int(seq or 0))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True, slots=True)
|
|
47
|
+
class Candidate:
|
|
48
|
+
"""A deployment the call may use: its tier, and whether it is only a last choice."""
|
|
49
|
+
|
|
50
|
+
key: str
|
|
51
|
+
tier: Literal["confirmed", "fallback"] = "confirmed"
|
|
52
|
+
last: bool = False
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True, slots=True)
|
|
56
|
+
class Ask:
|
|
57
|
+
"""One admission request (one attempt of one call)."""
|
|
58
|
+
|
|
59
|
+
alias: str
|
|
60
|
+
request_id: str
|
|
61
|
+
worker: str
|
|
62
|
+
attempt: int
|
|
63
|
+
cls: Class
|
|
64
|
+
position: float
|
|
65
|
+
candidates: Sequence[Candidate]
|
|
66
|
+
strategy: str = "ordered"
|
|
67
|
+
pick: str = "percent"
|
|
68
|
+
cost_r: int = 1
|
|
69
|
+
cost_t: int = 0
|
|
70
|
+
claim: bool = True
|
|
71
|
+
sticky_hash: str | None = None
|
|
72
|
+
sticky_ttl_secs: float = 1800.0
|
|
73
|
+
sticky_move_secs: float = 1.0
|
|
74
|
+
foreign_from: str | None = None
|
|
75
|
+
require_pct: int = 0
|
|
76
|
+
tag: str = ""
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def owner(self) -> str:
|
|
80
|
+
"""The owner token of this attempt's grant (`tag` tells two deliveries apart)."""
|
|
81
|
+
worker = f"{self.worker}@{self.tag}" if self.tag else self.worker
|
|
82
|
+
return owner_token(self.request_id, worker, self.attempt)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(frozen=True, slots=True)
|
|
86
|
+
class Decision:
|
|
87
|
+
"""What the script decided.
|
|
88
|
+
|
|
89
|
+
`kind`: granted, wait, duplicate, done, cancelled, stale_config, no_config, none (no
|
|
90
|
+
deployment usable), busy (another model refused a fallback call).
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
kind: str
|
|
94
|
+
deployment: str | None = None
|
|
95
|
+
minute: int | None = None
|
|
96
|
+
wait_secs: float = 0.0
|
|
97
|
+
reason: str = ""
|
|
98
|
+
changes: int | None = None
|
|
99
|
+
tier: str | None = None
|
|
100
|
+
extra: dict[str, Any] = field(default_factory=dict)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class Admission:
|
|
104
|
+
"""The admission scripts of one environment (key prefix)."""
|
|
105
|
+
|
|
106
|
+
def __init__(
|
|
107
|
+
self,
|
|
108
|
+
client: redis.asyncio.Redis,
|
|
109
|
+
keys: Keys,
|
|
110
|
+
*,
|
|
111
|
+
lease_secs: float = 60.0,
|
|
112
|
+
waiter_secs: float = 2.0,
|
|
113
|
+
rng: random.Random | None = None,
|
|
114
|
+
timings: EngineTimings | None = None,
|
|
115
|
+
) -> None:
|
|
116
|
+
"""Register the scripts; no I/O until the first call.
|
|
117
|
+
|
|
118
|
+
`timings` (`Tuning.engine`) carries the scripts' pacing: probe and slot waits, the
|
|
119
|
+
limiter's cut, budget key lifetime; each script falls back to the same defaults.
|
|
120
|
+
"""
|
|
121
|
+
self._client = client
|
|
122
|
+
self.keys = keys
|
|
123
|
+
self.lease_ms = int(lease_secs * 1000)
|
|
124
|
+
self.waiter_ms = int(waiter_secs * 1000)
|
|
125
|
+
self._rng = rng or random.Random()
|
|
126
|
+
pace = timings or EngineTimings()
|
|
127
|
+
self._t = {
|
|
128
|
+
"probe_wait_ms": pace.admission_probe_wait_ms,
|
|
129
|
+
"slot_wait_ms": pace.admission_slot_wait_ms,
|
|
130
|
+
"slot_jitter_ms": pace.admission_slot_jitter_ms,
|
|
131
|
+
"wait_ms": pace.admission_wait_ms,
|
|
132
|
+
"limiter_cut_ms": pace.limiter_cut_ms,
|
|
133
|
+
"limiter_cut_factor": pace.limiter_cut_factor,
|
|
134
|
+
"budget_key_ms": pace.budget_key_ms,
|
|
135
|
+
}
|
|
136
|
+
self._pick = client.register_script(lua.PICK_TAKE)
|
|
137
|
+
self._finish = client.register_script(lua.FINISH)
|
|
138
|
+
self._undo = client.register_script(lua.UNDO)
|
|
139
|
+
self._renew = client.register_script(lua.RENEW)
|
|
140
|
+
self._sweep = client.register_script(lua.SWEEP)
|
|
141
|
+
self._reregister = client.register_script(lua.REREGISTER)
|
|
142
|
+
|
|
143
|
+
async def _call(self, script: Any, body: dict[str, Any]) -> Any:
|
|
144
|
+
return await script([], [self.keys.prefix, orjson.dumps({**body, "t": self._t})])
|
|
145
|
+
|
|
146
|
+
async def pick_take(self, ask: Ask, changes: int) -> Decision:
|
|
147
|
+
"""Choose and take, or say how long to wait and why."""
|
|
148
|
+
body: dict[str, Any] = {
|
|
149
|
+
"alias": ask.alias,
|
|
150
|
+
"changes": changes,
|
|
151
|
+
"request_id": ask.request_id,
|
|
152
|
+
"owner": ask.owner,
|
|
153
|
+
"worker": ask.worker,
|
|
154
|
+
"cls": ask.cls,
|
|
155
|
+
"pos": ask.position,
|
|
156
|
+
"cost_r": ask.cost_r,
|
|
157
|
+
"cost_t": ask.cost_t,
|
|
158
|
+
"candidates": [{"key": c.key, "tier": c.tier, "last": c.last} for c in ask.candidates],
|
|
159
|
+
"strategy": ask.strategy,
|
|
160
|
+
"pick": ask.pick,
|
|
161
|
+
"claim": ask.claim,
|
|
162
|
+
"lease_ms": self.lease_ms,
|
|
163
|
+
"waiter_ms": self.waiter_ms,
|
|
164
|
+
"rand": self._rng.random(),
|
|
165
|
+
}
|
|
166
|
+
if ask.sticky_hash:
|
|
167
|
+
body["sticky"] = {
|
|
168
|
+
"hash": ask.sticky_hash,
|
|
169
|
+
"ttl_ms": int(ask.sticky_ttl_secs * 1000),
|
|
170
|
+
"move_ms": int(ask.sticky_move_secs * 1000),
|
|
171
|
+
}
|
|
172
|
+
if ask.foreign_from is not None:
|
|
173
|
+
body["foreign"] = {"from_alias": ask.foreign_from, "require_pct": ask.require_pct}
|
|
174
|
+
raw = orjson.loads(await self._call(self._pick, body))
|
|
175
|
+
kind = str(raw.pop("r"))
|
|
176
|
+
return Decision(
|
|
177
|
+
kind=kind,
|
|
178
|
+
deployment=raw.pop("dep", None),
|
|
179
|
+
minute=raw.pop("minute", None),
|
|
180
|
+
wait_secs=float(raw.pop("ms", 0)) / 1000.0,
|
|
181
|
+
reason=str(raw.pop("reason", "")),
|
|
182
|
+
changes=raw.pop("changes", None),
|
|
183
|
+
tier=raw.pop("tier", None),
|
|
184
|
+
extra=raw,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
async def finish(
|
|
188
|
+
self,
|
|
189
|
+
alias: str,
|
|
190
|
+
owner: str,
|
|
191
|
+
request_id: str,
|
|
192
|
+
*,
|
|
193
|
+
refund_r: int,
|
|
194
|
+
refund_t: int,
|
|
195
|
+
outcome: Outcome,
|
|
196
|
+
latency_ms: float = -1,
|
|
197
|
+
) -> bool:
|
|
198
|
+
"""Settle a grant: give back what was not used, release the slot, learn the outcome.
|
|
199
|
+
|
|
200
|
+
`refund_t` may be negative (the call used more than it reserved). If the store is
|
|
201
|
+
unreachable this is skipped (False): the grant's lease expires and SWEEP frees the slot,
|
|
202
|
+
so a store blip after the provider answered never throws the answer away.
|
|
203
|
+
"""
|
|
204
|
+
body = {
|
|
205
|
+
"alias": alias,
|
|
206
|
+
"owner": owner,
|
|
207
|
+
"request_id": request_id,
|
|
208
|
+
"refund_r": refund_r,
|
|
209
|
+
"refund_t": refund_t,
|
|
210
|
+
"outcome": outcome,
|
|
211
|
+
"latency_ms": latency_ms,
|
|
212
|
+
}
|
|
213
|
+
try:
|
|
214
|
+
return bool(await self._call(self._finish, body))
|
|
215
|
+
except (StoreConnectionError, StoreTimeoutError, ReadOnlyError, OSError) as exc:
|
|
216
|
+
logger.warning("finish_skipped", model=alias, request_id=request_id, error=str(exc))
|
|
217
|
+
return False
|
|
218
|
+
|
|
219
|
+
async def undo(self, alias: str, owner: str, request_id: str) -> bool:
|
|
220
|
+
"""Give a grant back entirely, without learning (lost reply, refused claim)."""
|
|
221
|
+
body = {"alias": alias, "owner": owner, "request_id": request_id}
|
|
222
|
+
return bool(await self._call(self._undo, body))
|
|
223
|
+
|
|
224
|
+
async def renew(self, items: Sequence[tuple[str, str, str]]) -> int:
|
|
225
|
+
"""Extend leases of held grants: (alias, owner, request id) each."""
|
|
226
|
+
if not items:
|
|
227
|
+
return 0
|
|
228
|
+
body = {
|
|
229
|
+
"lease_ms": self.lease_ms,
|
|
230
|
+
"items": [{"alias": a, "owner": o, "request_id": r} for a, o, r in items],
|
|
231
|
+
}
|
|
232
|
+
return int(await self._call(self._renew, body))
|
|
233
|
+
|
|
234
|
+
async def reregister(self, items: Sequence[dict[str, Any]]) -> int:
|
|
235
|
+
"""After a store reset, put back the slots and entries of running calls."""
|
|
236
|
+
if not items:
|
|
237
|
+
return 0
|
|
238
|
+
body = {"lease_ms": self.lease_ms, "items": list(items)}
|
|
239
|
+
return int(await self._call(self._reregister, body))
|
|
240
|
+
|
|
241
|
+
async def sweep(self, alias: str) -> int:
|
|
242
|
+
"""Remove in-progress entries of owners that stopped renewing."""
|
|
243
|
+
return int(await self._call(self._sweep, {"alias": alias}))
|
llm_queue/engine/cli.py
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
"""The `llm-queue` command line for 0.3 (plan §3.12, §3.18): registry, status, operations.
|
|
2
|
+
|
|
3
|
+
Every command reads `LLMQ_*` from the environment (`InfraSettings.from_env`) and talks to the
|
|
4
|
+
stores directly; writes go through the registry store (validated, protected fields, audited
|
|
5
|
+
with `--actor`, default the OS user). Output is JSON, one document per command.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import asyncio
|
|
12
|
+
import getpass
|
|
13
|
+
import json
|
|
14
|
+
import sys
|
|
15
|
+
from collections.abc import Mapping, Sequence
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import orjson
|
|
19
|
+
import redis.asyncio
|
|
20
|
+
|
|
21
|
+
from llm_queue.engine.http import redacted
|
|
22
|
+
from llm_queue.engine.keys import Keys
|
|
23
|
+
from llm_queue.engine.queue import StreamQueues
|
|
24
|
+
from llm_queue.engine.settings import InfraSettings
|
|
25
|
+
from llm_queue.engine.status import StatusReader
|
|
26
|
+
from llm_queue.engine.store_client import store_client
|
|
27
|
+
from llm_queue.errors import ConfigError, LLMQueueError
|
|
28
|
+
from llm_queue.registry.store import RegistryStore
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _value(text: str) -> Any:
|
|
32
|
+
"""A command-line value: JSON when it parses (`400`, `true`, `["a"]`), else the text."""
|
|
33
|
+
try:
|
|
34
|
+
return json.loads(text)
|
|
35
|
+
except json.JSONDecodeError:
|
|
36
|
+
return text
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parser() -> argparse.ArgumentParser:
|
|
40
|
+
parser = argparse.ArgumentParser(prog="llm-queue", description=__doc__.splitlines()[0])
|
|
41
|
+
parser.add_argument("--actor", default=None, help="who makes a change (audit)")
|
|
42
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
43
|
+
|
|
44
|
+
reg = sub.add_parser("registry", help="read and change the registry").add_subparsers(
|
|
45
|
+
dest="action", required=True
|
|
46
|
+
)
|
|
47
|
+
get = reg.add_parser("get")
|
|
48
|
+
get.add_argument("--model")
|
|
49
|
+
setp = reg.add_parser("set", help="live edit: path=value ... (JSON values)")
|
|
50
|
+
setp.add_argument("--model")
|
|
51
|
+
setp.add_argument("pairs", nargs="+")
|
|
52
|
+
for name in ("enable", "disable"):
|
|
53
|
+
p = reg.add_parser(name)
|
|
54
|
+
p.add_argument("model")
|
|
55
|
+
p.add_argument("deployment")
|
|
56
|
+
reg.add_parser("diff", help="what is live but not in this config")
|
|
57
|
+
log = reg.add_parser("log")
|
|
58
|
+
log.add_argument("--count", type=int, default=20)
|
|
59
|
+
log.add_argument("--path", default="")
|
|
60
|
+
pub = reg.add_parser("publish", help="publish LLMQ_REGISTRY, or roll back")
|
|
61
|
+
pub.add_argument("--from-version", type=int, dest="from_version")
|
|
62
|
+
reg.add_parser("gc", help="delete keys of models that no longer exist")
|
|
63
|
+
|
|
64
|
+
settings = sub.add_parser("settings").add_subparsers(dest="action", required=True)
|
|
65
|
+
sget = settings.add_parser("get")
|
|
66
|
+
sget.add_argument("--model", required=True)
|
|
67
|
+
sget.add_argument("--deployment")
|
|
68
|
+
|
|
69
|
+
status = sub.add_parser("status")
|
|
70
|
+
status.add_argument("--model")
|
|
71
|
+
request = sub.add_parser("request")
|
|
72
|
+
request.add_argument("request_id")
|
|
73
|
+
|
|
74
|
+
for name in ("block", "unblock"):
|
|
75
|
+
p = sub.add_parser(name)
|
|
76
|
+
p.add_argument("--region")
|
|
77
|
+
p.add_argument("--account")
|
|
78
|
+
|
|
79
|
+
drain = sub.add_parser("drain", help="disable a model for new calls; serve waiting ones")
|
|
80
|
+
drain.add_argument("model")
|
|
81
|
+
drain.add_argument("--wait", action="store_true")
|
|
82
|
+
drain.add_argument("--timeout", type=float, default=600.0)
|
|
83
|
+
|
|
84
|
+
dlq = sub.add_parser("dlq").add_subparsers(dest="action", required=True)
|
|
85
|
+
for name in ("list", "show", "redrive", "purge"):
|
|
86
|
+
p = dlq.add_parser(name)
|
|
87
|
+
p.add_argument("--model", required=True)
|
|
88
|
+
p.add_argument("--priority", action="store_true")
|
|
89
|
+
p.add_argument("--count", type=int, default=10)
|
|
90
|
+
if name == "show":
|
|
91
|
+
p.add_argument("--unsafe", action="store_true", help="print message bodies")
|
|
92
|
+
|
|
93
|
+
queues = sub.add_parser("queues").add_subparsers(dest="action", required=True)
|
|
94
|
+
queues.add_parser("list")
|
|
95
|
+
return parser
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Cli:
|
|
99
|
+
"""The command implementations over one environment's stores."""
|
|
100
|
+
|
|
101
|
+
def __init__(
|
|
102
|
+
self,
|
|
103
|
+
settings: InfraSettings,
|
|
104
|
+
client: redis.asyncio.Redis,
|
|
105
|
+
actor: str,
|
|
106
|
+
queue_client: redis.asyncio.Redis | None = None,
|
|
107
|
+
) -> None:
|
|
108
|
+
"""Wire the stores: queues live on `queue_client` when `LLMQ_QUEUE_STORE_URL` is set."""
|
|
109
|
+
self.settings = settings
|
|
110
|
+
self.client = client
|
|
111
|
+
self.actor = actor
|
|
112
|
+
self.keys = Keys(settings.key_prefix)
|
|
113
|
+
config = None
|
|
114
|
+
if settings.registry or settings.registry_file:
|
|
115
|
+
config = orjson.loads(settings.registry_document())
|
|
116
|
+
self.registry = RegistryStore(
|
|
117
|
+
client,
|
|
118
|
+
settings.key_prefix,
|
|
119
|
+
config=config,
|
|
120
|
+
live_edit=settings.registry_live_edit,
|
|
121
|
+
allowed_hosts=settings.base_url_allowed_hosts,
|
|
122
|
+
audit_retention_days=settings.audit_retention_days,
|
|
123
|
+
)
|
|
124
|
+
self.queues = StreamQueues(queue_client or client, self.keys)
|
|
125
|
+
|
|
126
|
+
async def loaded(self) -> Any:
|
|
127
|
+
"""The stored registry; `ConfigError` when none is published."""
|
|
128
|
+
view = await self.registry.load()
|
|
129
|
+
if view is None:
|
|
130
|
+
raise ConfigError("no registry is published yet (llm-queue registry publish)")
|
|
131
|
+
return view
|
|
132
|
+
|
|
133
|
+
def _queue(self, view: Any, model: str, priority: bool) -> str:
|
|
134
|
+
alias = view.registry.alias_of(model)
|
|
135
|
+
name = self.settings.queue_prefix + view.queue_name(alias)
|
|
136
|
+
return f"{name}-priority" if priority else name
|
|
137
|
+
|
|
138
|
+
async def run(self, args: argparse.Namespace) -> Any:
|
|
139
|
+
"""Run one parsed command; returns what to print."""
|
|
140
|
+
handler = getattr(self, f"_{args.command}")
|
|
141
|
+
return await handler(args)
|
|
142
|
+
|
|
143
|
+
async def _registry(self, args: argparse.Namespace) -> Any:
|
|
144
|
+
action = args.action
|
|
145
|
+
if action == "publish":
|
|
146
|
+
if args.from_version is not None:
|
|
147
|
+
result = await self.registry.rollback(args.from_version, actor=self.actor)
|
|
148
|
+
else:
|
|
149
|
+
if self.registry.config is None:
|
|
150
|
+
raise ConfigError("LLMQ_REGISTRY (or LLMQ_REGISTRY_FILE) is not set")
|
|
151
|
+
result = await self.registry.publish(self.registry.config, actor=self.actor)
|
|
152
|
+
return {"action": result.action, "version": result.version}
|
|
153
|
+
if action == "log":
|
|
154
|
+
return await self.registry.log(args.count, args.path)
|
|
155
|
+
if action == "diff":
|
|
156
|
+
return redacted(await self.registry.diff_with_config())
|
|
157
|
+
view = await self.loaded()
|
|
158
|
+
if action == "get":
|
|
159
|
+
doc = dict(view.registry.raw)
|
|
160
|
+
if args.model:
|
|
161
|
+
doc = doc["models"][view.registry.alias_of(args.model)]
|
|
162
|
+
return redacted(doc)
|
|
163
|
+
if action == "set":
|
|
164
|
+
values = {}
|
|
165
|
+
for pair in args.pairs:
|
|
166
|
+
path, sep, text = pair.partition("=")
|
|
167
|
+
if not sep:
|
|
168
|
+
raise ConfigError(f"expected path=value, got {pair!r}")
|
|
169
|
+
values[path] = _value(text)
|
|
170
|
+
changed = await self.registry.set(values, actor=self.actor, model=args.model)
|
|
171
|
+
return {"changes": self.registry.changes, "version": changed.registry.version}
|
|
172
|
+
if action in ("enable", "disable"):
|
|
173
|
+
await self.registry.set_enabled(
|
|
174
|
+
args.model, args.deployment, action == "enable", actor=self.actor
|
|
175
|
+
)
|
|
176
|
+
return {"model": args.model, "deployment": args.deployment, action: True}
|
|
177
|
+
if action == "gc":
|
|
178
|
+
return await self._gc(view)
|
|
179
|
+
raise ConfigError(f"unknown registry action {action}")
|
|
180
|
+
|
|
181
|
+
async def _gc(self, view: Any) -> dict[str, Any]:
|
|
182
|
+
"""Delete keys of models that are not in the registry any more."""
|
|
183
|
+
known = set(view.registry.models)
|
|
184
|
+
removed = 0
|
|
185
|
+
async for key in self.client.scan_iter(match=f"{self.keys.prefix}:{{m:*", count=500):
|
|
186
|
+
text = key.decode() if isinstance(key, bytes) else str(key)
|
|
187
|
+
alias = text.split("{m:", 1)[1].split("}", 1)[0]
|
|
188
|
+
if alias not in known:
|
|
189
|
+
await self.client.delete(key)
|
|
190
|
+
removed += 1
|
|
191
|
+
return {"deleted_keys": removed}
|
|
192
|
+
|
|
193
|
+
async def _settings(self, args: argparse.Namespace) -> Any:
|
|
194
|
+
view = await self.loaded()
|
|
195
|
+
table = view.table(args.model, args.deployment)
|
|
196
|
+
return {name: {"value": e.value, "level": e.level} for name, e in table.items()}
|
|
197
|
+
|
|
198
|
+
async def _status(self, args: argparse.Namespace) -> Any:
|
|
199
|
+
view = await self.loaded()
|
|
200
|
+
reader = StatusReader(self.client, self.keys, self.queues, self.settings.queue_prefix)
|
|
201
|
+
if args.model:
|
|
202
|
+
return await reader.model(view, view.registry.alias_of(args.model))
|
|
203
|
+
return await reader.report(view)
|
|
204
|
+
|
|
205
|
+
async def _request(self, args: argparse.Namespace) -> Any:
|
|
206
|
+
reader = StatusReader(self.client, self.keys, self.queues, self.settings.queue_prefix)
|
|
207
|
+
return await reader.request(args.request_id) or {"error": "unknown or expired request"}
|
|
208
|
+
|
|
209
|
+
async def _block(self, args: argparse.Namespace, on: bool = True) -> Any:
|
|
210
|
+
if not args.region and not args.account:
|
|
211
|
+
raise ConfigError("give --region or --account")
|
|
212
|
+
await self.loaded()
|
|
213
|
+
view = await self.registry.block(
|
|
214
|
+
region=args.region, account=args.account, on=on, actor=self.actor
|
|
215
|
+
)
|
|
216
|
+
routing = view.registry.routing
|
|
217
|
+
return {
|
|
218
|
+
"blocked_regions": list(routing.blocked_regions),
|
|
219
|
+
"blocked_accounts": list(routing.blocked_accounts),
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async def _unblock(self, args: argparse.Namespace) -> Any:
|
|
223
|
+
return await self._block(args, on=False)
|
|
224
|
+
|
|
225
|
+
async def _drain(self, args: argparse.Namespace) -> Any:
|
|
226
|
+
view = await self.loaded()
|
|
227
|
+
alias = view.registry.alias_of(args.model)
|
|
228
|
+
await self.registry.set({"enabled": False}, model=alias, actor=self.actor)
|
|
229
|
+
if not args.wait:
|
|
230
|
+
return {"model": alias, "enabled": False}
|
|
231
|
+
loop = asyncio.get_running_loop()
|
|
232
|
+
until = loop.time() + args.timeout
|
|
233
|
+
while loop.time() < until:
|
|
234
|
+
depth = [await self.queues.depth(self._queue(view, alias, p)) for p in (True, False)]
|
|
235
|
+
if all(total == 0 for total, _, _ in depth):
|
|
236
|
+
return {"model": alias, "enabled": False, "drained": True}
|
|
237
|
+
await asyncio.sleep(1)
|
|
238
|
+
return {"model": alias, "enabled": False, "drained": False}
|
|
239
|
+
|
|
240
|
+
async def _dlq(self, args: argparse.Namespace) -> Any:
|
|
241
|
+
view = await self.loaded()
|
|
242
|
+
queue = self._queue(view, args.model, args.priority)
|
|
243
|
+
if args.action == "list":
|
|
244
|
+
letters = await self.queues.dead_letters(queue, args.count)
|
|
245
|
+
return [{k: v for k, v in letter.items() if k != "body"} for letter in letters]
|
|
246
|
+
if args.action == "show":
|
|
247
|
+
letters = await self.queues.dead_letters(queue, args.count)
|
|
248
|
+
if not args.unsafe:
|
|
249
|
+
return [{k: v for k, v in letter.items() if k != "body"} for letter in letters]
|
|
250
|
+
return [
|
|
251
|
+
{**letter, "body": letter["body"].decode(errors="replace")} for letter in letters
|
|
252
|
+
]
|
|
253
|
+
if args.action == "redrive":
|
|
254
|
+
return {"redriven": await self.queues.redrive(queue, args.count)}
|
|
255
|
+
await self.queues.purge_dead(queue)
|
|
256
|
+
return {"purged": queue}
|
|
257
|
+
|
|
258
|
+
async def _queues(self, args: argparse.Namespace) -> Any:
|
|
259
|
+
view = await self.loaded()
|
|
260
|
+
out = {}
|
|
261
|
+
for alias in view.registry.models:
|
|
262
|
+
for priority in (False, True):
|
|
263
|
+
name = self._queue(view, alias, priority)
|
|
264
|
+
total, held, dead = await self.queues.depth(name)
|
|
265
|
+
out[name] = {"messages": total, "held": held, "dead": dead}
|
|
266
|
+
return out
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
async def amain(argv: Sequence[str], environ: Mapping[str, str] | None = None) -> Any:
|
|
270
|
+
"""Parse and run one command; returns its JSON-able result."""
|
|
271
|
+
args = _parser().parse_args(list(argv))
|
|
272
|
+
settings = InfraSettings.from_env(environ)
|
|
273
|
+
client = store_client(settings.store_url)
|
|
274
|
+
queue_client = store_client(settings.queue_store_url) if settings.queue_store_url else None
|
|
275
|
+
try:
|
|
276
|
+
cli = Cli(settings, client, args.actor or getpass.getuser(), queue_client)
|
|
277
|
+
return await cli.run(args)
|
|
278
|
+
finally:
|
|
279
|
+
await client.aclose()
|
|
280
|
+
if queue_client is not None:
|
|
281
|
+
await queue_client.aclose()
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def main(argv: Sequence[str] | None = None, environ: Mapping[str, str] | None = None) -> int:
|
|
285
|
+
"""Console entry point: print the result as JSON; exit 1 with the message on errors."""
|
|
286
|
+
try:
|
|
287
|
+
result = asyncio.run(amain(sys.argv[1:] if argv is None else argv, environ))
|
|
288
|
+
except (LLMQueueError, KeyError) as exc: # a bad name or edit: one line, not a traceback
|
|
289
|
+
print(str(exc), file=sys.stderr)
|
|
290
|
+
return 1
|
|
291
|
+
print(json.dumps(result, indent=2, default=str))
|
|
292
|
+
return 0
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Embeddings over the queue (plan step A13): batching, provider bodies, vectors in order.
|
|
2
|
+
|
|
3
|
+
`embed()` splits texts into batches of the model's `embed_batch_size` (capped by what the
|
|
4
|
+
provider accepts: Titan 1, Cohere 96, OpenAI 2048), sends each batch as one call (one request
|
|
5
|
+
against RPM; tokens only when token limiting is on for the model), runs them concurrently and
|
|
6
|
+
returns one vector per text in the input order. Inference defaults (dimensions, input_type,
|
|
7
|
+
...) are added by the worker from the registry.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
from llm_queue.clients.providers import decode_invoke_body
|
|
17
|
+
from llm_queue.engine.errors import CallFailed
|
|
18
|
+
from llm_queue.providers.embeddings import batches, max_batch_size, parse_embeddings
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from llm_queue.engine.runtime import Engine
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def batch_size(provider: str, model_id: str, setting: int) -> int:
|
|
25
|
+
"""Texts per call: the model's `embed_batch_size`, never above the provider's limit."""
|
|
26
|
+
return max(1, min(int(setting), max_batch_size(provider, model_id)))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def request_for(
|
|
30
|
+
provider: str, model_id: str, texts: Sequence[str], input_type: str | None = None
|
|
31
|
+
) -> dict[str, Any]:
|
|
32
|
+
"""The provider-native request of one batch (defaults are merged by the worker).
|
|
33
|
+
|
|
34
|
+
`input_type` (Cohere: `search_query` / `search_document`) wins over the registry default.
|
|
35
|
+
"""
|
|
36
|
+
if provider == "openai":
|
|
37
|
+
return {"input": list(texts)}
|
|
38
|
+
if "cohere" in model_id.lower():
|
|
39
|
+
body: dict[str, Any] = {"texts": list(texts)}
|
|
40
|
+
if input_type:
|
|
41
|
+
body["input_type"] = input_type
|
|
42
|
+
return {"body": body}
|
|
43
|
+
return {"body": {"inputText": texts[0]}}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def vectors_of(provider: str, response: dict[str, Any]) -> list[list[float]]:
|
|
47
|
+
"""The vectors of one reply, in the order of its texts."""
|
|
48
|
+
if provider == "openai":
|
|
49
|
+
data = sorted(response.get("data", []), key=lambda item: item.get("index", 0))
|
|
50
|
+
return [list(item["embedding"]) for item in data]
|
|
51
|
+
return parse_embeddings(decode_invoke_body(response))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
async def embed(
|
|
55
|
+
engine: Engine,
|
|
56
|
+
model: str,
|
|
57
|
+
texts: Sequence[str],
|
|
58
|
+
*,
|
|
59
|
+
priority: bool = False,
|
|
60
|
+
timeout_secs: float | None = None,
|
|
61
|
+
concurrency: int = 16,
|
|
62
|
+
input_type: str | None = None,
|
|
63
|
+
**call: Any,
|
|
64
|
+
) -> list[list[float]]:
|
|
65
|
+
"""One vector per text, in order; the batches run through the queue concurrently."""
|
|
66
|
+
if not texts:
|
|
67
|
+
return []
|
|
68
|
+
view = engine.registry.current
|
|
69
|
+
alias = view.registry.alias_of(model)
|
|
70
|
+
entry = view.registry.models[alias]
|
|
71
|
+
size = batch_size(entry.provider, entry.model_id, int(view.value(alias, "embed_batch_size")))
|
|
72
|
+
groups = list(batches(list(texts), size))
|
|
73
|
+
gate = asyncio.Semaphore(concurrency)
|
|
74
|
+
|
|
75
|
+
async def one(group: list[str]) -> list[list[float]]:
|
|
76
|
+
async with gate:
|
|
77
|
+
reply = await engine.call(
|
|
78
|
+
alias,
|
|
79
|
+
request_for(entry.provider, entry.model_id, group, input_type),
|
|
80
|
+
priority=priority,
|
|
81
|
+
timeout_secs=timeout_secs,
|
|
82
|
+
**call,
|
|
83
|
+
)
|
|
84
|
+
if reply.response is None:
|
|
85
|
+
raise CallFailed(
|
|
86
|
+
"provider_error", f"{alias}: no embedding body", request_id=reply.request_id
|
|
87
|
+
)
|
|
88
|
+
vectors = vectors_of(entry.provider, reply.response)
|
|
89
|
+
if len(vectors) != len(group):
|
|
90
|
+
message = f"{alias}: {len(group)} texts gave {len(vectors)} vectors"
|
|
91
|
+
raise CallFailed("provider_error", message, request_id=reply.request_id)
|
|
92
|
+
return vectors
|
|
93
|
+
|
|
94
|
+
tasks = [asyncio.ensure_future(one(group)) for group in groups]
|
|
95
|
+
try:
|
|
96
|
+
results = await asyncio.gather(*tasks)
|
|
97
|
+
except BaseException:
|
|
98
|
+
# One batch failed: stop the rest, which would only spend budget on a lost result.
|
|
99
|
+
for task in tasks:
|
|
100
|
+
task.cancel()
|
|
101
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
102
|
+
raise
|
|
103
|
+
return [vector for group in results for vector in group]
|