llm-queue 0.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_queue/__init__.py +105 -0
- llm_queue/_logging.py +41 -0
- llm_queue/_version.py +5 -0
- llm_queue/clients/__init__.py +42 -0
- llm_queue/clients/pool.py +271 -0
- llm_queue/clients/providers.py +360 -0
- llm_queue/clients/timing.py +52 -0
- llm_queue/context.py +54 -0
- llm_queue/core.py +482 -0
- llm_queue/engine/__init__.py +3 -0
- llm_queue/engine/_admission_lua.py +696 -0
- llm_queue/engine/admission.py +243 -0
- llm_queue/engine/cli.py +292 -0
- llm_queue/engine/embed.py +103 -0
- llm_queue/engine/envelope.py +102 -0
- llm_queue/engine/errors.py +51 -0
- llm_queue/engine/health.py +139 -0
- llm_queue/engine/hooks.py +119 -0
- llm_queue/engine/http.py +180 -0
- llm_queue/engine/inference.py +121 -0
- llm_queue/engine/jobs.py +137 -0
- llm_queue/engine/keys.py +70 -0
- llm_queue/engine/metrics.py +141 -0
- llm_queue/engine/offload.py +82 -0
- llm_queue/engine/queue.py +447 -0
- llm_queue/engine/requests.py +317 -0
- llm_queue/engine/runtime.py +2357 -0
- llm_queue/engine/settings.py +186 -0
- llm_queue/engine/status.py +267 -0
- llm_queue/engine/store_client.py +65 -0
- llm_queue/engine/store_guard.py +40 -0
- llm_queue/engine/streaming.py +216 -0
- llm_queue/engine/tokens.py +129 -0
- llm_queue/errors.py +73 -0
- llm_queue/http_transport.py +211 -0
- llm_queue/ids.py +16 -0
- llm_queue/langchain/__init__.py +61 -0
- llm_queue/langchain/_client.py +94 -0
- llm_queue/langchain/bedrock.py +312 -0
- llm_queue/langchain/openai.py +102 -0
- llm_queue/providers/__init__.py +43 -0
- llm_queue/providers/classify.py +251 -0
- llm_queue/providers/embeddings.py +87 -0
- llm_queue/py.typed +0 -0
- llm_queue/registry/__init__.py +36 -0
- llm_queue/registry/_lua.py +78 -0
- llm_queue/registry/resolve.py +199 -0
- llm_queue/registry/schema.py +521 -0
- llm_queue/registry/settings.py +364 -0
- llm_queue/registry/store.py +614 -0
- llm_queue/registry/validate.py +258 -0
- llm_queue/tuning.py +151 -0
- llm_queue-0.1.0b1.dist-info/METADATA +96 -0
- llm_queue-0.1.0b1.dist-info/RECORD +58 -0
- llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
- llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
- llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
- llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Inference defaults and limits merged into a request, per provider (plan §3.9 rules 1-8).
|
|
2
|
+
|
|
3
|
+
The registry's `inference` values fill what the call did not set; `inference_limits.locked`
|
|
4
|
+
names parameters the call may not change; `inference_limits.max_tokens` caps the output
|
|
5
|
+
(`clamp` or `reject`). Values are written in the provider's own request shape.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import copy
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from llm_queue.errors import LLMQueueError
|
|
15
|
+
|
|
16
|
+
# Registry name -> (Converse path, OpenAI field).
|
|
17
|
+
_CHAT_BEDROCK = {
|
|
18
|
+
"max_tokens": ("inferenceConfig", "maxTokens"),
|
|
19
|
+
"temperature": ("inferenceConfig", "temperature"),
|
|
20
|
+
"top_p": ("inferenceConfig", "topP"),
|
|
21
|
+
"stop": ("inferenceConfig", "stopSequences"),
|
|
22
|
+
"top_k": ("additionalModelRequestFields", "top_k"),
|
|
23
|
+
"frequency_penalty": ("additionalModelRequestFields", "frequency_penalty"),
|
|
24
|
+
"presence_penalty": ("additionalModelRequestFields", "presence_penalty"),
|
|
25
|
+
}
|
|
26
|
+
_WRITTEN = ("inferenceConfig", "additionalModelRequestFields", "body")
|
|
27
|
+
_CHAT_OPENAI = (
|
|
28
|
+
"max_tokens",
|
|
29
|
+
"temperature",
|
|
30
|
+
"top_p",
|
|
31
|
+
"stop",
|
|
32
|
+
"seed",
|
|
33
|
+
"frequency_penalty",
|
|
34
|
+
"presence_penalty",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class InferenceRejected(LLMQueueError):
|
|
39
|
+
"""The call asked for more than `inference_limits` allow and `on_exceed` is `reject`."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _get(params: Mapping[str, Any], path: tuple[str, str]) -> Any:
|
|
43
|
+
return (params.get(path[0]) or {}).get(path[1])
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _set(params: dict[str, Any], path: tuple[str, str], value: Any) -> None:
|
|
47
|
+
params.setdefault(path[0], {})[path[1]] = value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def apply(
|
|
51
|
+
provider: str,
|
|
52
|
+
op: str,
|
|
53
|
+
params: Mapping[str, Any],
|
|
54
|
+
defaults: Mapping[str, Any],
|
|
55
|
+
limits: Mapping[str, Any],
|
|
56
|
+
) -> dict[str, Any]:
|
|
57
|
+
"""The request with defaults filled, locked values enforced and `max_tokens` capped."""
|
|
58
|
+
# Only the top level and the sub-dicts written below are copied (the messages are never
|
|
59
|
+
# changed): a deepcopy of the whole request was 1.86 ms per call at 217 KB (measured).
|
|
60
|
+
out = dict(params)
|
|
61
|
+
for key in _WRITTEN:
|
|
62
|
+
if isinstance(out.get(key), Mapping):
|
|
63
|
+
out[key] = dict(out[key])
|
|
64
|
+
locked = set(limits.get("locked", ()))
|
|
65
|
+
if provider == "bedrock" and op == "converse":
|
|
66
|
+
for name, path in _CHAT_BEDROCK.items():
|
|
67
|
+
if name in defaults and (name in locked or _get(out, path) is None):
|
|
68
|
+
value = defaults[name]
|
|
69
|
+
if name == "stop" and isinstance(value, str):
|
|
70
|
+
value = [value]
|
|
71
|
+
_set(out, path, value)
|
|
72
|
+
extra = defaults.get("extra")
|
|
73
|
+
if extra:
|
|
74
|
+
fields = out.setdefault("additionalModelRequestFields", {})
|
|
75
|
+
for key, value in extra.items():
|
|
76
|
+
fields.setdefault(key, value)
|
|
77
|
+
cap = limits.get("max_tokens")
|
|
78
|
+
current = _get(out, _CHAT_BEDROCK["max_tokens"])
|
|
79
|
+
if cap is not None and isinstance(current, int) and current > cap:
|
|
80
|
+
if limits.get("on_exceed") == "reject":
|
|
81
|
+
raise InferenceRejected(f"max_tokens {current} is above the limit {cap}")
|
|
82
|
+
_set(out, _CHAT_BEDROCK["max_tokens"], cap)
|
|
83
|
+
return out
|
|
84
|
+
if provider == "openai" and op == "chat":
|
|
85
|
+
for name in _CHAT_OPENAI:
|
|
86
|
+
if name in defaults and (name in locked or out.get(name) is None):
|
|
87
|
+
out[name] = defaults[name]
|
|
88
|
+
for key, value in (defaults.get("extra") or {}).items():
|
|
89
|
+
out.setdefault(key, value)
|
|
90
|
+
cap = limits.get("max_tokens")
|
|
91
|
+
current = out.get("max_tokens")
|
|
92
|
+
if cap is not None and isinstance(current, int) and current > cap:
|
|
93
|
+
if limits.get("on_exceed") == "reject":
|
|
94
|
+
raise InferenceRejected(f"max_tokens {current} is above the limit {cap}")
|
|
95
|
+
out["max_tokens"] = cap
|
|
96
|
+
return out
|
|
97
|
+
if op == "embeddings" and "dimensions" in defaults:
|
|
98
|
+
out.setdefault("dimensions", defaults["dimensions"])
|
|
99
|
+
if op == "invoke_model" and isinstance(out.get("body"), Mapping):
|
|
100
|
+
body = dict(out["body"])
|
|
101
|
+
names = ("dimensions", "normalize", "input_type", "truncate", "embedding_types")
|
|
102
|
+
for name in names:
|
|
103
|
+
if name in defaults and (name in locked or name not in body):
|
|
104
|
+
body[name] = defaults[name]
|
|
105
|
+
if "inputText" in body: # Titan takes dimensions / normalize only
|
|
106
|
+
body = {k: v for k, v in body.items() if k not in ("input_type", "truncate")}
|
|
107
|
+
else: # Cohere takes input_type / truncate / embedding_types only
|
|
108
|
+
body = {k: v for k, v in body.items() if k not in ("dimensions", "normalize")}
|
|
109
|
+
out["body"] = body
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def merge_params(request: Mapping[str, Any], extra: Mapping[str, Any]) -> dict[str, Any]:
|
|
114
|
+
"""Merge a deployment's `params` (guardrail, model fields) into a request, one level deep."""
|
|
115
|
+
out = dict(request)
|
|
116
|
+
for key, value in extra.items():
|
|
117
|
+
if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
|
|
118
|
+
out[key] = {**out[key], **value}
|
|
119
|
+
else:
|
|
120
|
+
out[key] = copy.deepcopy(value)
|
|
121
|
+
return out
|
llm_queue/engine/jobs.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Jobs on the stream queue (plan E136): `register_job` / `send_job`, run at least once.
|
|
2
|
+
|
|
3
|
+
A job is any registered async function taking one JSON payload. `send_job` puts it on
|
|
4
|
+
`<queue_prefix>jobs-<name>`; workers of processes that registered the name run it. A failed
|
|
5
|
+
job is retried after a backoff and goes to the dead-letter stream after `max_receives`; a held
|
|
6
|
+
job's visibility is extended while it runs. Jobs take no LLM budget (their own LLM calls do).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import contextlib
|
|
13
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
import orjson
|
|
17
|
+
|
|
18
|
+
from llm_queue._logging import get_logger
|
|
19
|
+
from llm_queue.engine.queue import Message, QueueSettings, StreamQueues
|
|
20
|
+
from llm_queue.errors import ConfigError
|
|
21
|
+
from llm_queue.ids import new_id
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
logger = get_logger(__name__)
|
|
27
|
+
JobFn = Callable[[dict[str, Any]], Awaitable[Any]]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class JobRunner:
|
|
31
|
+
"""Registered jobs of one process, their senders and their worker loops."""
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
queues: StreamQueues,
|
|
36
|
+
*,
|
|
37
|
+
queue_prefix: str = "",
|
|
38
|
+
settings: QueueSettings | None = None,
|
|
39
|
+
concurrency: int = 4,
|
|
40
|
+
) -> None:
|
|
41
|
+
"""Nothing runs until `start`."""
|
|
42
|
+
self.queues = queues
|
|
43
|
+
self.queue_prefix = queue_prefix
|
|
44
|
+
self.settings = settings or QueueSettings(visibility_secs=60, max_receives=5, max_waiting=0)
|
|
45
|
+
self.concurrency = concurrency
|
|
46
|
+
self._jobs: dict[str, JobFn] = {}
|
|
47
|
+
self._tasks: list[asyncio.Task[None]] = []
|
|
48
|
+
self._running: set[asyncio.Task[None]] = set()
|
|
49
|
+
self._stopping = False
|
|
50
|
+
self._paused = False
|
|
51
|
+
|
|
52
|
+
def queue(self, name: str) -> str:
|
|
53
|
+
"""The queue of one job name."""
|
|
54
|
+
return f"{self.queue_prefix}jobs-{name}"
|
|
55
|
+
|
|
56
|
+
def register(self, name: str, fn: JobFn) -> None:
|
|
57
|
+
"""Allow this process to run job `name` (before `start`)."""
|
|
58
|
+
if not name or not callable(fn):
|
|
59
|
+
raise ConfigError("a job needs a name and an async function")
|
|
60
|
+
self._jobs[name] = fn
|
|
61
|
+
|
|
62
|
+
async def send(
|
|
63
|
+
self, name: str, payload: Mapping[str, Any], *, job_id: str | None = None
|
|
64
|
+
) -> str:
|
|
65
|
+
"""Queue a job; returns its id (jobs run at least once: make them idempotent)."""
|
|
66
|
+
ident = job_id or new_id()
|
|
67
|
+
body = orjson.dumps({"id": ident, "name": name, "payload": payload})
|
|
68
|
+
await self.queues.send(self.queue(name), body, self.settings)
|
|
69
|
+
return ident
|
|
70
|
+
|
|
71
|
+
async def start(self) -> None:
|
|
72
|
+
"""One worker loop per registered job."""
|
|
73
|
+
for name in self._jobs:
|
|
74
|
+
self._tasks.append(asyncio.create_task(self._loop(name)))
|
|
75
|
+
|
|
76
|
+
def pause(self) -> None:
|
|
77
|
+
"""Take no new jobs; running ones go on (`stop()` follows)."""
|
|
78
|
+
self._paused = True
|
|
79
|
+
|
|
80
|
+
async def stop(self, drain_secs: float = 10.0) -> None:
|
|
81
|
+
"""Stop receiving, let running jobs finish (up to `drain_secs`), then stop."""
|
|
82
|
+
self._stopping = True
|
|
83
|
+
if self._running:
|
|
84
|
+
await asyncio.wait(set(self._running), timeout=drain_secs)
|
|
85
|
+
for task in [*self._tasks, *self._running]:
|
|
86
|
+
task.cancel()
|
|
87
|
+
for task in [*self._tasks, *self._running]:
|
|
88
|
+
with contextlib.suppress(asyncio.CancelledError, Exception):
|
|
89
|
+
await task
|
|
90
|
+
|
|
91
|
+
async def _loop(self, name: str) -> None:
|
|
92
|
+
gate = asyncio.Semaphore(self.concurrency)
|
|
93
|
+
while not self._stopping:
|
|
94
|
+
if self._paused:
|
|
95
|
+
await asyncio.sleep(0.2)
|
|
96
|
+
continue
|
|
97
|
+
try:
|
|
98
|
+
await gate.acquire()
|
|
99
|
+
messages, dead = await self.queues.receive_wait(
|
|
100
|
+
self.queue(name), 1, self.settings, wait_secs=1.0
|
|
101
|
+
)
|
|
102
|
+
for letter in dead:
|
|
103
|
+
logger.error("job_dead_lettered", job=name, message=letter.id)
|
|
104
|
+
if not messages:
|
|
105
|
+
gate.release()
|
|
106
|
+
continue
|
|
107
|
+
task = asyncio.create_task(self._run(name, messages[0], gate))
|
|
108
|
+
self._running.add(task)
|
|
109
|
+
task.add_done_callback(self._running.discard)
|
|
110
|
+
except Exception as exc: # a store blip must not end the loop
|
|
111
|
+
gate.release()
|
|
112
|
+
logger.warning("job_receive_failed", job=name, error=str(exc))
|
|
113
|
+
await asyncio.sleep(1)
|
|
114
|
+
|
|
115
|
+
async def _run(self, name: str, message: Message, gate: asyncio.Semaphore) -> None:
|
|
116
|
+
keep = asyncio.create_task(self._keep_visible(message))
|
|
117
|
+
try:
|
|
118
|
+
body = orjson.loads(message.body)
|
|
119
|
+
await self._jobs[name](body["payload"])
|
|
120
|
+
await self.queues.ack(message)
|
|
121
|
+
logger.info("job_done", job=name, id=body.get("id"))
|
|
122
|
+
except Exception as exc:
|
|
123
|
+
outcome = await self.queues.fail(message, self.settings, delay_secs=5.0)
|
|
124
|
+
logger.warning("job_failed", job=name, error=str(exc), outcome=outcome)
|
|
125
|
+
finally:
|
|
126
|
+
keep.cancel()
|
|
127
|
+
gate.release()
|
|
128
|
+
|
|
129
|
+
async def _keep_visible(self, message: Message) -> None:
|
|
130
|
+
"""Keep a running job hidden; a store blip must not end it (the job would run twice)."""
|
|
131
|
+
while True:
|
|
132
|
+
await asyncio.sleep(max(1.0, self.settings.visibility_secs / 3))
|
|
133
|
+
try:
|
|
134
|
+
if not await self.queues.extend(message, self.settings.visibility_secs):
|
|
135
|
+
logger.warning("job_lost_visibility", queue=message.queue, message=message.id)
|
|
136
|
+
except Exception as exc:
|
|
137
|
+
logger.warning("job_extend_failed", message=message.id, error=str(exc))
|
llm_queue/engine/keys.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Key names of the 0.3 engine (key schema 3), all built from one environment prefix.
|
|
2
|
+
|
|
3
|
+
Keys of one model share the tag `{m:ALIAS}` and keys of one request `{r:ID}`; the admission and
|
|
4
|
+
finish scripts touch both, which is fine on the single-primary store 0.3 supports (plan §3.2).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class Keys:
|
|
14
|
+
"""Key builder for one prefix, e.g. `llmq:prod` -> `llmq:prod:{m:gpt120b}:wait:p`."""
|
|
15
|
+
|
|
16
|
+
prefix: str
|
|
17
|
+
|
|
18
|
+
# ---------------------------------------------------------------- per model
|
|
19
|
+
|
|
20
|
+
def model(self, alias: str, name: str) -> str:
|
|
21
|
+
"""A key of one model, e.g. `cfg`, `rot`, `health`, `inprog`, `wait:p`."""
|
|
22
|
+
return f"{self.prefix}:{{m:{alias}}}:{name}"
|
|
23
|
+
|
|
24
|
+
def rate(self, alias: str, deployment: str) -> str:
|
|
25
|
+
"""RPM / TPM budget of one deployment (hash of clocks and minute counters)."""
|
|
26
|
+
return self.model(alias, f"rate:{deployment}")
|
|
27
|
+
|
|
28
|
+
def limiter(self, alias: str, deployment: str) -> str:
|
|
29
|
+
"""Adaptive concurrency limit of one deployment (hash)."""
|
|
30
|
+
return self.model(alias, f"ac:{deployment}")
|
|
31
|
+
|
|
32
|
+
def breaker(self, alias: str, deployment: str) -> str:
|
|
33
|
+
"""Circuit breaker of one deployment (hash)."""
|
|
34
|
+
return self.model(alias, f"cb:{deployment}")
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------- per request
|
|
37
|
+
|
|
38
|
+
def request(self, request_id: str, name: str) -> str:
|
|
39
|
+
"""A key of one request: `job` (claim), `res` (result), `params`, `cancel`, `chunks`."""
|
|
40
|
+
return f"{self.prefix}:{{r:{request_id}}}:{name}"
|
|
41
|
+
|
|
42
|
+
def status(self, request_id: str) -> str:
|
|
43
|
+
"""Request status (hash, `request_status`)."""
|
|
44
|
+
return self.request(request_id, "status")
|
|
45
|
+
|
|
46
|
+
# ---------------------------------------------------------------- per process
|
|
47
|
+
|
|
48
|
+
def caller(self, caller: str) -> str:
|
|
49
|
+
"""Liveness of one calling process (`pod:boot`)."""
|
|
50
|
+
return f"{self.prefix}:caller:{caller}"
|
|
51
|
+
|
|
52
|
+
def done_channel(self, caller: str) -> str:
|
|
53
|
+
"""Pub/sub channel that wakes one caller's result reader."""
|
|
54
|
+
return f"{self.prefix}:done:{caller}"
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def cancel_channel(self) -> str:
|
|
58
|
+
"""Pub/sub channel naming request ids just cancelled."""
|
|
59
|
+
return f"{self.prefix}:cancel"
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def epoch(self) -> str:
|
|
63
|
+
"""Set once per store life; missing means the store restarted or lost its data."""
|
|
64
|
+
return f"{self.prefix}:epoch"
|
|
65
|
+
|
|
66
|
+
# ---------------------------------------------------------------- queues
|
|
67
|
+
|
|
68
|
+
def queue(self, name: str, part: str) -> str:
|
|
69
|
+
"""A key of one queue: `s` (stream), `vis`, `gen`, `fails`, `st`, `dlq`, `meta`."""
|
|
70
|
+
return f"{self.prefix}:{{q:{name}}}:{part}"
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Metric sinks (plan E131): CloudWatch embedded metric format by default, Prometheus optional.
|
|
2
|
+
|
|
3
|
+
The engine calls `count`, `observe` and `gauge` with labels model, account, region (and
|
|
4
|
+
priority on waits). `EmfSink` writes one JSON line per point that CloudWatch turns into a
|
|
5
|
+
metric; `PrometheusSink` keeps `prometheus_client` metrics for a `/metrics` endpoint.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Callable, Mapping
|
|
13
|
+
from typing import Any, Protocol
|
|
14
|
+
|
|
15
|
+
import orjson
|
|
16
|
+
|
|
17
|
+
from llm_queue.errors import MissingExtra
|
|
18
|
+
|
|
19
|
+
NAMESPACE = "llm-queue"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Metrics(Protocol):
|
|
23
|
+
"""What the engine needs from a metrics sink."""
|
|
24
|
+
|
|
25
|
+
def count(self, name: str, value: float = 1.0, **labels: str) -> None:
|
|
26
|
+
"""Add to a counter."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
def observe(self, name: str, value: float, **labels: str) -> None:
|
|
30
|
+
"""Record one sample of a distribution (seconds, tokens)."""
|
|
31
|
+
...
|
|
32
|
+
|
|
33
|
+
def gauge(self, name: str, value: float, **labels: str) -> None:
|
|
34
|
+
"""Set a gauge."""
|
|
35
|
+
...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class NullSink:
|
|
39
|
+
"""Drops every point (tests, or metrics turned off)."""
|
|
40
|
+
|
|
41
|
+
def count(self, name: str, value: float = 1.0, **labels: str) -> None:
|
|
42
|
+
"""Ignore."""
|
|
43
|
+
|
|
44
|
+
def observe(self, name: str, value: float, **labels: str) -> None:
|
|
45
|
+
"""Ignore."""
|
|
46
|
+
|
|
47
|
+
def gauge(self, name: str, value: float, **labels: str) -> None:
|
|
48
|
+
"""Ignore."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class EmfSink:
|
|
52
|
+
"""CloudWatch embedded metric format: one JSON line per point on `write` (stdout)."""
|
|
53
|
+
|
|
54
|
+
def __init__(
|
|
55
|
+
self,
|
|
56
|
+
write: Callable[[str], Any] | None = None,
|
|
57
|
+
namespace: str = NAMESPACE,
|
|
58
|
+
static: Mapping[str, str] | None = None,
|
|
59
|
+
) -> None:
|
|
60
|
+
"""`write` receives each line (default: stdout); `static` labels go on every point."""
|
|
61
|
+
self._write = write or (lambda line: sys.stdout.write(line + "\n"))
|
|
62
|
+
self.namespace = namespace
|
|
63
|
+
self.static = dict(static or {})
|
|
64
|
+
|
|
65
|
+
def _emit(self, name: str, value: float, unit: str, labels: Mapping[str, str]) -> None:
|
|
66
|
+
labels = {**self.static, **labels}
|
|
67
|
+
dims = sorted(labels)
|
|
68
|
+
line = {
|
|
69
|
+
"_aws": {
|
|
70
|
+
"Timestamp": int(time.time() * 1000),
|
|
71
|
+
"CloudWatchMetrics": [
|
|
72
|
+
{
|
|
73
|
+
"Namespace": self.namespace,
|
|
74
|
+
"Dimensions": [dims],
|
|
75
|
+
"Metrics": [{"Name": name, "Unit": unit}],
|
|
76
|
+
}
|
|
77
|
+
],
|
|
78
|
+
},
|
|
79
|
+
name: value,
|
|
80
|
+
**labels,
|
|
81
|
+
}
|
|
82
|
+
self._write(orjson.dumps(line).decode()) # compact; 0.8 us vs 6.2 us for json.dumps
|
|
83
|
+
|
|
84
|
+
def count(self, name: str, value: float = 1.0, **labels: str) -> None:
|
|
85
|
+
"""A count point."""
|
|
86
|
+
self._emit(name, value, "Count", labels)
|
|
87
|
+
|
|
88
|
+
def observe(self, name: str, value: float, **labels: str) -> None:
|
|
89
|
+
"""A sample point (seconds for `_seconds` names)."""
|
|
90
|
+
self._emit(name, value, "Seconds" if name.endswith("_seconds") else "None", labels)
|
|
91
|
+
|
|
92
|
+
def gauge(self, name: str, value: float, **labels: str) -> None:
|
|
93
|
+
"""A gauge point."""
|
|
94
|
+
self._emit(name, value, "None", labels)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class PrometheusSink:
|
|
98
|
+
"""`prometheus_client` metrics (needs the `metrics` extra); serve them with its exposition."""
|
|
99
|
+
|
|
100
|
+
def __init__(self, registry: Any = None) -> None:
|
|
101
|
+
"""Create metrics lazily, by name and label set."""
|
|
102
|
+
try:
|
|
103
|
+
import prometheus_client
|
|
104
|
+
except ImportError as exc:
|
|
105
|
+
raise MissingExtra("PrometheusSink", "metrics") from exc
|
|
106
|
+
|
|
107
|
+
self._prom = prometheus_client
|
|
108
|
+
self._registry = registry or prometheus_client.REGISTRY
|
|
109
|
+
self._made: dict[str, Any] = {}
|
|
110
|
+
|
|
111
|
+
def _metric(self, kind: str, name: str, labels: Mapping[str, str]) -> Any:
|
|
112
|
+
if name not in self._made:
|
|
113
|
+
cls = {
|
|
114
|
+
"count": self._prom.Counter,
|
|
115
|
+
"observe": self._prom.Histogram,
|
|
116
|
+
"gauge": self._prom.Gauge,
|
|
117
|
+
}[kind]
|
|
118
|
+
self._made[name] = cls(name, name, sorted(labels), registry=self._registry)
|
|
119
|
+
metric = self._made[name]
|
|
120
|
+
return metric.labels(**labels) if labels else metric
|
|
121
|
+
|
|
122
|
+
def count(self, name: str, value: float = 1.0, **labels: str) -> None:
|
|
123
|
+
"""Add to a counter."""
|
|
124
|
+
self._metric("count", name.removesuffix("_total"), labels).inc(value)
|
|
125
|
+
|
|
126
|
+
def observe(self, name: str, value: float, **labels: str) -> None:
|
|
127
|
+
"""Observe into a histogram."""
|
|
128
|
+
self._metric("observe", name, labels).observe(value)
|
|
129
|
+
|
|
130
|
+
def gauge(self, name: str, value: float, **labels: str) -> None:
|
|
131
|
+
"""Set a gauge."""
|
|
132
|
+
self._metric("gauge", name, labels).set(value)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def sink_for(name: str) -> Metrics:
|
|
136
|
+
"""The sink named by the `metrics_sink` setting: `emf`, `prometheus` or `none`."""
|
|
137
|
+
if name == "prometheus":
|
|
138
|
+
return PrometheusSink()
|
|
139
|
+
if name == "none":
|
|
140
|
+
return NullSink()
|
|
141
|
+
return EmfSink()
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Large results in object storage (plan E137): the state store keeps only a reference.
|
|
2
|
+
|
|
3
|
+
A result above `result_offload_bytes` is written to an S3 bucket (encrypted, with a one-day
|
|
4
|
+
lifecycle rule set on the bucket by the owner) and the stored reply is a small marker. Readers
|
|
5
|
+
fetch it transparently. Used for bulk embedding batches, whose vectors are large.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Protocol
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from aiobotocore.session import AioSession
|
|
14
|
+
|
|
15
|
+
MARKER = b"llmq-offload:"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ResultOffload(Protocol):
|
|
19
|
+
"""Somewhere to keep a large result body."""
|
|
20
|
+
|
|
21
|
+
async def put(self, key: str, data: bytes) -> str:
|
|
22
|
+
"""Store `data`; return a reference."""
|
|
23
|
+
...
|
|
24
|
+
|
|
25
|
+
async def get(self, ref: str) -> bytes | None:
|
|
26
|
+
"""The data of a reference (None when it expired)."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class S3Offload:
|
|
31
|
+
"""`ResultOffload` on an S3 bucket (server-side encryption; expiry by the bucket's rule)."""
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
bucket: str,
|
|
36
|
+
*,
|
|
37
|
+
prefix: str = "llmq-results/",
|
|
38
|
+
session: AioSession | None = None,
|
|
39
|
+
client_kwargs: dict[str, Any] | None = None,
|
|
40
|
+
) -> None:
|
|
41
|
+
"""Keep settings; the client is created on first use."""
|
|
42
|
+
self.bucket = bucket
|
|
43
|
+
self.prefix = prefix
|
|
44
|
+
self._session = session
|
|
45
|
+
self._kwargs = dict(client_kwargs or {})
|
|
46
|
+
self._client: Any = None
|
|
47
|
+
self._context: Any = None
|
|
48
|
+
|
|
49
|
+
async def _s3(self) -> Any:
|
|
50
|
+
if self._client is None:
|
|
51
|
+
from aiobotocore.session import get_session
|
|
52
|
+
|
|
53
|
+
session = self._session or get_session()
|
|
54
|
+
self._context = session.create_client("s3", **self._kwargs)
|
|
55
|
+
self._client = await self._context.__aenter__()
|
|
56
|
+
return self._client
|
|
57
|
+
|
|
58
|
+
async def put(self, key: str, data: bytes) -> str:
|
|
59
|
+
"""Write the object (AES256 server-side encryption)."""
|
|
60
|
+
s3 = await self._s3()
|
|
61
|
+
name = f"{self.prefix}{key}"
|
|
62
|
+
await s3.put_object(Bucket=self.bucket, Key=name, Body=data, ServerSideEncryption="AES256")
|
|
63
|
+
return name
|
|
64
|
+
|
|
65
|
+
async def get(self, ref: str) -> bytes | None:
|
|
66
|
+
"""Read the object; None when it is gone."""
|
|
67
|
+
s3 = await self._s3()
|
|
68
|
+
try:
|
|
69
|
+
reply = await s3.get_object(Bucket=self.bucket, Key=ref)
|
|
70
|
+
except Exception as exc:
|
|
71
|
+
if "NoSuchKey" in str(exc) or "404" in str(exc):
|
|
72
|
+
return None
|
|
73
|
+
raise
|
|
74
|
+
async with reply["Body"] as body:
|
|
75
|
+
data: bytes = await body.read()
|
|
76
|
+
return data
|
|
77
|
+
|
|
78
|
+
async def aclose(self) -> None:
|
|
79
|
+
"""Close the client."""
|
|
80
|
+
if self._context is not None:
|
|
81
|
+
await self._context.__aexit__(None, None, None)
|
|
82
|
+
self._client = self._context = None
|