llm-queue 0.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. llm_queue/__init__.py +105 -0
  2. llm_queue/_logging.py +41 -0
  3. llm_queue/_version.py +5 -0
  4. llm_queue/clients/__init__.py +42 -0
  5. llm_queue/clients/pool.py +271 -0
  6. llm_queue/clients/providers.py +360 -0
  7. llm_queue/clients/timing.py +52 -0
  8. llm_queue/context.py +54 -0
  9. llm_queue/core.py +482 -0
  10. llm_queue/engine/__init__.py +3 -0
  11. llm_queue/engine/_admission_lua.py +696 -0
  12. llm_queue/engine/admission.py +243 -0
  13. llm_queue/engine/cli.py +292 -0
  14. llm_queue/engine/embed.py +103 -0
  15. llm_queue/engine/envelope.py +102 -0
  16. llm_queue/engine/errors.py +51 -0
  17. llm_queue/engine/health.py +139 -0
  18. llm_queue/engine/hooks.py +119 -0
  19. llm_queue/engine/http.py +180 -0
  20. llm_queue/engine/inference.py +121 -0
  21. llm_queue/engine/jobs.py +137 -0
  22. llm_queue/engine/keys.py +70 -0
  23. llm_queue/engine/metrics.py +141 -0
  24. llm_queue/engine/offload.py +82 -0
  25. llm_queue/engine/queue.py +447 -0
  26. llm_queue/engine/requests.py +317 -0
  27. llm_queue/engine/runtime.py +2357 -0
  28. llm_queue/engine/settings.py +186 -0
  29. llm_queue/engine/status.py +267 -0
  30. llm_queue/engine/store_client.py +65 -0
  31. llm_queue/engine/store_guard.py +40 -0
  32. llm_queue/engine/streaming.py +216 -0
  33. llm_queue/engine/tokens.py +129 -0
  34. llm_queue/errors.py +73 -0
  35. llm_queue/http_transport.py +211 -0
  36. llm_queue/ids.py +16 -0
  37. llm_queue/langchain/__init__.py +61 -0
  38. llm_queue/langchain/_client.py +94 -0
  39. llm_queue/langchain/bedrock.py +312 -0
  40. llm_queue/langchain/openai.py +102 -0
  41. llm_queue/providers/__init__.py +43 -0
  42. llm_queue/providers/classify.py +251 -0
  43. llm_queue/providers/embeddings.py +87 -0
  44. llm_queue/py.typed +0 -0
  45. llm_queue/registry/__init__.py +36 -0
  46. llm_queue/registry/_lua.py +78 -0
  47. llm_queue/registry/resolve.py +199 -0
  48. llm_queue/registry/schema.py +521 -0
  49. llm_queue/registry/settings.py +364 -0
  50. llm_queue/registry/store.py +614 -0
  51. llm_queue/registry/validate.py +258 -0
  52. llm_queue/tuning.py +151 -0
  53. llm_queue-0.1.0b1.dist-info/METADATA +96 -0
  54. llm_queue-0.1.0b1.dist-info/RECORD +58 -0
  55. llm_queue-0.1.0b1.dist-info/WHEEL +4 -0
  56. llm_queue-0.1.0b1.dist-info/entry_points.txt +2 -0
  57. llm_queue-0.1.0b1.dist-info/licenses/LICENSE +21 -0
  58. llm_queue-0.1.0b1.dist-info/licenses/NOTICE +14 -0
@@ -0,0 +1,121 @@
1
+ """Inference defaults and limits merged into a request, per provider (plan §3.9 rules 1-8).
2
+
3
+ The registry's `inference` values fill what the call did not set; `inference_limits.locked`
4
+ names parameters the call may not change; `inference_limits.max_tokens` caps the output
5
+ (`clamp` or `reject`). Values are written in the provider's own request shape.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import copy
11
+ from collections.abc import Mapping
12
+ from typing import Any
13
+
14
+ from llm_queue.errors import LLMQueueError
15
+
16
+ # Registry name -> (Converse path, OpenAI field).
17
+ _CHAT_BEDROCK = {
18
+ "max_tokens": ("inferenceConfig", "maxTokens"),
19
+ "temperature": ("inferenceConfig", "temperature"),
20
+ "top_p": ("inferenceConfig", "topP"),
21
+ "stop": ("inferenceConfig", "stopSequences"),
22
+ "top_k": ("additionalModelRequestFields", "top_k"),
23
+ "frequency_penalty": ("additionalModelRequestFields", "frequency_penalty"),
24
+ "presence_penalty": ("additionalModelRequestFields", "presence_penalty"),
25
+ }
26
+ _WRITTEN = ("inferenceConfig", "additionalModelRequestFields", "body")
27
+ _CHAT_OPENAI = (
28
+ "max_tokens",
29
+ "temperature",
30
+ "top_p",
31
+ "stop",
32
+ "seed",
33
+ "frequency_penalty",
34
+ "presence_penalty",
35
+ )
36
+
37
+
38
+ class InferenceRejected(LLMQueueError):
39
+ """The call asked for more than `inference_limits` allow and `on_exceed` is `reject`."""
40
+
41
+
42
+ def _get(params: Mapping[str, Any], path: tuple[str, str]) -> Any:
43
+ return (params.get(path[0]) or {}).get(path[1])
44
+
45
+
46
+ def _set(params: dict[str, Any], path: tuple[str, str], value: Any) -> None:
47
+ params.setdefault(path[0], {})[path[1]] = value
48
+
49
+
50
+ def apply(
51
+ provider: str,
52
+ op: str,
53
+ params: Mapping[str, Any],
54
+ defaults: Mapping[str, Any],
55
+ limits: Mapping[str, Any],
56
+ ) -> dict[str, Any]:
57
+ """The request with defaults filled, locked values enforced and `max_tokens` capped."""
58
+ # Only the top level and the sub-dicts written below are copied (the messages are never
59
+ # changed): a deepcopy of the whole request was 1.86 ms per call at 217 KB (measured).
60
+ out = dict(params)
61
+ for key in _WRITTEN:
62
+ if isinstance(out.get(key), Mapping):
63
+ out[key] = dict(out[key])
64
+ locked = set(limits.get("locked", ()))
65
+ if provider == "bedrock" and op == "converse":
66
+ for name, path in _CHAT_BEDROCK.items():
67
+ if name in defaults and (name in locked or _get(out, path) is None):
68
+ value = defaults[name]
69
+ if name == "stop" and isinstance(value, str):
70
+ value = [value]
71
+ _set(out, path, value)
72
+ extra = defaults.get("extra")
73
+ if extra:
74
+ fields = out.setdefault("additionalModelRequestFields", {})
75
+ for key, value in extra.items():
76
+ fields.setdefault(key, value)
77
+ cap = limits.get("max_tokens")
78
+ current = _get(out, _CHAT_BEDROCK["max_tokens"])
79
+ if cap is not None and isinstance(current, int) and current > cap:
80
+ if limits.get("on_exceed") == "reject":
81
+ raise InferenceRejected(f"max_tokens {current} is above the limit {cap}")
82
+ _set(out, _CHAT_BEDROCK["max_tokens"], cap)
83
+ return out
84
+ if provider == "openai" and op == "chat":
85
+ for name in _CHAT_OPENAI:
86
+ if name in defaults and (name in locked or out.get(name) is None):
87
+ out[name] = defaults[name]
88
+ for key, value in (defaults.get("extra") or {}).items():
89
+ out.setdefault(key, value)
90
+ cap = limits.get("max_tokens")
91
+ current = out.get("max_tokens")
92
+ if cap is not None and isinstance(current, int) and current > cap:
93
+ if limits.get("on_exceed") == "reject":
94
+ raise InferenceRejected(f"max_tokens {current} is above the limit {cap}")
95
+ out["max_tokens"] = cap
96
+ return out
97
+ if op == "embeddings" and "dimensions" in defaults:
98
+ out.setdefault("dimensions", defaults["dimensions"])
99
+ if op == "invoke_model" and isinstance(out.get("body"), Mapping):
100
+ body = dict(out["body"])
101
+ names = ("dimensions", "normalize", "input_type", "truncate", "embedding_types")
102
+ for name in names:
103
+ if name in defaults and (name in locked or name not in body):
104
+ body[name] = defaults[name]
105
+ if "inputText" in body: # Titan takes dimensions / normalize only
106
+ body = {k: v for k, v in body.items() if k not in ("input_type", "truncate")}
107
+ else: # Cohere takes input_type / truncate / embedding_types only
108
+ body = {k: v for k, v in body.items() if k not in ("dimensions", "normalize")}
109
+ out["body"] = body
110
+ return out
111
+
112
+
113
+ def merge_params(request: Mapping[str, Any], extra: Mapping[str, Any]) -> dict[str, Any]:
114
+ """Merge a deployment's `params` (guardrail, model fields) into a request, one level deep."""
115
+ out = dict(request)
116
+ for key, value in extra.items():
117
+ if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
118
+ out[key] = {**out[key], **value}
119
+ else:
120
+ out[key] = copy.deepcopy(value)
121
+ return out
@@ -0,0 +1,137 @@
1
+ """Jobs on the stream queue (plan E136): `register_job` / `send_job`, run at least once.
2
+
3
+ A job is any registered async function taking one JSON payload. `send_job` puts it on
4
+ `<queue_prefix>jobs-<name>`; workers of processes that registered the name run it. A failed
5
+ job is retried after a backoff and goes to the dead-letter stream after `max_receives`; a held
6
+ job's visibility is extended while it runs. Jobs take no LLM budget (their own LLM calls do).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import contextlib
13
+ from collections.abc import Awaitable, Callable, Mapping
14
+ from typing import TYPE_CHECKING, Any
15
+
16
+ import orjson
17
+
18
+ from llm_queue._logging import get_logger
19
+ from llm_queue.engine.queue import Message, QueueSettings, StreamQueues
20
+ from llm_queue.errors import ConfigError
21
+ from llm_queue.ids import new_id
22
+
23
+ if TYPE_CHECKING:
24
+ pass
25
+
26
+ logger = get_logger(__name__)
27
+ JobFn = Callable[[dict[str, Any]], Awaitable[Any]]
28
+
29
+
30
+ class JobRunner:
31
+ """Registered jobs of one process, their senders and their worker loops."""
32
+
33
+ def __init__(
34
+ self,
35
+ queues: StreamQueues,
36
+ *,
37
+ queue_prefix: str = "",
38
+ settings: QueueSettings | None = None,
39
+ concurrency: int = 4,
40
+ ) -> None:
41
+ """Nothing runs until `start`."""
42
+ self.queues = queues
43
+ self.queue_prefix = queue_prefix
44
+ self.settings = settings or QueueSettings(visibility_secs=60, max_receives=5, max_waiting=0)
45
+ self.concurrency = concurrency
46
+ self._jobs: dict[str, JobFn] = {}
47
+ self._tasks: list[asyncio.Task[None]] = []
48
+ self._running: set[asyncio.Task[None]] = set()
49
+ self._stopping = False
50
+ self._paused = False
51
+
52
+ def queue(self, name: str) -> str:
53
+ """The queue of one job name."""
54
+ return f"{self.queue_prefix}jobs-{name}"
55
+
56
+ def register(self, name: str, fn: JobFn) -> None:
57
+ """Allow this process to run job `name` (before `start`)."""
58
+ if not name or not callable(fn):
59
+ raise ConfigError("a job needs a name and an async function")
60
+ self._jobs[name] = fn
61
+
62
+ async def send(
63
+ self, name: str, payload: Mapping[str, Any], *, job_id: str | None = None
64
+ ) -> str:
65
+ """Queue a job; returns its id (jobs run at least once: make them idempotent)."""
66
+ ident = job_id or new_id()
67
+ body = orjson.dumps({"id": ident, "name": name, "payload": payload})
68
+ await self.queues.send(self.queue(name), body, self.settings)
69
+ return ident
70
+
71
+ async def start(self) -> None:
72
+ """One worker loop per registered job."""
73
+ for name in self._jobs:
74
+ self._tasks.append(asyncio.create_task(self._loop(name)))
75
+
76
+ def pause(self) -> None:
77
+ """Take no new jobs; running ones go on (`stop()` follows)."""
78
+ self._paused = True
79
+
80
+ async def stop(self, drain_secs: float = 10.0) -> None:
81
+ """Stop receiving, let running jobs finish (up to `drain_secs`), then stop."""
82
+ self._stopping = True
83
+ if self._running:
84
+ await asyncio.wait(set(self._running), timeout=drain_secs)
85
+ for task in [*self._tasks, *self._running]:
86
+ task.cancel()
87
+ for task in [*self._tasks, *self._running]:
88
+ with contextlib.suppress(asyncio.CancelledError, Exception):
89
+ await task
90
+
91
+ async def _loop(self, name: str) -> None:
92
+ gate = asyncio.Semaphore(self.concurrency)
93
+ while not self._stopping:
94
+ if self._paused:
95
+ await asyncio.sleep(0.2)
96
+ continue
97
+ try:
98
+ await gate.acquire()
99
+ messages, dead = await self.queues.receive_wait(
100
+ self.queue(name), 1, self.settings, wait_secs=1.0
101
+ )
102
+ for letter in dead:
103
+ logger.error("job_dead_lettered", job=name, message=letter.id)
104
+ if not messages:
105
+ gate.release()
106
+ continue
107
+ task = asyncio.create_task(self._run(name, messages[0], gate))
108
+ self._running.add(task)
109
+ task.add_done_callback(self._running.discard)
110
+ except Exception as exc: # a store blip must not end the loop
111
+ gate.release()
112
+ logger.warning("job_receive_failed", job=name, error=str(exc))
113
+ await asyncio.sleep(1)
114
+
115
+ async def _run(self, name: str, message: Message, gate: asyncio.Semaphore) -> None:
116
+ keep = asyncio.create_task(self._keep_visible(message))
117
+ try:
118
+ body = orjson.loads(message.body)
119
+ await self._jobs[name](body["payload"])
120
+ await self.queues.ack(message)
121
+ logger.info("job_done", job=name, id=body.get("id"))
122
+ except Exception as exc:
123
+ outcome = await self.queues.fail(message, self.settings, delay_secs=5.0)
124
+ logger.warning("job_failed", job=name, error=str(exc), outcome=outcome)
125
+ finally:
126
+ keep.cancel()
127
+ gate.release()
128
+
129
+ async def _keep_visible(self, message: Message) -> None:
130
+ """Keep a running job hidden; a store blip must not end it (the job would run twice)."""
131
+ while True:
132
+ await asyncio.sleep(max(1.0, self.settings.visibility_secs / 3))
133
+ try:
134
+ if not await self.queues.extend(message, self.settings.visibility_secs):
135
+ logger.warning("job_lost_visibility", queue=message.queue, message=message.id)
136
+ except Exception as exc:
137
+ logger.warning("job_extend_failed", message=message.id, error=str(exc))
@@ -0,0 +1,70 @@
1
+ """Key names of the 0.3 engine (key schema 3), all built from one environment prefix.
2
+
3
+ Keys of one model share the tag `{m:ALIAS}` and keys of one request `{r:ID}`; the admission and
4
+ finish scripts touch both, which is fine on the single-primary store 0.3 supports (plan §3.2).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass
10
+
11
+
12
+ @dataclass(frozen=True, slots=True)
13
+ class Keys:
14
+ """Key builder for one prefix, e.g. `llmq:prod` -> `llmq:prod:{m:gpt120b}:wait:p`."""
15
+
16
+ prefix: str
17
+
18
+ # ---------------------------------------------------------------- per model
19
+
20
+ def model(self, alias: str, name: str) -> str:
21
+ """A key of one model, e.g. `cfg`, `rot`, `health`, `inprog`, `wait:p`."""
22
+ return f"{self.prefix}:{{m:{alias}}}:{name}"
23
+
24
+ def rate(self, alias: str, deployment: str) -> str:
25
+ """RPM / TPM budget of one deployment (hash of clocks and minute counters)."""
26
+ return self.model(alias, f"rate:{deployment}")
27
+
28
+ def limiter(self, alias: str, deployment: str) -> str:
29
+ """Adaptive concurrency limit of one deployment (hash)."""
30
+ return self.model(alias, f"ac:{deployment}")
31
+
32
+ def breaker(self, alias: str, deployment: str) -> str:
33
+ """Circuit breaker of one deployment (hash)."""
34
+ return self.model(alias, f"cb:{deployment}")
35
+
36
+ # ---------------------------------------------------------------- per request
37
+
38
+ def request(self, request_id: str, name: str) -> str:
39
+ """A key of one request: `job` (claim), `res` (result), `params`, `cancel`, `chunks`."""
40
+ return f"{self.prefix}:{{r:{request_id}}}:{name}"
41
+
42
+ def status(self, request_id: str) -> str:
43
+ """Request status (hash, `request_status`)."""
44
+ return self.request(request_id, "status")
45
+
46
+ # ---------------------------------------------------------------- per process
47
+
48
+ def caller(self, caller: str) -> str:
49
+ """Liveness of one calling process (`pod:boot`)."""
50
+ return f"{self.prefix}:caller:{caller}"
51
+
52
+ def done_channel(self, caller: str) -> str:
53
+ """Pub/sub channel that wakes one caller's result reader."""
54
+ return f"{self.prefix}:done:{caller}"
55
+
56
+ @property
57
+ def cancel_channel(self) -> str:
58
+ """Pub/sub channel naming request ids just cancelled."""
59
+ return f"{self.prefix}:cancel"
60
+
61
+ @property
62
+ def epoch(self) -> str:
63
+ """Set once per store life; missing means the store restarted or lost its data."""
64
+ return f"{self.prefix}:epoch"
65
+
66
+ # ---------------------------------------------------------------- queues
67
+
68
+ def queue(self, name: str, part: str) -> str:
69
+ """A key of one queue: `s` (stream), `vis`, `gen`, `fails`, `st`, `dlq`, `meta`."""
70
+ return f"{self.prefix}:{{q:{name}}}:{part}"
@@ -0,0 +1,141 @@
1
+ """Metric sinks (plan E131): CloudWatch embedded metric format by default, Prometheus optional.
2
+
3
+ The engine calls `count`, `observe` and `gauge` with labels model, account, region (and
4
+ priority on waits). `EmfSink` writes one JSON line per point that CloudWatch turns into a
5
+ metric; `PrometheusSink` keeps `prometheus_client` metrics for a `/metrics` endpoint.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import sys
11
+ import time
12
+ from collections.abc import Callable, Mapping
13
+ from typing import Any, Protocol
14
+
15
+ import orjson
16
+
17
+ from llm_queue.errors import MissingExtra
18
+
19
+ NAMESPACE = "llm-queue"
20
+
21
+
22
+ class Metrics(Protocol):
23
+ """What the engine needs from a metrics sink."""
24
+
25
+ def count(self, name: str, value: float = 1.0, **labels: str) -> None:
26
+ """Add to a counter."""
27
+ ...
28
+
29
+ def observe(self, name: str, value: float, **labels: str) -> None:
30
+ """Record one sample of a distribution (seconds, tokens)."""
31
+ ...
32
+
33
+ def gauge(self, name: str, value: float, **labels: str) -> None:
34
+ """Set a gauge."""
35
+ ...
36
+
37
+
38
+ class NullSink:
39
+ """Drops every point (tests, or metrics turned off)."""
40
+
41
+ def count(self, name: str, value: float = 1.0, **labels: str) -> None:
42
+ """Ignore."""
43
+
44
+ def observe(self, name: str, value: float, **labels: str) -> None:
45
+ """Ignore."""
46
+
47
+ def gauge(self, name: str, value: float, **labels: str) -> None:
48
+ """Ignore."""
49
+
50
+
51
+ class EmfSink:
52
+ """CloudWatch embedded metric format: one JSON line per point on `write` (stdout)."""
53
+
54
+ def __init__(
55
+ self,
56
+ write: Callable[[str], Any] | None = None,
57
+ namespace: str = NAMESPACE,
58
+ static: Mapping[str, str] | None = None,
59
+ ) -> None:
60
+ """`write` receives each line (default: stdout); `static` labels go on every point."""
61
+ self._write = write or (lambda line: sys.stdout.write(line + "\n"))
62
+ self.namespace = namespace
63
+ self.static = dict(static or {})
64
+
65
+ def _emit(self, name: str, value: float, unit: str, labels: Mapping[str, str]) -> None:
66
+ labels = {**self.static, **labels}
67
+ dims = sorted(labels)
68
+ line = {
69
+ "_aws": {
70
+ "Timestamp": int(time.time() * 1000),
71
+ "CloudWatchMetrics": [
72
+ {
73
+ "Namespace": self.namespace,
74
+ "Dimensions": [dims],
75
+ "Metrics": [{"Name": name, "Unit": unit}],
76
+ }
77
+ ],
78
+ },
79
+ name: value,
80
+ **labels,
81
+ }
82
+ self._write(orjson.dumps(line).decode()) # compact; 0.8 us vs 6.2 us for json.dumps
83
+
84
+ def count(self, name: str, value: float = 1.0, **labels: str) -> None:
85
+ """A count point."""
86
+ self._emit(name, value, "Count", labels)
87
+
88
+ def observe(self, name: str, value: float, **labels: str) -> None:
89
+ """A sample point (seconds for `_seconds` names)."""
90
+ self._emit(name, value, "Seconds" if name.endswith("_seconds") else "None", labels)
91
+
92
+ def gauge(self, name: str, value: float, **labels: str) -> None:
93
+ """A gauge point."""
94
+ self._emit(name, value, "None", labels)
95
+
96
+
97
+ class PrometheusSink:
98
+ """`prometheus_client` metrics (needs the `metrics` extra); serve them with its exposition."""
99
+
100
+ def __init__(self, registry: Any = None) -> None:
101
+ """Create metrics lazily, by name and label set."""
102
+ try:
103
+ import prometheus_client
104
+ except ImportError as exc:
105
+ raise MissingExtra("PrometheusSink", "metrics") from exc
106
+
107
+ self._prom = prometheus_client
108
+ self._registry = registry or prometheus_client.REGISTRY
109
+ self._made: dict[str, Any] = {}
110
+
111
+ def _metric(self, kind: str, name: str, labels: Mapping[str, str]) -> Any:
112
+ if name not in self._made:
113
+ cls = {
114
+ "count": self._prom.Counter,
115
+ "observe": self._prom.Histogram,
116
+ "gauge": self._prom.Gauge,
117
+ }[kind]
118
+ self._made[name] = cls(name, name, sorted(labels), registry=self._registry)
119
+ metric = self._made[name]
120
+ return metric.labels(**labels) if labels else metric
121
+
122
+ def count(self, name: str, value: float = 1.0, **labels: str) -> None:
123
+ """Add to a counter."""
124
+ self._metric("count", name.removesuffix("_total"), labels).inc(value)
125
+
126
+ def observe(self, name: str, value: float, **labels: str) -> None:
127
+ """Observe into a histogram."""
128
+ self._metric("observe", name, labels).observe(value)
129
+
130
+ def gauge(self, name: str, value: float, **labels: str) -> None:
131
+ """Set a gauge."""
132
+ self._metric("gauge", name, labels).set(value)
133
+
134
+
135
+ def sink_for(name: str) -> Metrics:
136
+ """The sink named by the `metrics_sink` setting: `emf`, `prometheus` or `none`."""
137
+ if name == "prometheus":
138
+ return PrometheusSink()
139
+ if name == "none":
140
+ return NullSink()
141
+ return EmfSink()
@@ -0,0 +1,82 @@
1
+ """Large results in object storage (plan E137): the state store keeps only a reference.
2
+
3
+ A result above `result_offload_bytes` is written to an S3 bucket (encrypted, with a one-day
4
+ lifecycle rule set on the bucket by the owner) and the stored reply is a small marker. Readers
5
+ fetch it transparently. Used for bulk embedding batches, whose vectors are large.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import TYPE_CHECKING, Any, Protocol
11
+
12
+ if TYPE_CHECKING:
13
+ from aiobotocore.session import AioSession
14
+
15
+ MARKER = b"llmq-offload:"
16
+
17
+
18
+ class ResultOffload(Protocol):
19
+ """Somewhere to keep a large result body."""
20
+
21
+ async def put(self, key: str, data: bytes) -> str:
22
+ """Store `data`; return a reference."""
23
+ ...
24
+
25
+ async def get(self, ref: str) -> bytes | None:
26
+ """The data of a reference (None when it expired)."""
27
+ ...
28
+
29
+
30
+ class S3Offload:
31
+ """`ResultOffload` on an S3 bucket (server-side encryption; expiry by the bucket's rule)."""
32
+
33
+ def __init__(
34
+ self,
35
+ bucket: str,
36
+ *,
37
+ prefix: str = "llmq-results/",
38
+ session: AioSession | None = None,
39
+ client_kwargs: dict[str, Any] | None = None,
40
+ ) -> None:
41
+ """Keep settings; the client is created on first use."""
42
+ self.bucket = bucket
43
+ self.prefix = prefix
44
+ self._session = session
45
+ self._kwargs = dict(client_kwargs or {})
46
+ self._client: Any = None
47
+ self._context: Any = None
48
+
49
+ async def _s3(self) -> Any:
50
+ if self._client is None:
51
+ from aiobotocore.session import get_session
52
+
53
+ session = self._session or get_session()
54
+ self._context = session.create_client("s3", **self._kwargs)
55
+ self._client = await self._context.__aenter__()
56
+ return self._client
57
+
58
+ async def put(self, key: str, data: bytes) -> str:
59
+ """Write the object (AES256 server-side encryption)."""
60
+ s3 = await self._s3()
61
+ name = f"{self.prefix}{key}"
62
+ await s3.put_object(Bucket=self.bucket, Key=name, Body=data, ServerSideEncryption="AES256")
63
+ return name
64
+
65
+ async def get(self, ref: str) -> bytes | None:
66
+ """Read the object; None when it is gone."""
67
+ s3 = await self._s3()
68
+ try:
69
+ reply = await s3.get_object(Bucket=self.bucket, Key=ref)
70
+ except Exception as exc:
71
+ if "NoSuchKey" in str(exc) or "404" in str(exc):
72
+ return None
73
+ raise
74
+ async with reply["Body"] as body:
75
+ data: bytes = await body.read()
76
+ return data
77
+
78
+ async def aclose(self) -> None:
79
+ """Close the client."""
80
+ if self._context is not None:
81
+ await self._context.__aexit__(None, None, None)
82
+ self._client = self._context = None