021-python-clients 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- 021_python_clients-0.1.0.dist-info/METADATA +327 -0
- 021_python_clients-0.1.0.dist-info/RECORD +21 -0
- 021_python_clients-0.1.0.dist-info/WHEEL +4 -0
- 021_python_clients-0.1.0.dist-info/licenses/LICENSE +202 -0
- clients_021/__init__.py +22 -0
- clients_021/_aws.py +55 -0
- clients_021/_observe.py +106 -0
- clients_021/email/__init__.py +16 -0
- clients_021/email/_message.py +77 -0
- clients_021/email/resend.py +143 -0
- clients_021/email/ses.py +119 -0
- clients_021/errors.py +20 -0
- clients_021/http.py +131 -0
- clients_021/httpx2.py +93 -0
- clients_021/kms.py +178 -0
- clients_021/py.typed +0 -0
- clients_021/queue/__init__.py +6 -0
- clients_021/queue/qstash.py +219 -0
- clients_021/queue/sqs.py +177 -0
- clients_021/redis.py +124 -0
- clients_021/s3.py +164 -0
clients_021/httpx2.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""An observed httpx2 transport, for SDKs built on httpx2 (Anthropic, OpenAI and others).
|
|
2
|
+
|
|
3
|
+
from anthropic import AsyncAnthropic, DefaultAsyncHttpxClient
|
|
4
|
+
from clients_021.httpx2 import ObservedTransport
|
|
5
|
+
|
|
6
|
+
transport = ObservedTransport(obs, dependency="anthropic")
|
|
7
|
+
client = AsyncAnthropic(http_client=DefaultAsyncHttpxClient(transport=transport))
|
|
8
|
+
|
|
9
|
+
Needs the `httpx2` extra. Records the same `http.client.request.*` instruments and CLIENT
|
|
10
|
+
span as 021-python-observability's httpx transport, under `dependency`, with method,
|
|
11
|
+
status and a bounded error type; never the URL. Trace context is not sent by default: these
|
|
12
|
+
are third-party APIs (`propagate=True` sends `traceparent` to services you own).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
import time
|
|
19
|
+
|
|
20
|
+
import httpx2
|
|
21
|
+
from observability_021 import Observability
|
|
22
|
+
from observability_021.config import identifier
|
|
23
|
+
from opentelemetry.trace import SpanKind, StatusCode
|
|
24
|
+
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
25
|
+
|
|
26
|
+
_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "POST", "PUT", "PATCH", "DELETE"})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ObservedTransport(httpx2.AsyncBaseTransport):
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
obs: Observability,
|
|
33
|
+
*,
|
|
34
|
+
dependency: str,
|
|
35
|
+
transport: httpx2.AsyncBaseTransport | None = None,
|
|
36
|
+
propagate: bool = False,
|
|
37
|
+
) -> None:
|
|
38
|
+
self._obs = obs
|
|
39
|
+
self._dependency = identifier(dependency)
|
|
40
|
+
self._transport = transport if transport is not None else httpx2.AsyncHTTPTransport()
|
|
41
|
+
self._propagate = propagate
|
|
42
|
+
|
|
43
|
+
async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
|
|
44
|
+
method = request.method if request.method in _METHODS else "_OTHER"
|
|
45
|
+
attributes: dict[str, str | int] = {
|
|
46
|
+
"dependency": self._dependency,
|
|
47
|
+
"http.request.method": method,
|
|
48
|
+
}
|
|
49
|
+
started = time.perf_counter()
|
|
50
|
+
tracer = self._obs.tracer
|
|
51
|
+
with tracer.start_as_current_span(
|
|
52
|
+
f"HTTP {self._dependency}",
|
|
53
|
+
kind=SpanKind.CLIENT,
|
|
54
|
+
attributes=attributes,
|
|
55
|
+
record_exception=False,
|
|
56
|
+
set_status_on_exception=False,
|
|
57
|
+
) as span:
|
|
58
|
+
if self._propagate:
|
|
59
|
+
# Replace the whole pair: the new parent may have no tracestate.
|
|
60
|
+
request.headers.pop("traceparent", None)
|
|
61
|
+
request.headers.pop("tracestate", None)
|
|
62
|
+
carrier: dict[str, str] = {}
|
|
63
|
+
TraceContextTextMapPropagator().inject(carrier)
|
|
64
|
+
for key, value in carrier.items():
|
|
65
|
+
request.headers[key] = value
|
|
66
|
+
try:
|
|
67
|
+
response = await self._transport.handle_async_request(request)
|
|
68
|
+
except BaseException as error:
|
|
69
|
+
attributes["error.type"] = (
|
|
70
|
+
"cancelled"
|
|
71
|
+
if isinstance(error, asyncio.CancelledError)
|
|
72
|
+
else "timeout"
|
|
73
|
+
if isinstance(error, httpx2.TimeoutException)
|
|
74
|
+
else "connection_error"
|
|
75
|
+
if isinstance(
|
|
76
|
+
error, httpx2.NetworkError | httpx2.RemoteProtocolError | httpx2.ProxyError
|
|
77
|
+
)
|
|
78
|
+
else type(error).__name__
|
|
79
|
+
)
|
|
80
|
+
span.set_status(StatusCode.ERROR)
|
|
81
|
+
raise
|
|
82
|
+
else:
|
|
83
|
+
attributes["http.response.status_code"] = response.status_code
|
|
84
|
+
if response.status_code >= 400:
|
|
85
|
+
span.set_status(StatusCode.ERROR)
|
|
86
|
+
return response
|
|
87
|
+
finally:
|
|
88
|
+
span.set_attributes(attributes)
|
|
89
|
+
self._obs.client_requests.add(1, attributes)
|
|
90
|
+
self._obs.client_duration.record(time.perf_counter() - started, attributes)
|
|
91
|
+
|
|
92
|
+
async def aclose(self) -> None:
|
|
93
|
+
await self._transport.aclose()
|
clients_021/kms.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""AWS KMS, observed: encrypt, decrypt and data keys, always bound to an encryption context.
|
|
2
|
+
|
|
3
|
+
from clients_021.kms import KmsClient
|
|
4
|
+
|
|
5
|
+
keys = KmsClient.create(obs, settings.session_key_id, region="us-east-1")
|
|
6
|
+
context = {"purpose": "session", "tenant": tenant_id}
|
|
7
|
+
blob = await keys.encrypt(secret, context=context)
|
|
8
|
+
secret = await keys.decrypt(blob, context=context)
|
|
9
|
+
envelope = await keys.seal(large_payload, context=context) # any size, AES-256-GCM
|
|
10
|
+
large_payload = await keys.unseal(envelope, context=context)
|
|
11
|
+
|
|
12
|
+
Needs the `aws` extra. The encryption context is required: decrypting with another context
|
|
13
|
+
fails, so a ciphertext only opens for the purpose it was made for. KMS logs it in CloudTrail
|
|
14
|
+
in plaintext: it identifies the purpose, it is never a secret. Calls record
|
|
15
|
+
`crypto.client.operation.*` with `operation` (`encrypt`, `decrypt`, `generate_data_key`) and
|
|
16
|
+
a span with `rpc.service=KMS`; never key ids, contexts, plaintext or ciphertext.
|
|
17
|
+
Timeouts, connection failures, throttling and 5xx raise DependencyUnavailable. Access
|
|
18
|
+
denied, a disabled key, a wrong context or a tampered ciphertext (`InvalidCiphertextException`)
|
|
19
|
+
and a ciphertext from another key (`IncorrectKeyException`) keep their botocore type.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import struct
|
|
28
|
+
from collections.abc import Mapping
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from typing import TYPE_CHECKING, Any
|
|
31
|
+
|
|
32
|
+
import boto3
|
|
33
|
+
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
34
|
+
from observability_021 import Observability
|
|
35
|
+
from observability_021.config import identifier
|
|
36
|
+
|
|
37
|
+
from clients_021 import _aws
|
|
38
|
+
from clients_021._observe import observed
|
|
39
|
+
|
|
40
|
+
if TYPE_CHECKING:
|
|
41
|
+
from mypy_boto3_kms import KMSClient
|
|
42
|
+
else:
|
|
43
|
+
KMSClient = Any
|
|
44
|
+
|
|
45
|
+
SYSTEM = {"rpc.system": "aws-api", "rpc.service": "KMS"}
|
|
46
|
+
# Envelope layout: version (1 byte), wrapped key length (2 bytes, big endian), wrapped key,
|
|
47
|
+
# nonce (12 bytes), AES-256-GCM ciphertext and tag. The header and context are the AAD.
|
|
48
|
+
_ENVELOPE = struct.Struct(">BH")
|
|
49
|
+
_VERSION = 1
|
|
50
|
+
_NONCE = 12
|
|
51
|
+
_TAG = 16
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True, repr=False)
|
|
55
|
+
class DataKey:
|
|
56
|
+
"""A data key: use `plaintext` locally and drop it; store only `ciphertext`."""
|
|
57
|
+
|
|
58
|
+
plaintext: bytes
|
|
59
|
+
ciphertext: bytes
|
|
60
|
+
|
|
61
|
+
def __repr__(self) -> str:
|
|
62
|
+
return "DataKey(<redacted>)"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _context(context: Mapping[str, str]) -> dict[str, str]:
|
|
66
|
+
if not context or not all(context):
|
|
67
|
+
raise ValueError("context must be a non-empty mapping with non-empty keys")
|
|
68
|
+
return dict(context)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _aad(header: bytes, context: dict[str, str]) -> bytes:
|
|
72
|
+
canonical = json.dumps(context, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
73
|
+
return header + canonical.encode()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class KmsClient:
|
|
77
|
+
def __init__(
|
|
78
|
+
self, obs: Observability, client: KMSClient, key_id: str, *, dependency: str = "kms"
|
|
79
|
+
) -> None:
|
|
80
|
+
if not key_id:
|
|
81
|
+
raise ValueError("key_id is required")
|
|
82
|
+
self._obs = obs
|
|
83
|
+
self._client = client
|
|
84
|
+
self._key_id = key_id
|
|
85
|
+
self.dependency = identifier(dependency)
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def create(
|
|
89
|
+
cls,
|
|
90
|
+
obs: Observability,
|
|
91
|
+
key_id: str,
|
|
92
|
+
*,
|
|
93
|
+
region: str | None = None,
|
|
94
|
+
endpoint_url: str | None = None,
|
|
95
|
+
dependency: str = "kms",
|
|
96
|
+
timeout_seconds: float = 5.0,
|
|
97
|
+
) -> KmsClient:
|
|
98
|
+
client: KMSClient = boto3.client( # pyright: ignore[reportUnknownMemberType]
|
|
99
|
+
"kms",
|
|
100
|
+
region_name=region,
|
|
101
|
+
endpoint_url=endpoint_url,
|
|
102
|
+
config=_aws.config(
|
|
103
|
+
connect_seconds=min(timeout_seconds, 3.0), read_seconds=timeout_seconds
|
|
104
|
+
),
|
|
105
|
+
)
|
|
106
|
+
return cls(obs, client, key_id, dependency=dependency)
|
|
107
|
+
|
|
108
|
+
def _observed(self, operation: str): # noqa: ANN202 - an async context manager
|
|
109
|
+
return observed(
|
|
110
|
+
self._obs,
|
|
111
|
+
family="crypto",
|
|
112
|
+
system=SYSTEM,
|
|
113
|
+
dependency=self.dependency,
|
|
114
|
+
operation=operation,
|
|
115
|
+
classify=_aws.classify,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
async def encrypt(self, plaintext: bytes, *, context: Mapping[str, str]) -> bytes:
|
|
119
|
+
"""Up to 4096 bytes (a KMS limit); use `seal` for anything larger."""
|
|
120
|
+
encryption_context = _context(context)
|
|
121
|
+
async with self._observed("encrypt"):
|
|
122
|
+
response = await asyncio.to_thread(
|
|
123
|
+
self._client.encrypt,
|
|
124
|
+
KeyId=self._key_id,
|
|
125
|
+
Plaintext=plaintext,
|
|
126
|
+
EncryptionContext=encryption_context,
|
|
127
|
+
)
|
|
128
|
+
return response["CiphertextBlob"]
|
|
129
|
+
|
|
130
|
+
async def decrypt(self, ciphertext: bytes, *, context: Mapping[str, str]) -> bytes:
|
|
131
|
+
"""Names the key, as KMS recommends: a ciphertext from another key is rejected."""
|
|
132
|
+
encryption_context = _context(context)
|
|
133
|
+
async with self._observed("decrypt"):
|
|
134
|
+
response = await asyncio.to_thread(
|
|
135
|
+
self._client.decrypt,
|
|
136
|
+
KeyId=self._key_id,
|
|
137
|
+
CiphertextBlob=ciphertext,
|
|
138
|
+
EncryptionContext=encryption_context,
|
|
139
|
+
)
|
|
140
|
+
return response["Plaintext"]
|
|
141
|
+
|
|
142
|
+
async def generate_data_key(self, *, context: Mapping[str, str]) -> DataKey:
|
|
143
|
+
"""A fresh AES-256 key, in plaintext and wrapped under this client's KMS key."""
|
|
144
|
+
encryption_context = _context(context)
|
|
145
|
+
async with self._observed("generate_data_key"):
|
|
146
|
+
response = await asyncio.to_thread(
|
|
147
|
+
self._client.generate_data_key,
|
|
148
|
+
KeyId=self._key_id,
|
|
149
|
+
KeySpec="AES_256",
|
|
150
|
+
EncryptionContext=encryption_context,
|
|
151
|
+
)
|
|
152
|
+
return DataKey(response["Plaintext"], response["CiphertextBlob"])
|
|
153
|
+
|
|
154
|
+
async def seal(self, plaintext: bytes, *, context: Mapping[str, str]) -> bytes:
|
|
155
|
+
"""Envelope encryption: a new data key per call (one KMS request), AES-256-GCM locally."""
|
|
156
|
+
encryption_context = _context(context)
|
|
157
|
+
key = await self.generate_data_key(context=encryption_context)
|
|
158
|
+
header = _ENVELOPE.pack(_VERSION, len(key.ciphertext)) + key.ciphertext
|
|
159
|
+
nonce = os.urandom(_NONCE)
|
|
160
|
+
sealed = AESGCM(key.plaintext).encrypt(nonce, plaintext, _aad(header, encryption_context))
|
|
161
|
+
return header + nonce + sealed
|
|
162
|
+
|
|
163
|
+
async def unseal(self, envelope: bytes, *, context: Mapping[str, str]) -> bytes:
|
|
164
|
+
"""Raises ValueError for a malformed envelope, cryptography's InvalidTag if tampered."""
|
|
165
|
+
encryption_context = _context(context)
|
|
166
|
+
if len(envelope) < _ENVELOPE.size:
|
|
167
|
+
raise ValueError("not a sealed envelope")
|
|
168
|
+
version, size = _ENVELOPE.unpack_from(envelope)
|
|
169
|
+
end = _ENVELOPE.size + size
|
|
170
|
+
if version != _VERSION or size == 0 or len(envelope) < end + _NONCE + _TAG:
|
|
171
|
+
raise ValueError("not a sealed envelope")
|
|
172
|
+
key = await self.decrypt(envelope[_ENVELOPE.size : end], context=encryption_context)
|
|
173
|
+
nonce, sealed = envelope[end : end + _NONCE], envelope[end + _NONCE :]
|
|
174
|
+
return AESGCM(key).decrypt(nonce, sealed, _aad(envelope[:end], encryption_context))
|
|
175
|
+
|
|
176
|
+
async def aclose(self) -> None:
|
|
177
|
+
"""Release the underlying SDK connection pools after in-flight calls finish."""
|
|
178
|
+
await asyncio.to_thread(self._client.close)
|
clients_021/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Queue providers grouped by capability, with provider-specific delivery models.
|
|
2
|
+
|
|
3
|
+
Import `SqsClient` from `clients_021.queue.sqs` (requires the `aws` extra) or
|
|
4
|
+
`QStashClient` from `clients_021.queue.qstash` (no extra). SQS is polled by consumers;
|
|
5
|
+
QStash delivers to HTTP endpoints. Importing this package loads neither provider SDK.
|
|
6
|
+
"""
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""QStash JSON publishing over the shared HTTP client; no optional SDK is required.
|
|
2
|
+
|
|
3
|
+
Publishing is one POST. Delivery retries belong to QStash; publication retries belong
|
|
4
|
+
to the caller, which must reuse the same deduplication id after an uncertain outcome.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Any, cast
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
from observability_021 import Observability
|
|
18
|
+
from observability_021.config import identifier
|
|
19
|
+
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
20
|
+
|
|
21
|
+
from clients_021._observe import observed
|
|
22
|
+
from clients_021.errors import DependencyUnavailable
|
|
23
|
+
from clients_021.http import HttpClient, HttpResponse
|
|
24
|
+
|
|
25
|
+
_SYSTEM = {"messaging.system": "upstash_qstash"}
|
|
26
|
+
_HEADER_NAME = re.compile(r"[!#$%&'*+.^_`|~0-9a-zA-Z-]+")
|
|
27
|
+
_TRACE_HEADERS = {"traceparent", "tracestate"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class PublishedMessage:
|
|
32
|
+
message_id: str
|
|
33
|
+
deduplicated: bool
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class QStashResponseError(Exception):
|
|
37
|
+
"""A rejected request or invalid acknowledgement, without the provider's body."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, status_code: int, *, reason: str = "rejected") -> None:
|
|
40
|
+
super().__init__(f"QStash publish: {reason} (HTTP {status_code})")
|
|
41
|
+
self.status_code = status_code
|
|
42
|
+
self.reason = reason
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _classify(error: BaseException) -> str | None:
|
|
46
|
+
if isinstance(error, DependencyUnavailable):
|
|
47
|
+
return error.reason
|
|
48
|
+
if isinstance(error, QStashResponseError):
|
|
49
|
+
if error.status_code == 429:
|
|
50
|
+
return "throttled"
|
|
51
|
+
if 500 <= error.status_code < 600:
|
|
52
|
+
return f"status_{error.status_code}"
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _url(value: str, *, name: str, base: bool = False) -> str:
|
|
57
|
+
try:
|
|
58
|
+
parsed = urlsplit(value)
|
|
59
|
+
port = parsed.port # Invalid ports must fail before a request is constructed.
|
|
60
|
+
valid = (
|
|
61
|
+
parsed.scheme in {"https", "http"}
|
|
62
|
+
and parsed.hostname
|
|
63
|
+
and parsed.username is None
|
|
64
|
+
and parsed.password is None
|
|
65
|
+
and not parsed.fragment
|
|
66
|
+
and not (base and parsed.query)
|
|
67
|
+
and (port is None or port > 0)
|
|
68
|
+
and not any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
|
|
69
|
+
)
|
|
70
|
+
except ValueError:
|
|
71
|
+
valid = False
|
|
72
|
+
if not valid:
|
|
73
|
+
raise ValueError(f"{name} must be an HTTP(S) URL without credentials or a fragment")
|
|
74
|
+
return value.rstrip("/") if base else value
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _header_value(value: str, *, name: str) -> str:
|
|
78
|
+
if (
|
|
79
|
+
not value
|
|
80
|
+
or value != value.strip()
|
|
81
|
+
or not value.isascii()
|
|
82
|
+
or any(ord(char) < 32 or ord(char) == 127 for char in value)
|
|
83
|
+
):
|
|
84
|
+
raise ValueError(f"{name} must be a non-empty printable ASCII header value")
|
|
85
|
+
return value
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _integer(value: int, *, name: str, minimum: int) -> None:
|
|
89
|
+
if type(value) is not int or value < minimum:
|
|
90
|
+
raise ValueError(f"{name} must be an integer of at least {minimum}")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _published(response: HttpResponse) -> PublishedMessage:
|
|
94
|
+
try:
|
|
95
|
+
data: Any = response.json()
|
|
96
|
+
except (ValueError, UnicodeDecodeError):
|
|
97
|
+
raise QStashResponseError(response.status_code, reason="invalid_response") from None
|
|
98
|
+
if not isinstance(data, dict):
|
|
99
|
+
raise QStashResponseError(response.status_code, reason="invalid_response")
|
|
100
|
+
payload = cast(dict[str, Any], data)
|
|
101
|
+
message_id = payload.get("messageId")
|
|
102
|
+
deduplicated = payload.get("deduplicated", response.status_code == 202)
|
|
103
|
+
if (
|
|
104
|
+
not isinstance(message_id, str)
|
|
105
|
+
or not message_id.strip()
|
|
106
|
+
or not isinstance(deduplicated, bool)
|
|
107
|
+
):
|
|
108
|
+
raise QStashResponseError(response.status_code, reason="invalid_response")
|
|
109
|
+
return PublishedMessage(message_id, deduplicated)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class QStashClient:
|
|
113
|
+
def __init__(
|
|
114
|
+
self,
|
|
115
|
+
obs: Observability,
|
|
116
|
+
*,
|
|
117
|
+
token: str,
|
|
118
|
+
base_url: str,
|
|
119
|
+
timeout_seconds: float,
|
|
120
|
+
dependency: str = "qstash",
|
|
121
|
+
propagate: bool = True,
|
|
122
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
123
|
+
) -> None:
|
|
124
|
+
self.dependency = identifier(dependency)
|
|
125
|
+
self._base_url = _url(base_url, name="base_url", base=True)
|
|
126
|
+
self._token = _header_value(token, name="token")
|
|
127
|
+
self._obs = obs
|
|
128
|
+
self._propagate = propagate
|
|
129
|
+
self._closed = False
|
|
130
|
+
self._http = HttpClient(
|
|
131
|
+
obs, timeout_seconds=timeout_seconds, retries=0, transport=transport
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
async def publish_json(
|
|
135
|
+
self,
|
|
136
|
+
destination: str,
|
|
137
|
+
body: Any,
|
|
138
|
+
*,
|
|
139
|
+
deduplication_id: str,
|
|
140
|
+
delivery_retries: int = 3,
|
|
141
|
+
delivery_timeout_seconds: int = 60,
|
|
142
|
+
delay_seconds: int = 0,
|
|
143
|
+
label: str | None = None,
|
|
144
|
+
forward_headers: Mapping[str, str] | None = None,
|
|
145
|
+
) -> PublishedMessage:
|
|
146
|
+
"""Accept JSON for delivery to one URL, preserving the supplied deduplication id.
|
|
147
|
+
|
|
148
|
+
Delivery options affect QStash's calls to the destination, not this POST.
|
|
149
|
+
Trace headers are owned by this client; propagate=False omits forwarded tracing.
|
|
150
|
+
"""
|
|
151
|
+
if self._closed:
|
|
152
|
+
raise RuntimeError("QStashClient is closed")
|
|
153
|
+
destination = _url(destination, name="destination")
|
|
154
|
+
_integer(delivery_retries, name="delivery_retries", minimum=0)
|
|
155
|
+
_integer(delivery_timeout_seconds, name="delivery_timeout_seconds", minimum=1)
|
|
156
|
+
_integer(delay_seconds, name="delay_seconds", minimum=0)
|
|
157
|
+
deduplication_id = _header_value(deduplication_id, name="deduplication_id")
|
|
158
|
+
if ":" in deduplication_id:
|
|
159
|
+
raise ValueError("deduplication_id must not contain ':'")
|
|
160
|
+
content = json.dumps(
|
|
161
|
+
body, ensure_ascii=False, allow_nan=False, separators=(",", ":")
|
|
162
|
+
).encode()
|
|
163
|
+
headers = {
|
|
164
|
+
"Authorization": f"Bearer {self._token}",
|
|
165
|
+
"Content-Type": "application/json",
|
|
166
|
+
"Upstash-Method": "POST",
|
|
167
|
+
"Upstash-Deduplication-Id": deduplication_id,
|
|
168
|
+
"Upstash-Retries": str(delivery_retries),
|
|
169
|
+
"Upstash-Timeout": f"{delivery_timeout_seconds}s",
|
|
170
|
+
"Upstash-Delay": f"{delay_seconds}s",
|
|
171
|
+
}
|
|
172
|
+
if label is not None:
|
|
173
|
+
headers["Upstash-Label"] = _header_value(label, name="label")
|
|
174
|
+
for key, value in (forward_headers or {}).items():
|
|
175
|
+
if not _HEADER_NAME.fullmatch(key):
|
|
176
|
+
raise ValueError("forward_headers contains an invalid header name")
|
|
177
|
+
name = key.lower()
|
|
178
|
+
if name in _TRACE_HEADERS:
|
|
179
|
+
continue
|
|
180
|
+
if name in {
|
|
181
|
+
"content-type",
|
|
182
|
+
"content-length",
|
|
183
|
+
"host",
|
|
184
|
+
"transfer-encoding",
|
|
185
|
+
"connection",
|
|
186
|
+
}:
|
|
187
|
+
raise ValueError(
|
|
188
|
+
"forward_headers cannot override HTTP framing or JSON content type"
|
|
189
|
+
)
|
|
190
|
+
headers[f"Upstash-Forward-{name}"] = _header_value(value, name="forwarded value")
|
|
191
|
+
async with observed(
|
|
192
|
+
self._obs,
|
|
193
|
+
family="messaging",
|
|
194
|
+
system=_SYSTEM,
|
|
195
|
+
dependency=self.dependency,
|
|
196
|
+
operation="publish",
|
|
197
|
+
classify=_classify,
|
|
198
|
+
):
|
|
199
|
+
if self._propagate:
|
|
200
|
+
carrier: dict[str, str] = {}
|
|
201
|
+
TraceContextTextMapPropagator().inject(carrier)
|
|
202
|
+
headers.update({f"Upstash-Forward-{key}": value for key, value in carrier.items()})
|
|
203
|
+
# QStash forwards this URL's query. Quoting the whole destination turns
|
|
204
|
+
# the query into a literal path at the consumer instead.
|
|
205
|
+
response = await self._http.request(
|
|
206
|
+
"POST",
|
|
207
|
+
f"{self._base_url}/v2/publish/{destination}",
|
|
208
|
+
dependency=self.dependency,
|
|
209
|
+
headers=headers,
|
|
210
|
+
content=content,
|
|
211
|
+
)
|
|
212
|
+
if not 200 <= response.status_code < 300:
|
|
213
|
+
raise QStashResponseError(response.status_code)
|
|
214
|
+
return _published(response)
|
|
215
|
+
|
|
216
|
+
async def aclose(self) -> None:
|
|
217
|
+
"""Release the HTTP pool and injected transport after in-flight calls finish."""
|
|
218
|
+
self._closed = True
|
|
219
|
+
await self._http.aclose()
|
clients_021/queue/sqs.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""SQS, observed, with the trace carried inside each message.
|
|
2
|
+
|
|
3
|
+
from clients_021.queue.sqs import SqsClient
|
|
4
|
+
|
|
5
|
+
queue = SqsClient.create(obs, queue_url, region="us-east-1")
|
|
6
|
+
await queue.send(body, attributes={"type": "notes.created.v1"})
|
|
7
|
+
for message in await queue.receive(max_messages=10, wait_seconds=20):
|
|
8
|
+
with tracer.start_as_current_span("handle", context=message.trace_context()):
|
|
9
|
+
...
|
|
10
|
+
await queue.delete(message.receipt)
|
|
11
|
+
|
|
12
|
+
Needs the `aws` extra. `send` adds `traceparent` (and `tracestate`) as message attributes,
|
|
13
|
+
so the consumer's span continues the producer's trace. Calls record
|
|
14
|
+
`messaging.client.operation.*` with `dependency` and `operation` (`send`, `receive`,
|
|
15
|
+
`delete`, `change_visibility`). Timeouts, connection failures, throttling and 5xx raise
|
|
16
|
+
DependencyUnavailable; access denied or a missing queue keep their botocore type.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
from collections.abc import Mapping
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import TYPE_CHECKING, Any
|
|
25
|
+
|
|
26
|
+
import boto3
|
|
27
|
+
from observability_021 import Observability
|
|
28
|
+
from observability_021.config import identifier
|
|
29
|
+
from opentelemetry.context import Context
|
|
30
|
+
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
31
|
+
|
|
32
|
+
from clients_021 import _aws
|
|
33
|
+
from clients_021._observe import observed
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from mypy_boto3_sqs import SQSClient
|
|
37
|
+
else:
|
|
38
|
+
SQSClient = Any
|
|
39
|
+
|
|
40
|
+
SYSTEM = {"messaging.system": "aws_sqs"}
|
|
41
|
+
_TRACE_KEYS = ("traceparent", "tracestate")
|
|
42
|
+
MAX_BATCH = 10
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class ReceivedMessage:
|
|
47
|
+
id: str
|
|
48
|
+
body: str
|
|
49
|
+
receipt: str
|
|
50
|
+
# How many times SQS has handed this message out (1 on the first delivery).
|
|
51
|
+
receive_count: int
|
|
52
|
+
attributes: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
53
|
+
|
|
54
|
+
def trace_context(self) -> Context:
|
|
55
|
+
"""The producer's trace, to start the consumer's span in."""
|
|
56
|
+
return TraceContextTextMapPropagator().extract(
|
|
57
|
+
{k: v for k, v in self.attributes.items() if k in _TRACE_KEYS}
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SqsClient:
|
|
62
|
+
def __init__(
|
|
63
|
+
self, obs: Observability, client: SQSClient, queue_url: str, *, dependency: str = "sqs"
|
|
64
|
+
) -> None:
|
|
65
|
+
self._obs = obs
|
|
66
|
+
self._client = client
|
|
67
|
+
self._queue_url = queue_url
|
|
68
|
+
self.dependency = identifier(dependency)
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def create(
|
|
72
|
+
cls,
|
|
73
|
+
obs: Observability,
|
|
74
|
+
queue_url: str,
|
|
75
|
+
*,
|
|
76
|
+
region: str | None = None,
|
|
77
|
+
endpoint_url: str | None = None,
|
|
78
|
+
dependency: str = "sqs",
|
|
79
|
+
timeout_seconds: float = 5.0,
|
|
80
|
+
) -> SqsClient:
|
|
81
|
+
# Long polling waits up to 20 s server-side: the read timeout must outlast it.
|
|
82
|
+
client: SQSClient = boto3.client( # pyright: ignore[reportUnknownMemberType]
|
|
83
|
+
"sqs",
|
|
84
|
+
region_name=region,
|
|
85
|
+
endpoint_url=endpoint_url,
|
|
86
|
+
config=_aws.config(connect_seconds=timeout_seconds, read_seconds=20 + timeout_seconds),
|
|
87
|
+
)
|
|
88
|
+
return cls(obs, client, queue_url, dependency=dependency)
|
|
89
|
+
|
|
90
|
+
def _observed(self, operation: str): # noqa: ANN202 - an async context manager
|
|
91
|
+
return observed(
|
|
92
|
+
self._obs,
|
|
93
|
+
family="messaging",
|
|
94
|
+
system=SYSTEM,
|
|
95
|
+
dependency=self.dependency,
|
|
96
|
+
operation=operation,
|
|
97
|
+
classify=_aws.classify,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
async def send(
|
|
101
|
+
self, body: str, *, attributes: Mapping[str, str] | None = None, delay_seconds: int = 0
|
|
102
|
+
) -> str:
|
|
103
|
+
"""Returns the message id. Attributes are strings; the trace context is added here."""
|
|
104
|
+
carrier: dict[str, str] = {}
|
|
105
|
+
message_attributes: dict[str, Any] = {
|
|
106
|
+
key: {"DataType": "String", "StringValue": value}
|
|
107
|
+
for key, value in (attributes or {}).items()
|
|
108
|
+
if key not in _TRACE_KEYS
|
|
109
|
+
}
|
|
110
|
+
async with self._observed("send"):
|
|
111
|
+
TraceContextTextMapPropagator().inject(carrier)
|
|
112
|
+
for key, value in carrier.items():
|
|
113
|
+
message_attributes[key] = {"DataType": "String", "StringValue": value}
|
|
114
|
+
response = await asyncio.to_thread(
|
|
115
|
+
self._client.send_message,
|
|
116
|
+
QueueUrl=self._queue_url,
|
|
117
|
+
MessageBody=body,
|
|
118
|
+
MessageAttributes=message_attributes,
|
|
119
|
+
DelaySeconds=delay_seconds,
|
|
120
|
+
)
|
|
121
|
+
return response["MessageId"]
|
|
122
|
+
|
|
123
|
+
async def receive(
|
|
124
|
+
self,
|
|
125
|
+
*,
|
|
126
|
+
max_messages: int = MAX_BATCH,
|
|
127
|
+
wait_seconds: int = 20,
|
|
128
|
+
visibility_timeout: int | None = None,
|
|
129
|
+
) -> list[ReceivedMessage]:
|
|
130
|
+
extra: dict[str, Any] = {}
|
|
131
|
+
if visibility_timeout is not None:
|
|
132
|
+
extra["VisibilityTimeout"] = visibility_timeout
|
|
133
|
+
async with self._observed("receive"):
|
|
134
|
+
response = await asyncio.to_thread(
|
|
135
|
+
self._client.receive_message,
|
|
136
|
+
QueueUrl=self._queue_url,
|
|
137
|
+
MaxNumberOfMessages=max(1, min(max_messages, MAX_BATCH)),
|
|
138
|
+
WaitTimeSeconds=wait_seconds,
|
|
139
|
+
MessageAttributeNames=["All"],
|
|
140
|
+
MessageSystemAttributeNames=["ApproximateReceiveCount"],
|
|
141
|
+
**extra,
|
|
142
|
+
)
|
|
143
|
+
return [
|
|
144
|
+
ReceivedMessage(
|
|
145
|
+
id=message.get("MessageId", ""),
|
|
146
|
+
body=message.get("Body", ""),
|
|
147
|
+
receipt=message.get("ReceiptHandle", ""),
|
|
148
|
+
receive_count=int(
|
|
149
|
+
message.get("Attributes", {}).get("ApproximateReceiveCount", "1")
|
|
150
|
+
),
|
|
151
|
+
attributes={
|
|
152
|
+
key: str(value.get("StringValue", ""))
|
|
153
|
+
for key, value in message.get("MessageAttributes", {}).items()
|
|
154
|
+
},
|
|
155
|
+
)
|
|
156
|
+
for message in response.get("Messages", [])
|
|
157
|
+
]
|
|
158
|
+
|
|
159
|
+
async def delete(self, receipt: str) -> None:
|
|
160
|
+
async with self._observed("delete"):
|
|
161
|
+
await asyncio.to_thread(
|
|
162
|
+
self._client.delete_message, QueueUrl=self._queue_url, ReceiptHandle=receipt
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
async def change_visibility(self, receipt: str, *, seconds: int) -> None:
|
|
166
|
+
"""Hands the message back after `seconds` (0: now), for an earlier or later retry."""
|
|
167
|
+
async with self._observed("change_visibility"):
|
|
168
|
+
await asyncio.to_thread(
|
|
169
|
+
self._client.change_message_visibility,
|
|
170
|
+
QueueUrl=self._queue_url,
|
|
171
|
+
ReceiptHandle=receipt,
|
|
172
|
+
VisibilityTimeout=seconds,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
async def aclose(self) -> None:
|
|
176
|
+
"""Release the underlying SDK connection pools after in-flight calls finish."""
|
|
177
|
+
await asyncio.to_thread(self._client.close)
|