vercel-queue 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vercel/queue/__init__.py +8 -0
- vercel/queue/__main__.py +10 -0
- vercel/queue/_internal/__init__.py +1 -0
- vercel/queue/_internal/api_async.py +219 -0
- vercel/queue/_internal/api_common.py +104 -0
- vercel/queue/_internal/api_sync.py +137 -0
- vercel/queue/_internal/asgi.py +145 -0
- vercel/queue/_internal/asynctools.py +40 -0
- vercel/queue/_internal/cli.py +313 -0
- vercel/queue/_internal/client.py +1109 -0
- vercel/queue/_internal/client_sync.py +432 -0
- vercel/queue/_internal/config.py +160 -0
- vercel/queue/_internal/constants.py +50 -0
- vercel/queue/_internal/devserver.py +205 -0
- vercel/queue/_internal/embedded.py +1900 -0
- vercel/queue/_internal/errors.py +262 -0
- vercel/queue/_internal/http.py +634 -0
- vercel/queue/_internal/lease.py +1007 -0
- vercel/queue/_internal/log.py +143 -0
- vercel/queue/_internal/messages.py +122 -0
- vercel/queue/_internal/multipart.py +255 -0
- vercel/queue/_internal/names.py +101 -0
- vercel/queue/_internal/polling.py +200 -0
- vercel/queue/_internal/push.py +246 -0
- vercel/queue/_internal/response.py +111 -0
- vercel/queue/_internal/retry.py +86 -0
- vercel/queue/_internal/streams.py +313 -0
- vercel/queue/_internal/subscribers.py +1025 -0
- vercel/queue/_internal/transports.py +363 -0
- vercel/queue/_internal/types.py +300 -0
- vercel/queue/_internal/typeutils.py +203 -0
- vercel/queue/devserver.py +24 -0
- vercel/queue/embedded.py +50 -0
- vercel/queue/py.typed +1 -0
- vercel/queue/sync.py +8 -0
- vercel/queue/testing/__init__.py +14 -0
- vercel/queue/testing/pytest.py +42 -0
- vercel/queue/testing/state.py +32 -0
- vercel/queue/version.py +3 -0
- vercel_queue-0.7.0.dist-info/METADATA +681 -0
- vercel_queue-0.7.0.dist-info/RECORD +43 -0
- vercel_queue-0.7.0.dist-info/WHEEL +4 -0
- vercel_queue-0.7.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1900 @@
|
|
|
1
|
+
"""VQS server emulation for embedded/local use."""
|
|
2
|
+
|
|
3
|
+
# ruff: noqa: S107 # hardcoded-password-default -- dummy token
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import Any, Literal, NamedTuple, Protocol
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import heapq
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import re
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import weakref
|
|
18
|
+
from collections import deque
|
|
19
|
+
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, MutableMapping
|
|
20
|
+
from contextlib import (
|
|
21
|
+
AbstractAsyncContextManager,
|
|
22
|
+
AbstractContextManager,
|
|
23
|
+
asynccontextmanager,
|
|
24
|
+
contextmanager,
|
|
25
|
+
)
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from datetime import datetime, timedelta, timezone
|
|
28
|
+
from itertools import count
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from tempfile import TemporaryDirectory
|
|
31
|
+
from urllib.parse import unquote
|
|
32
|
+
|
|
33
|
+
import anyio
|
|
34
|
+
import httpx
|
|
35
|
+
from anyio.abc import TaskGroup
|
|
36
|
+
|
|
37
|
+
from .client import QueueClient
|
|
38
|
+
from .client_sync import QueueClient as SyncQueueClient
|
|
39
|
+
from .config import ALL_DEPLOYMENTS, BaseUrl, DeploymentOption
|
|
40
|
+
from .constants import (
|
|
41
|
+
CLOUD_EVENT_HEADER_TYPE,
|
|
42
|
+
CLOUD_EVENT_HEADER_VQS_CONSUMER_GROUP,
|
|
43
|
+
CLOUD_EVENT_HEADER_VQS_CREATED_AT,
|
|
44
|
+
CLOUD_EVENT_HEADER_VQS_DELIVERY_COUNT,
|
|
45
|
+
CLOUD_EVENT_HEADER_VQS_EXPIRES_AT,
|
|
46
|
+
CLOUD_EVENT_HEADER_VQS_MESSAGE_ID,
|
|
47
|
+
CLOUD_EVENT_HEADER_VQS_RECEIPT_HANDLE,
|
|
48
|
+
CLOUD_EVENT_HEADER_VQS_REGION,
|
|
49
|
+
CLOUD_EVENT_HEADER_VQS_TOPIC,
|
|
50
|
+
CLOUD_EVENT_HEADER_VQS_VISIBILITY_DEADLINE,
|
|
51
|
+
CLOUD_EVENT_TYPE_V2BETA,
|
|
52
|
+
CONTENT_TYPE_MULTIPART_MIXED,
|
|
53
|
+
CONTENT_TYPE_NDJSON,
|
|
54
|
+
DEFAULT_RETRY_AFTER_SECONDS,
|
|
55
|
+
HEADER_ACCEPT,
|
|
56
|
+
HEADER_CONTENT_TYPE,
|
|
57
|
+
VQS_HEADER_DELAY_SECONDS,
|
|
58
|
+
VQS_HEADER_DELIVERY_COUNT,
|
|
59
|
+
VQS_HEADER_DEPLOYMENT_ID,
|
|
60
|
+
VQS_HEADER_EXPIRES_AT,
|
|
61
|
+
VQS_HEADER_IDEMPOTENCY_KEY,
|
|
62
|
+
VQS_HEADER_MAX_MESSAGES,
|
|
63
|
+
VQS_HEADER_MESSAGE_ID,
|
|
64
|
+
VQS_HEADER_RECEIPT_HANDLE,
|
|
65
|
+
VQS_HEADER_RETENTION_SECONDS,
|
|
66
|
+
VQS_HEADER_TIMESTAMP,
|
|
67
|
+
VQS_HEADER_VISIBILITY_TIMEOUT_SECONDS,
|
|
68
|
+
VQS_NAME_PATTERN,
|
|
69
|
+
)
|
|
70
|
+
from .http import BaseQueueRuntime
|
|
71
|
+
from .log import debug_log
|
|
72
|
+
from .names import validate_name, validate_topic_name
|
|
73
|
+
from .subscribers import register_embedded_dispatcher, unregister_embedded_dispatcher
|
|
74
|
+
from .types import Duration
|
|
75
|
+
|
|
76
|
+
DEFAULT_DEPLOYMENT_PARTITION = "__all__"
|
|
77
|
+
DEFAULT_PULL_LEASE_SECONDS = 60
|
|
78
|
+
DEFAULT_PUSH_LEASE_SECONDS = 300
|
|
79
|
+
DEFAULT_RETENTION_SECONDS = 86_400
|
|
80
|
+
MAX_RETENTION_SECONDS = 604_800
|
|
81
|
+
MIN_RETENTION_SECONDS = 60
|
|
82
|
+
MAX_VISIBILITY_TIMEOUT_SECONDS = 3_600
|
|
83
|
+
PAYLOAD_SPILL_THRESHOLD_BYTES = 256 * 1000
|
|
84
|
+
BOUNDARY = "vqs-test-boundary"
|
|
85
|
+
QUEUE_PATH_PREFIX = ("api", "v3", "topic")
|
|
86
|
+
JSON_RESPONSE_HEADERS = [(b"content-type", b"application/json")]
|
|
87
|
+
QueueRouteAction = Literal["topic", "consumer", "message_id", "lease", "lease_visibility"]
|
|
88
|
+
ReceiveResponseFormat = Literal["multipart", "ndjson"]
|
|
89
|
+
SubscriptionMatchKind = Literal["exact", "prefix", "wildcard"]
|
|
90
|
+
DEPLOYMENT_ID_PATTERN = re.compile(VQS_NAME_PATTERN)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class EmbeddedQueueClock(Protocol):
|
|
94
|
+
def now(self) -> datetime: ...
|
|
95
|
+
|
|
96
|
+
def monotonic(self) -> float: ...
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class EmbeddedQueuePushClient(Protocol):
|
|
100
|
+
async def accept_and_handle(
|
|
101
|
+
self,
|
|
102
|
+
raw_body: bytes,
|
|
103
|
+
headers: Mapping[str, str],
|
|
104
|
+
) -> None: ...
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass
|
|
108
|
+
class RealEmbeddedQueueClock:
|
|
109
|
+
def now(self) -> datetime:
|
|
110
|
+
return datetime.now(timezone.utc)
|
|
111
|
+
|
|
112
|
+
def monotonic(self) -> float:
|
|
113
|
+
return time.monotonic()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class ManualEmbeddedQueueClock:
|
|
118
|
+
current: datetime = field(default_factory=lambda: datetime(2026, 1, 1, tzinfo=timezone.utc))
|
|
119
|
+
|
|
120
|
+
def now(self) -> datetime:
|
|
121
|
+
return self.current
|
|
122
|
+
|
|
123
|
+
def monotonic(self) -> float:
|
|
124
|
+
return self.current.timestamp()
|
|
125
|
+
|
|
126
|
+
def shift(self, seconds: float) -> None:
|
|
127
|
+
self.current += timedelta(seconds=seconds)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True)
|
|
131
|
+
class PushDelivery:
|
|
132
|
+
body: bytes
|
|
133
|
+
headers: dict[str, str]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass(frozen=True)
|
|
137
|
+
class _QueueRoute:
|
|
138
|
+
action: QueueRouteAction
|
|
139
|
+
topic: str
|
|
140
|
+
consumer: str | None = None
|
|
141
|
+
message_id: str | None = None
|
|
142
|
+
receipt_handle: str | None = None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(kw_only=True)
|
|
146
|
+
class _DispatcherRegistration:
|
|
147
|
+
topic_pattern: str
|
|
148
|
+
match_kind: SubscriptionMatchKind
|
|
149
|
+
consumer_group: str
|
|
150
|
+
retry_after_seconds: int | None
|
|
151
|
+
initial_delay_seconds: int | None
|
|
152
|
+
max_concurrency: int | None
|
|
153
|
+
max_attempts: int | None
|
|
154
|
+
registered_at: datetime
|
|
155
|
+
concurrency_limit: int | None = None
|
|
156
|
+
semaphore: anyio.Semaphore | None = None
|
|
157
|
+
|
|
158
|
+
def key(self) -> tuple[str, SubscriptionMatchKind, str]:
|
|
159
|
+
return self.topic_pattern, self.match_kind, self.consumer_group
|
|
160
|
+
|
|
161
|
+
def matches_topic(self, topic: str) -> bool:
|
|
162
|
+
if self.match_kind == "wildcard":
|
|
163
|
+
return True
|
|
164
|
+
if self.match_kind == "prefix":
|
|
165
|
+
return topic.startswith(self.topic_pattern)
|
|
166
|
+
return topic == self.topic_pattern
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass(frozen=True)
|
|
170
|
+
class EmbeddedQueueRequest:
|
|
171
|
+
method: str
|
|
172
|
+
path: str
|
|
173
|
+
headers: httpx.Headers
|
|
174
|
+
body: bytes
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass(frozen=True)
|
|
178
|
+
class EmbeddedQueueResponse:
|
|
179
|
+
method: str
|
|
180
|
+
action: QueueRouteAction
|
|
181
|
+
status_code: int
|
|
182
|
+
body: bytes = b""
|
|
183
|
+
headers: tuple[tuple[bytes, bytes], ...] = ()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@dataclass
|
|
187
|
+
class StoredPayload:
|
|
188
|
+
data: bytes | None = None
|
|
189
|
+
path: Path | None = None
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def size(self) -> int:
|
|
193
|
+
if self.data is not None:
|
|
194
|
+
return len(self.data)
|
|
195
|
+
if self.path is None:
|
|
196
|
+
return 0
|
|
197
|
+
return self.path.stat().st_size
|
|
198
|
+
|
|
199
|
+
@property
|
|
200
|
+
def spilled(self) -> bool:
|
|
201
|
+
return self.path is not None
|
|
202
|
+
|
|
203
|
+
def read(self) -> bytes:
|
|
204
|
+
if self.data is not None:
|
|
205
|
+
return self.data
|
|
206
|
+
if self.path is None:
|
|
207
|
+
return b""
|
|
208
|
+
return self.path.read_bytes()
|
|
209
|
+
|
|
210
|
+
def delete(self) -> None:
|
|
211
|
+
if self.path is not None:
|
|
212
|
+
try:
|
|
213
|
+
self.path.unlink()
|
|
214
|
+
except FileNotFoundError:
|
|
215
|
+
pass
|
|
216
|
+
self.path = None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass
|
|
220
|
+
class StoredMessage:
|
|
221
|
+
topic: str
|
|
222
|
+
message_id: str
|
|
223
|
+
payload: StoredPayload
|
|
224
|
+
content_type: str
|
|
225
|
+
deployment: str
|
|
226
|
+
idempotency_key: str | None
|
|
227
|
+
retention_seconds: int
|
|
228
|
+
delay_seconds: int
|
|
229
|
+
created_at: datetime
|
|
230
|
+
available_at: datetime
|
|
231
|
+
expires_at: datetime
|
|
232
|
+
duplicate_of: str | None = None
|
|
233
|
+
acknowledged_by_consumer: set[str] = field(default_factory=set)
|
|
234
|
+
delivery_count_by_consumer: dict[str, int] = field(default_factory=dict)
|
|
235
|
+
leases_by_consumer: dict[str, str] = field(default_factory=dict)
|
|
236
|
+
lease_deadline_by_consumer: dict[str, datetime] = field(default_factory=dict)
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def acknowledged(self) -> bool:
|
|
240
|
+
return bool(self.acknowledged_by_consumer)
|
|
241
|
+
|
|
242
|
+
@acknowledged.setter
|
|
243
|
+
def acknowledged(self, value: bool) -> None:
|
|
244
|
+
if value:
|
|
245
|
+
self.acknowledged_by_consumer.add(DEFAULT_DEPLOYMENT_PARTITION)
|
|
246
|
+
else:
|
|
247
|
+
self.acknowledged_by_consumer.clear()
|
|
248
|
+
|
|
249
|
+
def acknowledged_for(self, consumer: str) -> bool:
|
|
250
|
+
return (
|
|
251
|
+
consumer in self.acknowledged_by_consumer
|
|
252
|
+
or DEFAULT_DEPLOYMENT_PARTITION in self.acknowledged_by_consumer
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
def acknowledge_for(self, consumer: str) -> None:
|
|
256
|
+
self.acknowledged_by_consumer.add(consumer)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@dataclass(frozen=True)
|
|
260
|
+
class _LeasedMessage:
|
|
261
|
+
message: StoredMessage
|
|
262
|
+
receipt_handle: str
|
|
263
|
+
consumer: str
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class _PushIndexKey(NamedTuple):
|
|
267
|
+
topic: str
|
|
268
|
+
consumer: str
|
|
269
|
+
deployment: str | None
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@dataclass
|
|
273
|
+
class _PushDeliveryIndex:
|
|
274
|
+
cursor: int = 0
|
|
275
|
+
ready: deque[str] = field(default_factory=deque)
|
|
276
|
+
blocked: list[tuple[datetime, int, str]] = field(default_factory=list)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@dataclass
|
|
280
|
+
class EmbeddedQueueState:
|
|
281
|
+
clock: EmbeddedQueueClock = field(default_factory=RealEmbeddedQueueClock)
|
|
282
|
+
messages: list[StoredMessage] = field(default_factory=list)
|
|
283
|
+
by_topic: dict[str, list[StoredMessage]] = field(default_factory=dict)
|
|
284
|
+
by_id: dict[str, StoredMessage] = field(default_factory=dict)
|
|
285
|
+
idempotency: dict[tuple[str, str, str, bytes], str] = field(default_factory=dict)
|
|
286
|
+
by_receipt: dict[str, StoredMessage] = field(default_factory=dict)
|
|
287
|
+
push_delivery_indexes: dict[_PushIndexKey, _PushDeliveryIndex] = field(default_factory=dict)
|
|
288
|
+
push_blocked_sequence: count[int] = field(default_factory=count)
|
|
289
|
+
next_expires_at: datetime | None = None
|
|
290
|
+
requests: list[EmbeddedQueueRequest] = field(default_factory=list)
|
|
291
|
+
responses: list[EmbeddedQueueResponse] = field(default_factory=list)
|
|
292
|
+
ids: count[int] = field(default_factory=lambda: count(1))
|
|
293
|
+
receipts: count[int] = field(default_factory=lambda: count(1))
|
|
294
|
+
payload_spill_threshold_bytes: int = PAYLOAD_SPILL_THRESHOLD_BYTES
|
|
295
|
+
_payload_directory: TemporaryDirectory[str] = field(
|
|
296
|
+
default_factory=lambda: TemporaryDirectory(prefix="vercel-queue-payloads-"),
|
|
297
|
+
init=False,
|
|
298
|
+
repr=False,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
@property
|
|
302
|
+
def payload_directory(self) -> Path:
|
|
303
|
+
return Path(self._payload_directory.name)
|
|
304
|
+
|
|
305
|
+
def store_payload(self, message_id: str, payload: bytes) -> StoredPayload:
|
|
306
|
+
if len(payload) <= self.payload_spill_threshold_bytes:
|
|
307
|
+
return StoredPayload(data=payload)
|
|
308
|
+
path = self.payload_directory / f"{message_id}.body"
|
|
309
|
+
path.write_bytes(payload)
|
|
310
|
+
return StoredPayload(path=path)
|
|
311
|
+
|
|
312
|
+
def clear_payloads(self) -> None:
|
|
313
|
+
for message in self.messages:
|
|
314
|
+
message.payload.delete()
|
|
315
|
+
|
|
316
|
+
def reset(self) -> None:
|
|
317
|
+
"""Reset queue contents and request history for tests."""
|
|
318
|
+
self.clear_payloads()
|
|
319
|
+
self.messages.clear()
|
|
320
|
+
self.by_topic.clear()
|
|
321
|
+
self.by_id.clear()
|
|
322
|
+
self.idempotency.clear()
|
|
323
|
+
self.by_receipt.clear()
|
|
324
|
+
self.push_delivery_indexes.clear()
|
|
325
|
+
self.push_blocked_sequence = count()
|
|
326
|
+
self.next_expires_at = None
|
|
327
|
+
self.requests.clear()
|
|
328
|
+
self.responses.clear()
|
|
329
|
+
self.ids = count(1)
|
|
330
|
+
self.receipts = count(1)
|
|
331
|
+
self.now = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
332
|
+
|
|
333
|
+
def close(self) -> None:
|
|
334
|
+
self.clear_payloads()
|
|
335
|
+
self._payload_directory.cleanup()
|
|
336
|
+
|
|
337
|
+
def shift(self, seconds: float) -> None:
|
|
338
|
+
shift = getattr(self.clock, "shift", None)
|
|
339
|
+
if shift is None:
|
|
340
|
+
raise RuntimeError("broker clock does not support manual shifting")
|
|
341
|
+
shift(seconds)
|
|
342
|
+
|
|
343
|
+
@property
|
|
344
|
+
def now(self) -> datetime:
|
|
345
|
+
return self.clock.now()
|
|
346
|
+
|
|
347
|
+
@now.setter
|
|
348
|
+
def now(self, value: datetime) -> None:
|
|
349
|
+
self.clock = ManualEmbeddedQueueClock(value)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
@dataclass
|
|
353
|
+
class EmbeddedQueueServer:
|
|
354
|
+
state: EmbeddedQueueState = field(default_factory=EmbeddedQueueState)
|
|
355
|
+
_wake_callbacks: list[weakref.ReferenceType[Callable[[], None]]] = field(
|
|
356
|
+
default_factory=list,
|
|
357
|
+
init=False,
|
|
358
|
+
repr=False,
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
def add_wake_callback(self, callback: Callable[[], None]) -> None:
|
|
362
|
+
self._wake_callbacks[:] = [ref for ref in self._wake_callbacks if ref() is not None]
|
|
363
|
+
if any(ref() is callback for ref in self._wake_callbacks):
|
|
364
|
+
return
|
|
365
|
+
callback_ref: weakref.ReferenceType[Callable[[], None]]
|
|
366
|
+
try:
|
|
367
|
+
callback_ref = weakref.WeakMethod(callback) # type: ignore[arg-type]
|
|
368
|
+
except TypeError:
|
|
369
|
+
callback_ref = weakref.ref(callback)
|
|
370
|
+
self._wake_callbacks.append(callback_ref)
|
|
371
|
+
|
|
372
|
+
def shift(self, seconds: float) -> None:
|
|
373
|
+
self.state.shift(seconds)
|
|
374
|
+
|
|
375
|
+
@property
|
|
376
|
+
def now(self) -> datetime:
|
|
377
|
+
return self.state.now
|
|
378
|
+
|
|
379
|
+
@now.setter
|
|
380
|
+
def now(self, value: datetime) -> None:
|
|
381
|
+
self.state.now = value
|
|
382
|
+
|
|
383
|
+
def put(self, topic: str, payload: bytes, headers: Mapping[str, str]) -> StoredMessage:
|
|
384
|
+
deployment = headers.get(VQS_HEADER_DEPLOYMENT_ID, DEFAULT_DEPLOYMENT_PARTITION)
|
|
385
|
+
if deployment != DEFAULT_DEPLOYMENT_PARTITION:
|
|
386
|
+
_validate_deployment_id(deployment)
|
|
387
|
+
idempotency_key = headers.get(VQS_HEADER_IDEMPOTENCY_KEY)
|
|
388
|
+
content_type = headers.get(HEADER_CONTENT_TYPE, "application/octet-stream")
|
|
389
|
+
retention_seconds = _int_header(
|
|
390
|
+
headers.get(VQS_HEADER_RETENTION_SECONDS),
|
|
391
|
+
DEFAULT_RETENTION_SECONDS,
|
|
392
|
+
minimum=MIN_RETENTION_SECONDS,
|
|
393
|
+
maximum=MAX_RETENTION_SECONDS,
|
|
394
|
+
name=VQS_HEADER_RETENTION_SECONDS,
|
|
395
|
+
)
|
|
396
|
+
delay_seconds = _int_header(
|
|
397
|
+
headers.get(VQS_HEADER_DELAY_SECONDS),
|
|
398
|
+
0,
|
|
399
|
+
minimum=0,
|
|
400
|
+
maximum=sys.maxsize,
|
|
401
|
+
name=VQS_HEADER_DELAY_SECONDS,
|
|
402
|
+
)
|
|
403
|
+
if delay_seconds > retention_seconds:
|
|
404
|
+
raise ValueError("Vqs-Delay-Seconds cannot exceed retention time")
|
|
405
|
+
state = self.state
|
|
406
|
+
created_at = state.now
|
|
407
|
+
message_id = f"msg_{next(state.ids)}"
|
|
408
|
+
duplicate_of: str | None = None
|
|
409
|
+
if idempotency_key:
|
|
410
|
+
key = (topic, deployment, idempotency_key, payload)
|
|
411
|
+
duplicate_of = state.idempotency.get(key)
|
|
412
|
+
state.idempotency[key] = duplicate_of or message_id
|
|
413
|
+
message = StoredMessage(
|
|
414
|
+
topic=topic,
|
|
415
|
+
message_id=message_id,
|
|
416
|
+
payload=state.store_payload(message_id, payload),
|
|
417
|
+
content_type=content_type,
|
|
418
|
+
deployment=deployment,
|
|
419
|
+
idempotency_key=idempotency_key,
|
|
420
|
+
retention_seconds=retention_seconds,
|
|
421
|
+
delay_seconds=delay_seconds,
|
|
422
|
+
created_at=created_at,
|
|
423
|
+
available_at=created_at + timedelta(seconds=delay_seconds),
|
|
424
|
+
expires_at=created_at + timedelta(seconds=retention_seconds),
|
|
425
|
+
duplicate_of=duplicate_of,
|
|
426
|
+
)
|
|
427
|
+
state.messages.append(message)
|
|
428
|
+
state.by_topic.setdefault(topic, []).append(message)
|
|
429
|
+
state.by_id[message_id] = message
|
|
430
|
+
if state.next_expires_at is None or message.expires_at < state.next_expires_at:
|
|
431
|
+
state.next_expires_at = message.expires_at
|
|
432
|
+
self.wake_dispatchers()
|
|
433
|
+
return message
|
|
434
|
+
|
|
435
|
+
def respond_once(
|
|
436
|
+
self,
|
|
437
|
+
*,
|
|
438
|
+
method: str,
|
|
439
|
+
action: QueueRouteAction,
|
|
440
|
+
status_code: int,
|
|
441
|
+
body: bytes = b"",
|
|
442
|
+
headers: Mapping[str, str] | None = None,
|
|
443
|
+
) -> None:
|
|
444
|
+
self.state.responses.append(
|
|
445
|
+
EmbeddedQueueResponse(
|
|
446
|
+
method=method.upper(),
|
|
447
|
+
action=action,
|
|
448
|
+
status_code=status_code,
|
|
449
|
+
body=body,
|
|
450
|
+
headers=tuple(
|
|
451
|
+
(key.encode("latin-1"), value.encode("latin-1"))
|
|
452
|
+
for key, value in (headers or {}).items()
|
|
453
|
+
),
|
|
454
|
+
)
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
def response_for(self, request: _AsgiRequest, route: _QueueRoute) -> _AsgiResponse | None:
|
|
458
|
+
for index, response in enumerate(self.state.responses):
|
|
459
|
+
if response.method == request.method and response.action == route.action:
|
|
460
|
+
self.state.responses.pop(index)
|
|
461
|
+
return _AsgiResponse(response.status_code, response.body, list(response.headers))
|
|
462
|
+
return None
|
|
463
|
+
|
|
464
|
+
async def record_request(self, request: _AsgiRequest) -> None:
|
|
465
|
+
self.state.requests.append(
|
|
466
|
+
EmbeddedQueueRequest(
|
|
467
|
+
method=request.method,
|
|
468
|
+
path=request.path,
|
|
469
|
+
headers=httpx.Headers(request.headers),
|
|
470
|
+
body=await request.body(),
|
|
471
|
+
)
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
def wake_dispatchers(self) -> None:
|
|
475
|
+
live: list[weakref.ReferenceType[Callable[[], None]]] = []
|
|
476
|
+
for ref in self._wake_callbacks:
|
|
477
|
+
callback = ref()
|
|
478
|
+
if callback is None:
|
|
479
|
+
continue
|
|
480
|
+
live.append(ref)
|
|
481
|
+
callback()
|
|
482
|
+
self._wake_callbacks[:] = live
|
|
483
|
+
|
|
484
|
+
def visible_messages(
|
|
485
|
+
self,
|
|
486
|
+
topic: str,
|
|
487
|
+
consumer: str,
|
|
488
|
+
deployment: str | None,
|
|
489
|
+
) -> list[StoredMessage]:
|
|
490
|
+
self.cleanup_expired()
|
|
491
|
+
return [
|
|
492
|
+
message
|
|
493
|
+
for message in self.state.by_topic.get(topic, [])
|
|
494
|
+
if self._can_deliver(message, topic, consumer, deployment)
|
|
495
|
+
]
|
|
496
|
+
|
|
497
|
+
def next_push_message(
|
|
498
|
+
self,
|
|
499
|
+
topic: str,
|
|
500
|
+
consumer: str,
|
|
501
|
+
deployment: str | None,
|
|
502
|
+
) -> StoredMessage | None:
|
|
503
|
+
self.cleanup_expired()
|
|
504
|
+
now = self.state.now
|
|
505
|
+
index = self._push_delivery_index(topic, consumer, deployment)
|
|
506
|
+
self._push_index_messages(index, topic)
|
|
507
|
+
self._push_ready_blocked(index, now)
|
|
508
|
+
while index.ready:
|
|
509
|
+
message_id = index.ready.popleft()
|
|
510
|
+
message = self.state.by_id.get(message_id)
|
|
511
|
+
if message is None:
|
|
512
|
+
continue
|
|
513
|
+
deadline = self._message_visible_deadline_for_consumer(
|
|
514
|
+
message,
|
|
515
|
+
topic,
|
|
516
|
+
consumer,
|
|
517
|
+
deployment,
|
|
518
|
+
now,
|
|
519
|
+
)
|
|
520
|
+
if deadline is None:
|
|
521
|
+
continue
|
|
522
|
+
if deadline > now:
|
|
523
|
+
self._push_block_after(index, deadline, message.message_id)
|
|
524
|
+
continue
|
|
525
|
+
return message
|
|
526
|
+
return None
|
|
527
|
+
|
|
528
|
+
def find_by_id(self, topic: str, message_id: str) -> StoredMessage | None:
|
|
529
|
+
self.cleanup_expired()
|
|
530
|
+
message = self.state.by_id.get(message_id)
|
|
531
|
+
if message is None or message.topic != topic:
|
|
532
|
+
return None
|
|
533
|
+
return message
|
|
534
|
+
|
|
535
|
+
def lease(self, message: StoredMessage, consumer: str, seconds: int) -> str:
|
|
536
|
+
receipt_handle = (
|
|
537
|
+
f"rh_{next(self.state.receipts)}:{message.message_id}:{consumer} /needs encoding"
|
|
538
|
+
)
|
|
539
|
+
message.delivery_count_by_consumer[consumer] = (
|
|
540
|
+
message.delivery_count_by_consumer.get(consumer, 0) + 1
|
|
541
|
+
)
|
|
542
|
+
previous_receipt = message.leases_by_consumer.get(consumer)
|
|
543
|
+
if previous_receipt is not None:
|
|
544
|
+
self.state.by_receipt.pop(previous_receipt, None)
|
|
545
|
+
message.leases_by_consumer[consumer] = receipt_handle
|
|
546
|
+
self.state.by_receipt[receipt_handle] = message
|
|
547
|
+
message.lease_deadline_by_consumer[consumer] = self.now + timedelta(seconds=seconds)
|
|
548
|
+
return receipt_handle
|
|
549
|
+
|
|
550
|
+
def receipt_message(
|
|
551
|
+
self,
|
|
552
|
+
topic: str,
|
|
553
|
+
consumer: str,
|
|
554
|
+
receipt_handle: str,
|
|
555
|
+
deployment: str | None,
|
|
556
|
+
) -> StoredMessage | None:
|
|
557
|
+
self.cleanup_expired()
|
|
558
|
+
message = self.state.by_receipt.get(receipt_handle)
|
|
559
|
+
if message is None:
|
|
560
|
+
return None
|
|
561
|
+
if message.topic != topic or message.leases_by_consumer.get(consumer) != receipt_handle:
|
|
562
|
+
return None
|
|
563
|
+
if deployment is not None and message.deployment != deployment:
|
|
564
|
+
return None
|
|
565
|
+
return message
|
|
566
|
+
|
|
567
|
+
def receipt_exists(
|
|
568
|
+
self,
|
|
569
|
+
topic: str,
|
|
570
|
+
receipt_handle: str,
|
|
571
|
+
deployment: str | None,
|
|
572
|
+
) -> bool:
|
|
573
|
+
if (message := self.state.by_receipt.get(receipt_handle)) is not None:
|
|
574
|
+
return message.topic == topic and (
|
|
575
|
+
deployment is None or message.deployment == deployment
|
|
576
|
+
)
|
|
577
|
+
receipt_message_id = _message_id_from_receipt_handle(receipt_handle)
|
|
578
|
+
if receipt_message_id is None:
|
|
579
|
+
return False
|
|
580
|
+
message = self.state.by_id.get(receipt_message_id)
|
|
581
|
+
return (
|
|
582
|
+
message is not None
|
|
583
|
+
and message.topic == topic
|
|
584
|
+
and (deployment is None or message.deployment == deployment)
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
def push_delivery(
|
|
588
|
+
self,
|
|
589
|
+
topic: str,
|
|
590
|
+
consumer: str,
|
|
591
|
+
*,
|
|
592
|
+
lease_seconds: int = DEFAULT_PUSH_LEASE_SECONDS,
|
|
593
|
+
deployment: str | None = None,
|
|
594
|
+
region: str = "iad1",
|
|
595
|
+
) -> PushDelivery | None:
|
|
596
|
+
message = self.next_push_message(topic, consumer, deployment)
|
|
597
|
+
if message is None:
|
|
598
|
+
return None
|
|
599
|
+
receipt_handle = self.lease(message, consumer, lease_seconds)
|
|
600
|
+
visibility_deadline = message.lease_deadline_by_consumer[consumer]
|
|
601
|
+
self.requeue_push_message(message, consumer, visibility_deadline)
|
|
602
|
+
return PushDelivery(
|
|
603
|
+
body=message.payload.read(),
|
|
604
|
+
headers={
|
|
605
|
+
CLOUD_EVENT_HEADER_TYPE: CLOUD_EVENT_TYPE_V2BETA,
|
|
606
|
+
CLOUD_EVENT_HEADER_VQS_TOPIC: message.topic,
|
|
607
|
+
CLOUD_EVENT_HEADER_VQS_CONSUMER_GROUP: consumer,
|
|
608
|
+
CLOUD_EVENT_HEADER_VQS_MESSAGE_ID: message.message_id,
|
|
609
|
+
CLOUD_EVENT_HEADER_VQS_RECEIPT_HANDLE: receipt_handle,
|
|
610
|
+
CLOUD_EVENT_HEADER_VQS_DELIVERY_COUNT: str(
|
|
611
|
+
message.delivery_count_by_consumer[consumer]
|
|
612
|
+
),
|
|
613
|
+
CLOUD_EVENT_HEADER_VQS_CREATED_AT: _format_dt(message.created_at),
|
|
614
|
+
CLOUD_EVENT_HEADER_VQS_EXPIRES_AT: _format_dt(message.expires_at),
|
|
615
|
+
CLOUD_EVENT_HEADER_VQS_VISIBILITY_DEADLINE: _format_dt(visibility_deadline),
|
|
616
|
+
CLOUD_EVENT_HEADER_VQS_REGION: region,
|
|
617
|
+
HEADER_CONTENT_TYPE: message.content_type,
|
|
618
|
+
},
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
def next_visible_delay(self, topics: Iterable[str] | None = None) -> float | None:
|
|
622
|
+
self.cleanup_expired()
|
|
623
|
+
topic_set = set(topics) if topics is not None else None
|
|
624
|
+
next_deadline: datetime | None = None
|
|
625
|
+
now = self.state.now
|
|
626
|
+
for message in self.state.messages:
|
|
627
|
+
if message.duplicate_of is not None:
|
|
628
|
+
continue
|
|
629
|
+
if topic_set is not None and message.topic not in topic_set:
|
|
630
|
+
continue
|
|
631
|
+
if message.expires_at <= now:
|
|
632
|
+
continue
|
|
633
|
+
if next_deadline is None or message.available_at < next_deadline:
|
|
634
|
+
next_deadline = message.available_at
|
|
635
|
+
for deadline in message.lease_deadline_by_consumer.values():
|
|
636
|
+
if deadline > now and (next_deadline is None or deadline < next_deadline):
|
|
637
|
+
next_deadline = deadline
|
|
638
|
+
if next_deadline is None:
|
|
639
|
+
return None
|
|
640
|
+
return max(0.0, (next_deadline - now).total_seconds())
|
|
641
|
+
|
|
642
|
+
def cleanup_expired(self) -> None:
|
|
643
|
+
state = self.state
|
|
644
|
+
now = state.now
|
|
645
|
+
if state.next_expires_at is not None and state.next_expires_at > now:
|
|
646
|
+
return
|
|
647
|
+
expired = [message for message in state.messages if message.expires_at <= now]
|
|
648
|
+
if not expired:
|
|
649
|
+
state.next_expires_at = min(
|
|
650
|
+
(message.expires_at for message in state.messages),
|
|
651
|
+
default=None,
|
|
652
|
+
)
|
|
653
|
+
return
|
|
654
|
+
expired_ids = {message.message_id for message in expired}
|
|
655
|
+
state.messages = [
|
|
656
|
+
message for message in state.messages if message.message_id not in expired_ids
|
|
657
|
+
]
|
|
658
|
+
for topic, messages in list(state.by_topic.items()):
|
|
659
|
+
live = [message for message in messages if message.message_id not in expired_ids]
|
|
660
|
+
if live:
|
|
661
|
+
state.by_topic[topic] = live
|
|
662
|
+
else:
|
|
663
|
+
state.by_topic.pop(topic, None)
|
|
664
|
+
state.push_delivery_indexes.clear()
|
|
665
|
+
for message_id in expired_ids:
|
|
666
|
+
message = state.by_id.pop(message_id, None)
|
|
667
|
+
if message is not None:
|
|
668
|
+
for receipt_handle in message.leases_by_consumer.values():
|
|
669
|
+
state.by_receipt.pop(receipt_handle, None)
|
|
670
|
+
message.payload.delete()
|
|
671
|
+
state.next_expires_at = min(
|
|
672
|
+
(message.expires_at for message in state.messages),
|
|
673
|
+
default=None,
|
|
674
|
+
)
|
|
675
|
+
|
|
676
|
+
def _can_deliver(
|
|
677
|
+
self,
|
|
678
|
+
message: StoredMessage,
|
|
679
|
+
topic: str,
|
|
680
|
+
consumer: str,
|
|
681
|
+
deployment: str | None,
|
|
682
|
+
) -> bool:
|
|
683
|
+
now = self.state.now
|
|
684
|
+
if (
|
|
685
|
+
message.topic != topic
|
|
686
|
+
or message.acknowledged_for(consumer)
|
|
687
|
+
or message.duplicate_of is not None
|
|
688
|
+
):
|
|
689
|
+
return False
|
|
690
|
+
if message.expires_at <= now or message.available_at > now:
|
|
691
|
+
return False
|
|
692
|
+
if deployment is not None and message.deployment != deployment:
|
|
693
|
+
return False
|
|
694
|
+
return message.lease_deadline_by_consumer.get(consumer, now) <= now
|
|
695
|
+
|
|
696
|
+
def requeue_push_message(
|
|
697
|
+
self,
|
|
698
|
+
message: StoredMessage,
|
|
699
|
+
consumer: str,
|
|
700
|
+
deadline: datetime,
|
|
701
|
+
) -> None:
|
|
702
|
+
deployments: set[str | None] = {None}
|
|
703
|
+
if message.deployment != DEFAULT_DEPLOYMENT_PARTITION:
|
|
704
|
+
deployments.add(message.deployment)
|
|
705
|
+
wake = deadline <= self.state.now
|
|
706
|
+
for deployment in deployments:
|
|
707
|
+
index = self.state.push_delivery_indexes.get(
|
|
708
|
+
_PushIndexKey(message.topic, consumer, deployment)
|
|
709
|
+
)
|
|
710
|
+
if index is None:
|
|
711
|
+
continue
|
|
712
|
+
if wake:
|
|
713
|
+
index.ready.append(message.message_id)
|
|
714
|
+
else:
|
|
715
|
+
self._push_block_after(index, deadline, message.message_id)
|
|
716
|
+
if wake:
|
|
717
|
+
self.wake_dispatchers()
|
|
718
|
+
|
|
719
|
+
def _push_delivery_index(
|
|
720
|
+
self,
|
|
721
|
+
topic: str,
|
|
722
|
+
consumer: str,
|
|
723
|
+
deployment: str | None,
|
|
724
|
+
) -> _PushDeliveryIndex:
|
|
725
|
+
return self.state.push_delivery_indexes.setdefault(
|
|
726
|
+
_PushIndexKey(topic, consumer, deployment),
|
|
727
|
+
_PushDeliveryIndex(),
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
def _push_index_messages(self, index: _PushDeliveryIndex, topic: str) -> None:
|
|
731
|
+
messages = self.state.by_topic.get(topic, [])
|
|
732
|
+
start = min(index.cursor, len(messages))
|
|
733
|
+
for message in messages[start:]:
|
|
734
|
+
index.ready.append(message.message_id)
|
|
735
|
+
index.cursor = len(messages)
|
|
736
|
+
|
|
737
|
+
def _push_ready_blocked(self, index: _PushDeliveryIndex, now: datetime) -> None:
|
|
738
|
+
while index.blocked and index.blocked[0][0] <= now:
|
|
739
|
+
_deadline, _sequence, message_id = heapq.heappop(index.blocked)
|
|
740
|
+
index.ready.append(message_id)
|
|
741
|
+
|
|
742
|
+
def _push_block_after(
|
|
743
|
+
self,
|
|
744
|
+
index: _PushDeliveryIndex,
|
|
745
|
+
deadline: datetime,
|
|
746
|
+
message_id: str,
|
|
747
|
+
) -> None:
|
|
748
|
+
heapq.heappush(
|
|
749
|
+
index.blocked,
|
|
750
|
+
(deadline, next(self.state.push_blocked_sequence), message_id),
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
def _message_visible_deadline_for_consumer(
|
|
754
|
+
self,
|
|
755
|
+
message: StoredMessage,
|
|
756
|
+
topic: str,
|
|
757
|
+
consumer: str,
|
|
758
|
+
deployment: str | None,
|
|
759
|
+
now: datetime,
|
|
760
|
+
) -> datetime | None:
|
|
761
|
+
if (
|
|
762
|
+
message.topic != topic
|
|
763
|
+
or message.acknowledged_for(consumer)
|
|
764
|
+
or message.duplicate_of is not None
|
|
765
|
+
):
|
|
766
|
+
return None
|
|
767
|
+
if message.expires_at <= now:
|
|
768
|
+
return None
|
|
769
|
+
if deployment is not None and message.deployment != deployment:
|
|
770
|
+
return None
|
|
771
|
+
deadline = message.lease_deadline_by_consumer.get(consumer)
|
|
772
|
+
if deadline is not None and deadline > now:
|
|
773
|
+
return max(message.available_at, deadline)
|
|
774
|
+
return message.available_at
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
class EmbeddedQueueAsgiApp:
|
|
778
|
+
def __init__(self, server: EmbeddedQueueServer | None = None) -> None:
|
|
779
|
+
self._server = server or EmbeddedQueueServer()
|
|
780
|
+
self._async_client_factory = self._async_http_client_factory
|
|
781
|
+
|
|
782
|
+
@property
|
|
783
|
+
def state(self) -> EmbeddedQueueState:
|
|
784
|
+
return self._server.state
|
|
785
|
+
|
|
786
|
+
@property
|
|
787
|
+
def server(self) -> EmbeddedQueueServer:
|
|
788
|
+
return self._server
|
|
789
|
+
|
|
790
|
+
def get_async_client(
|
|
791
|
+
self,
|
|
792
|
+
*,
|
|
793
|
+
token: str | None = "local-token",
|
|
794
|
+
region: str | None = "iad1",
|
|
795
|
+
base_url: BaseUrl | None = "http://vqs.test",
|
|
796
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
797
|
+
headers: Mapping[str, str] | None = None,
|
|
798
|
+
timeout: Duration | None = 10.0,
|
|
799
|
+
) -> QueueClient:
|
|
800
|
+
return QueueClient(
|
|
801
|
+
token=token,
|
|
802
|
+
region=region,
|
|
803
|
+
base_url=base_url,
|
|
804
|
+
deployment=deployment,
|
|
805
|
+
headers=headers,
|
|
806
|
+
timeout=timeout,
|
|
807
|
+
http_client_factory=self._async_client_factory,
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
def acquire_async_client(
|
|
811
|
+
self,
|
|
812
|
+
*,
|
|
813
|
+
token: str | None = "local-token",
|
|
814
|
+
region: str | None = "iad1",
|
|
815
|
+
base_url: BaseUrl | None = "http://vqs.test",
|
|
816
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
817
|
+
headers: Mapping[str, str] | None = None,
|
|
818
|
+
timeout: Duration | None = 10.0,
|
|
819
|
+
) -> AbstractAsyncContextManager[QueueClient]:
|
|
820
|
+
@asynccontextmanager
|
|
821
|
+
async def _acquire() -> AsyncIterator[QueueClient]:
|
|
822
|
+
yield self.get_async_client(
|
|
823
|
+
token=token,
|
|
824
|
+
region=region,
|
|
825
|
+
base_url=base_url,
|
|
826
|
+
deployment=deployment,
|
|
827
|
+
headers=headers,
|
|
828
|
+
timeout=timeout,
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
return _acquire()
|
|
832
|
+
|
|
833
|
+
def get_sync_client(
|
|
834
|
+
self,
|
|
835
|
+
*,
|
|
836
|
+
token: str | None = "local-token",
|
|
837
|
+
region: str | None = "iad1",
|
|
838
|
+
base_url: BaseUrl | None = "http://vqs.test",
|
|
839
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
840
|
+
headers: Mapping[str, str] | None = None,
|
|
841
|
+
timeout: Duration | None = 10.0,
|
|
842
|
+
) -> SyncQueueClient:
|
|
843
|
+
return SyncQueueClient(
|
|
844
|
+
token=token,
|
|
845
|
+
region=region,
|
|
846
|
+
base_url=base_url,
|
|
847
|
+
deployment=deployment,
|
|
848
|
+
headers=headers,
|
|
849
|
+
timeout=timeout,
|
|
850
|
+
)
|
|
851
|
+
|
|
852
|
+
def acquire_sync_client(
|
|
853
|
+
self,
|
|
854
|
+
*,
|
|
855
|
+
token: str | None = "local-token",
|
|
856
|
+
region: str | None = "iad1",
|
|
857
|
+
base_url: BaseUrl | None = "http://vqs.test",
|
|
858
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
859
|
+
headers: Mapping[str, str] | None = None,
|
|
860
|
+
timeout: Duration | None = 10.0,
|
|
861
|
+
) -> AbstractContextManager[SyncQueueClient]:
|
|
862
|
+
@contextmanager
|
|
863
|
+
def _acquire() -> Iterator[SyncQueueClient]:
|
|
864
|
+
yield self.get_sync_client(
|
|
865
|
+
token=token,
|
|
866
|
+
region=region,
|
|
867
|
+
base_url=base_url,
|
|
868
|
+
deployment=deployment,
|
|
869
|
+
headers=headers,
|
|
870
|
+
timeout=timeout,
|
|
871
|
+
)
|
|
872
|
+
|
|
873
|
+
return _acquire()
|
|
874
|
+
|
|
875
|
+
def reset_clients(self) -> None:
|
|
876
|
+
pass
|
|
877
|
+
|
|
878
|
+
def _async_http_client_factory(self, **kwargs: object) -> httpx.AsyncClient:
|
|
879
|
+
del kwargs
|
|
880
|
+
return httpx.AsyncClient(
|
|
881
|
+
transport=httpx.ASGITransport(app=self),
|
|
882
|
+
base_url="http://vqs.test",
|
|
883
|
+
)
|
|
884
|
+
|
|
885
|
+
def create_async_client_factory(
|
|
886
|
+
self,
|
|
887
|
+
) -> Callable[..., httpx.AsyncClient]:
|
|
888
|
+
return self._async_client_factory
|
|
889
|
+
|
|
890
|
+
def iter_push_deliveries(
|
|
891
|
+
self,
|
|
892
|
+
topic: str,
|
|
893
|
+
consumer: str,
|
|
894
|
+
*,
|
|
895
|
+
lease_seconds: int = DEFAULT_PUSH_LEASE_SECONDS,
|
|
896
|
+
deployment: str | None = None,
|
|
897
|
+
region: str = "iad1",
|
|
898
|
+
) -> Iterator[PushDelivery]:
|
|
899
|
+
while True:
|
|
900
|
+
delivery = self._server.push_delivery(
|
|
901
|
+
topic,
|
|
902
|
+
consumer,
|
|
903
|
+
lease_seconds=lease_seconds,
|
|
904
|
+
deployment=deployment,
|
|
905
|
+
region=region,
|
|
906
|
+
)
|
|
907
|
+
if delivery is None:
|
|
908
|
+
return
|
|
909
|
+
yield delivery
|
|
910
|
+
|
|
911
|
+
async def __call__(
|
|
912
|
+
self,
|
|
913
|
+
scope: MutableMapping[str, Any],
|
|
914
|
+
receive: Callable[[], Any],
|
|
915
|
+
send: Callable[[MutableMapping[str, Any]], Any],
|
|
916
|
+
) -> None:
|
|
917
|
+
if scope["type"] != "http":
|
|
918
|
+
raise RuntimeError(f"unsupported ASGI scope type: {scope['type']!r}")
|
|
919
|
+
request = _AsgiRequest(scope, receive)
|
|
920
|
+
response = await self.handle(request)
|
|
921
|
+
await response.send(send)
|
|
922
|
+
|
|
923
|
+
async def handle(self, request: _AsgiRequest) -> _AsgiResponse:
|
|
924
|
+
route = _parse_queue_route(request.path_parts)
|
|
925
|
+
if route is None:
|
|
926
|
+
return _AsgiResponse(404)
|
|
927
|
+
|
|
928
|
+
await self._server.record_request(request)
|
|
929
|
+
|
|
930
|
+
try:
|
|
931
|
+
_validate_route(route)
|
|
932
|
+
except ValueError as exc:
|
|
933
|
+
return _bad_request(str(exc))
|
|
934
|
+
|
|
935
|
+
if response := self._server.response_for(request, route):
|
|
936
|
+
return response
|
|
937
|
+
|
|
938
|
+
match request.method, route.action:
|
|
939
|
+
case "POST", "topic":
|
|
940
|
+
return await self._send_message(route.topic, request)
|
|
941
|
+
case "POST", "consumer" if route.consumer is not None:
|
|
942
|
+
return self._poll_messages(route.topic, route.consumer, request)
|
|
943
|
+
case "POST", "message_id" if route.consumer is not None and route.message_id:
|
|
944
|
+
return self._poll_message_by_id(
|
|
945
|
+
route.topic,
|
|
946
|
+
route.consumer,
|
|
947
|
+
route.message_id,
|
|
948
|
+
request,
|
|
949
|
+
)
|
|
950
|
+
case (("DELETE" | "PATCH"), "lease") if route.consumer and route.receipt_handle:
|
|
951
|
+
return await self._handle_lease(
|
|
952
|
+
route.topic,
|
|
953
|
+
route.consumer,
|
|
954
|
+
route.receipt_handle,
|
|
955
|
+
request,
|
|
956
|
+
)
|
|
957
|
+
case "PATCH", "lease_visibility" if route.consumer and route.receipt_handle:
|
|
958
|
+
return await self._handle_lease(
|
|
959
|
+
route.topic,
|
|
960
|
+
route.consumer,
|
|
961
|
+
route.receipt_handle,
|
|
962
|
+
request,
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
return _AsgiResponse(404)
|
|
966
|
+
|
|
967
|
+
async def _send_message(self, topic: str, request: _AsgiRequest) -> _AsgiResponse:
|
|
968
|
+
try:
|
|
969
|
+
message = self._server.put(
|
|
970
|
+
topic,
|
|
971
|
+
await request.body(),
|
|
972
|
+
request.headers,
|
|
973
|
+
)
|
|
974
|
+
except ValueError as exc:
|
|
975
|
+
return _bad_request(str(exc))
|
|
976
|
+
return _json_response(
|
|
977
|
+
201,
|
|
978
|
+
{"messageId": message.message_id},
|
|
979
|
+
headers=[(VQS_HEADER_MESSAGE_ID.encode(), message.message_id.encode())],
|
|
980
|
+
)
|
|
981
|
+
|
|
982
|
+
def _poll_messages(self, topic: str, consumer: str, request: _AsgiRequest) -> _AsgiResponse:
|
|
983
|
+
response_format = _receive_response_format(request)
|
|
984
|
+
if isinstance(response_format, _AsgiResponse):
|
|
985
|
+
return response_format
|
|
986
|
+
try:
|
|
987
|
+
limit = _int_header(
|
|
988
|
+
request.headers.get(VQS_HEADER_MAX_MESSAGES),
|
|
989
|
+
1,
|
|
990
|
+
minimum=1,
|
|
991
|
+
maximum=10,
|
|
992
|
+
name=VQS_HEADER_MAX_MESSAGES,
|
|
993
|
+
)
|
|
994
|
+
lease_seconds = _int_header(
|
|
995
|
+
request.headers.get(VQS_HEADER_VISIBILITY_TIMEOUT_SECONDS),
|
|
996
|
+
DEFAULT_PULL_LEASE_SECONDS,
|
|
997
|
+
minimum=0,
|
|
998
|
+
maximum=MAX_VISIBILITY_TIMEOUT_SECONDS,
|
|
999
|
+
name=VQS_HEADER_VISIBILITY_TIMEOUT_SECONDS,
|
|
1000
|
+
)
|
|
1001
|
+
deployment = _deployment_header(request.headers)
|
|
1002
|
+
except ValueError as exc:
|
|
1003
|
+
return _bad_request(str(exc))
|
|
1004
|
+
messages = self._server.visible_messages(topic, consumer, deployment)[:limit]
|
|
1005
|
+
if not messages:
|
|
1006
|
+
debug_log(
|
|
1007
|
+
"receive.empty",
|
|
1008
|
+
topic=topic,
|
|
1009
|
+
consumer_group=consumer,
|
|
1010
|
+
status_code=204,
|
|
1011
|
+
)
|
|
1012
|
+
return _AsgiResponse(204)
|
|
1013
|
+
leased = [
|
|
1014
|
+
_LeasedMessage(
|
|
1015
|
+
message=message,
|
|
1016
|
+
receipt_handle=self._server.lease(message, consumer, lease_seconds),
|
|
1017
|
+
consumer=consumer,
|
|
1018
|
+
)
|
|
1019
|
+
for message in messages
|
|
1020
|
+
]
|
|
1021
|
+
return _receive_response(response_format, leased)
|
|
1022
|
+
|
|
1023
|
+
def _poll_message_by_id(
|
|
1024
|
+
self,
|
|
1025
|
+
topic: str,
|
|
1026
|
+
consumer: str,
|
|
1027
|
+
message_id: str,
|
|
1028
|
+
request: _AsgiRequest,
|
|
1029
|
+
) -> _AsgiResponse:
|
|
1030
|
+
response_format = _receive_response_format(request)
|
|
1031
|
+
if isinstance(response_format, _AsgiResponse):
|
|
1032
|
+
return response_format
|
|
1033
|
+
try:
|
|
1034
|
+
deployment = _deployment_header(request.headers)
|
|
1035
|
+
lease_seconds = _int_header(
|
|
1036
|
+
request.headers.get(VQS_HEADER_VISIBILITY_TIMEOUT_SECONDS),
|
|
1037
|
+
DEFAULT_PULL_LEASE_SECONDS,
|
|
1038
|
+
minimum=0,
|
|
1039
|
+
maximum=MAX_VISIBILITY_TIMEOUT_SECONDS,
|
|
1040
|
+
name=VQS_HEADER_VISIBILITY_TIMEOUT_SECONDS,
|
|
1041
|
+
)
|
|
1042
|
+
except ValueError as exc:
|
|
1043
|
+
return _bad_request(str(exc))
|
|
1044
|
+
found_message = self._server.find_by_id(topic, message_id)
|
|
1045
|
+
if (
|
|
1046
|
+
found_message is None
|
|
1047
|
+
or found_message.expires_at <= self._server.now
|
|
1048
|
+
or (deployment is not None and found_message.deployment != deployment)
|
|
1049
|
+
):
|
|
1050
|
+
return _json_response(404, {"error": "Message not found"})
|
|
1051
|
+
if found_message.duplicate_of is not None:
|
|
1052
|
+
debug_log(
|
|
1053
|
+
"receive.redirect_duplicate",
|
|
1054
|
+
requested_message_id=message_id,
|
|
1055
|
+
original_message_id=found_message.duplicate_of,
|
|
1056
|
+
)
|
|
1057
|
+
return _json_response(
|
|
1058
|
+
409,
|
|
1059
|
+
{
|
|
1060
|
+
"error": "This messageId was a duplicate - use originalMessageId instead",
|
|
1061
|
+
"originalMessageId": found_message.duplicate_of,
|
|
1062
|
+
},
|
|
1063
|
+
)
|
|
1064
|
+
if found_message.acknowledged_for(consumer):
|
|
1065
|
+
return _AsgiResponse(410)
|
|
1066
|
+
deadline = found_message.lease_deadline_by_consumer.get(consumer)
|
|
1067
|
+
if deadline is not None and deadline > self._server.now:
|
|
1068
|
+
retry_after = max(1, int((deadline - self._server.now).total_seconds()))
|
|
1069
|
+
return _json_response(
|
|
1070
|
+
409,
|
|
1071
|
+
{"error": "Message is locked by another consumer"},
|
|
1072
|
+
headers=[(b"retry-after", str(retry_after).encode())],
|
|
1073
|
+
)
|
|
1074
|
+
receipt_handle = self._server.lease(found_message, consumer, lease_seconds)
|
|
1075
|
+
return _receive_response(
|
|
1076
|
+
response_format,
|
|
1077
|
+
[
|
|
1078
|
+
_LeasedMessage(
|
|
1079
|
+
message=found_message,
|
|
1080
|
+
receipt_handle=receipt_handle,
|
|
1081
|
+
consumer=consumer,
|
|
1082
|
+
)
|
|
1083
|
+
],
|
|
1084
|
+
)
|
|
1085
|
+
|
|
1086
|
+
async def _handle_lease(
|
|
1087
|
+
self,
|
|
1088
|
+
topic: str,
|
|
1089
|
+
consumer: str,
|
|
1090
|
+
receipt_handle: str,
|
|
1091
|
+
request: _AsgiRequest,
|
|
1092
|
+
) -> _AsgiResponse:
|
|
1093
|
+
try:
|
|
1094
|
+
deployment = _deployment_header(request.headers)
|
|
1095
|
+
except ValueError as exc:
|
|
1096
|
+
return _bad_request(str(exc))
|
|
1097
|
+
receipt_message = self._server.receipt_message(
|
|
1098
|
+
topic,
|
|
1099
|
+
consumer,
|
|
1100
|
+
receipt_handle,
|
|
1101
|
+
deployment,
|
|
1102
|
+
)
|
|
1103
|
+
if receipt_message is None:
|
|
1104
|
+
if self._server.receipt_exists(topic, receipt_handle, deployment):
|
|
1105
|
+
return _json_response(409, {"error": "Message is not currently in-flight"})
|
|
1106
|
+
return _json_response(404, {"error": "Message not found"})
|
|
1107
|
+
if receipt_message.acknowledged_for(consumer):
|
|
1108
|
+
return _json_response(409, {"error": "Message is not currently in-flight"})
|
|
1109
|
+
deadline = receipt_message.lease_deadline_by_consumer.get(consumer)
|
|
1110
|
+
if deadline is None or deadline <= self._server.now:
|
|
1111
|
+
return _json_response(409, {"error": "Message lease has expired"})
|
|
1112
|
+
if request.method == "DELETE":
|
|
1113
|
+
receipt_message.acknowledge_for(consumer)
|
|
1114
|
+
self._server.state.by_receipt.pop(receipt_handle, None)
|
|
1115
|
+
return _AsgiResponse(204)
|
|
1116
|
+
if request.method != "PATCH":
|
|
1117
|
+
return _AsgiResponse(404)
|
|
1118
|
+
|
|
1119
|
+
try:
|
|
1120
|
+
body = json.loads((await request.body()).decode() or "{}")
|
|
1121
|
+
seconds = _visibility_timeout_seconds_from_body(body)
|
|
1122
|
+
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
1123
|
+
return _bad_request(str(exc))
|
|
1124
|
+
new_deadline = self._server.now + timedelta(seconds=seconds)
|
|
1125
|
+
if new_deadline > receipt_message.expires_at:
|
|
1126
|
+
return _json_response(
|
|
1127
|
+
400,
|
|
1128
|
+
{
|
|
1129
|
+
"error": "Visibility timeout cannot extend beyond message expiration",
|
|
1130
|
+
"messageExpiresAt": _format_dt(receipt_message.expires_at),
|
|
1131
|
+
"requestedExpiresAt": _format_dt(new_deadline),
|
|
1132
|
+
},
|
|
1133
|
+
)
|
|
1134
|
+
receipt_message.lease_deadline_by_consumer[consumer] = new_deadline
|
|
1135
|
+
self._server.requeue_push_message(receipt_message, consumer, new_deadline)
|
|
1136
|
+
return _json_response(200, {"success": True})
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
class EmbeddedQueueDispatcher:
|
|
1140
|
+
def __init__(
|
|
1141
|
+
self,
|
|
1142
|
+
server: EmbeddedQueueServer,
|
|
1143
|
+
client_factory: Callable[[], EmbeddedQueuePushClient],
|
|
1144
|
+
) -> None:
|
|
1145
|
+
self._server = server
|
|
1146
|
+
self._client_factory = client_factory
|
|
1147
|
+
self._registrations: list[_DispatcherRegistration] = []
|
|
1148
|
+
self._inflight: set[tuple[tuple[str, SubscriptionMatchKind, str], str]] = set()
|
|
1149
|
+
self._inflight_counts: dict[tuple[str, SubscriptionMatchKind, str], int] = {}
|
|
1150
|
+
self._wake_send, self._wake_receive = anyio.create_memory_object_stream[None](1)
|
|
1151
|
+
self._server.add_wake_callback(self.wake)
|
|
1152
|
+
register_embedded_dispatcher(self)
|
|
1153
|
+
|
|
1154
|
+
@property
|
|
1155
|
+
def server(self) -> EmbeddedQueueServer:
|
|
1156
|
+
return self._server
|
|
1157
|
+
|
|
1158
|
+
def register_subscription(
|
|
1159
|
+
self,
|
|
1160
|
+
*,
|
|
1161
|
+
topic: str,
|
|
1162
|
+
consumer_group: str,
|
|
1163
|
+
retry_after_seconds: int | None,
|
|
1164
|
+
initial_delay_seconds: int | None,
|
|
1165
|
+
max_concurrency: int | None,
|
|
1166
|
+
max_attempts: int | None,
|
|
1167
|
+
) -> None:
|
|
1168
|
+
if topic == "*":
|
|
1169
|
+
topic_pattern = topic
|
|
1170
|
+
match_kind: SubscriptionMatchKind = "wildcard"
|
|
1171
|
+
elif topic.endswith("*"):
|
|
1172
|
+
topic_pattern = topic[:-1]
|
|
1173
|
+
match_kind = "prefix"
|
|
1174
|
+
else:
|
|
1175
|
+
topic_pattern = topic
|
|
1176
|
+
match_kind = "exact"
|
|
1177
|
+
registration_key = (topic_pattern, match_kind, consumer_group)
|
|
1178
|
+
if any(registration.key() == registration_key for registration in self._registrations):
|
|
1179
|
+
return
|
|
1180
|
+
self._registrations.append(
|
|
1181
|
+
_DispatcherRegistration(
|
|
1182
|
+
topic_pattern=topic_pattern,
|
|
1183
|
+
match_kind=match_kind,
|
|
1184
|
+
consumer_group=consumer_group,
|
|
1185
|
+
retry_after_seconds=retry_after_seconds,
|
|
1186
|
+
initial_delay_seconds=initial_delay_seconds,
|
|
1187
|
+
max_concurrency=max_concurrency,
|
|
1188
|
+
max_attempts=max_attempts,
|
|
1189
|
+
registered_at=self._server.now,
|
|
1190
|
+
concurrency_limit=(
|
|
1191
|
+
max(1, max_concurrency) if max_concurrency is not None else None
|
|
1192
|
+
),
|
|
1193
|
+
semaphore=(
|
|
1194
|
+
anyio.Semaphore(max(1, max_concurrency))
|
|
1195
|
+
if max_concurrency is not None
|
|
1196
|
+
else None
|
|
1197
|
+
),
|
|
1198
|
+
)
|
|
1199
|
+
)
|
|
1200
|
+
debug_log(
|
|
1201
|
+
"embedded.subscription_registered",
|
|
1202
|
+
topic_pattern=topic_pattern,
|
|
1203
|
+
match_kind=match_kind,
|
|
1204
|
+
consumer_group=consumer_group,
|
|
1205
|
+
retry_after_seconds=retry_after_seconds,
|
|
1206
|
+
initial_delay_seconds=initial_delay_seconds,
|
|
1207
|
+
max_concurrency=max_concurrency,
|
|
1208
|
+
max_attempts=max_attempts,
|
|
1209
|
+
)
|
|
1210
|
+
self.wake()
|
|
1211
|
+
|
|
1212
|
+
def wake(self) -> None:
|
|
1213
|
+
try:
|
|
1214
|
+
self._wake_send.send_nowait(None)
|
|
1215
|
+
except anyio.WouldBlock:
|
|
1216
|
+
pass
|
|
1217
|
+
except anyio.ClosedResourceError:
|
|
1218
|
+
pass
|
|
1219
|
+
|
|
1220
|
+
async def run(self) -> None:
|
|
1221
|
+
async with anyio.create_task_group() as task_group:
|
|
1222
|
+
while True:
|
|
1223
|
+
delivered = False
|
|
1224
|
+
registrations = list(self._registrations)
|
|
1225
|
+
for registration in registrations:
|
|
1226
|
+
delivered = self._schedule_registration(registration, task_group) or delivered
|
|
1227
|
+
|
|
1228
|
+
delay = self._sleep_delay(delivered=delivered)
|
|
1229
|
+
if not delivered:
|
|
1230
|
+
debug_log("embedded.no_message_after_wake", sleep_delay_seconds=delay)
|
|
1231
|
+
self._drain_wake()
|
|
1232
|
+
with anyio.move_on_after(delay):
|
|
1233
|
+
await self._wake_receive.receive()
|
|
1234
|
+
|
|
1235
|
+
def _schedule_registration(
|
|
1236
|
+
self,
|
|
1237
|
+
registration: _DispatcherRegistration,
|
|
1238
|
+
task_group: TaskGroup,
|
|
1239
|
+
) -> bool:
|
|
1240
|
+
if not self._registration_ready(registration):
|
|
1241
|
+
return False
|
|
1242
|
+
|
|
1243
|
+
delivered = False
|
|
1244
|
+
for topic in self._topics_for_registration(registration):
|
|
1245
|
+
delivered = self._schedule_topic(registration, topic, task_group) or delivered
|
|
1246
|
+
return delivered
|
|
1247
|
+
|
|
1248
|
+
def _schedule_topic(
|
|
1249
|
+
self,
|
|
1250
|
+
registration: _DispatcherRegistration,
|
|
1251
|
+
topic: str,
|
|
1252
|
+
task_group: TaskGroup,
|
|
1253
|
+
) -> bool:
|
|
1254
|
+
delivered = False
|
|
1255
|
+
while self._can_schedule(registration):
|
|
1256
|
+
delivery = self._server.push_delivery(
|
|
1257
|
+
topic,
|
|
1258
|
+
registration.consumer_group,
|
|
1259
|
+
)
|
|
1260
|
+
if delivery is None:
|
|
1261
|
+
break
|
|
1262
|
+
delivered = True
|
|
1263
|
+
message_id = delivery.headers[CLOUD_EVENT_HEADER_VQS_MESSAGE_ID]
|
|
1264
|
+
registration_key = registration.key()
|
|
1265
|
+
self._inflight.add((registration_key, message_id))
|
|
1266
|
+
self._inflight_counts[registration_key] = (
|
|
1267
|
+
self._inflight_counts.get(registration_key, 0) + 1
|
|
1268
|
+
)
|
|
1269
|
+
debug_log(
|
|
1270
|
+
"embedded.delivery_scheduled",
|
|
1271
|
+
topic=topic,
|
|
1272
|
+
consumer_group=registration.consumer_group,
|
|
1273
|
+
)
|
|
1274
|
+
task_group.start_soon(
|
|
1275
|
+
self._deliver,
|
|
1276
|
+
registration,
|
|
1277
|
+
topic,
|
|
1278
|
+
message_id,
|
|
1279
|
+
delivery,
|
|
1280
|
+
name="vercel-embedded-queue-delivery",
|
|
1281
|
+
)
|
|
1282
|
+
return delivered
|
|
1283
|
+
|
|
1284
|
+
async def aclose(self) -> None:
|
|
1285
|
+
await self._wake_send.aclose()
|
|
1286
|
+
await self._wake_receive.aclose()
|
|
1287
|
+
|
|
1288
|
+
def unregister(self) -> None:
|
|
1289
|
+
unregister_embedded_dispatcher(self)
|
|
1290
|
+
|
|
1291
|
+
def _drain_wake(self) -> None:
|
|
1292
|
+
while True:
|
|
1293
|
+
try:
|
|
1294
|
+
self._wake_receive.receive_nowait()
|
|
1295
|
+
except anyio.WouldBlock:
|
|
1296
|
+
return
|
|
1297
|
+
except anyio.EndOfStream:
|
|
1298
|
+
return
|
|
1299
|
+
|
|
1300
|
+
def _sleep_delay(self, *, delivered: bool) -> float:
|
|
1301
|
+
if delivered:
|
|
1302
|
+
return 0.0
|
|
1303
|
+
next_delay = self._next_registration_visible_delay()
|
|
1304
|
+
if next_delay is None:
|
|
1305
|
+
return 0.1
|
|
1306
|
+
return min(max(next_delay, 0.0), 0.1)
|
|
1307
|
+
|
|
1308
|
+
def _next_registration_visible_delay(self) -> float | None:
|
|
1309
|
+
self._server.cleanup_expired()
|
|
1310
|
+
now = self._server.now
|
|
1311
|
+
next_deadline: datetime | None = None
|
|
1312
|
+
for registration in self._registrations:
|
|
1313
|
+
if not self._registration_ready(registration):
|
|
1314
|
+
ready_at = registration.registered_at + timedelta(
|
|
1315
|
+
seconds=registration.initial_delay_seconds or 0
|
|
1316
|
+
)
|
|
1317
|
+
next_deadline = _min_datetime(next_deadline, ready_at)
|
|
1318
|
+
continue
|
|
1319
|
+
if not self._can_schedule(registration):
|
|
1320
|
+
continue
|
|
1321
|
+
for topic in self._topics_for_registration(registration):
|
|
1322
|
+
topic_messages = self._server.state.by_topic.get(topic, [])
|
|
1323
|
+
for message in topic_messages:
|
|
1324
|
+
deadline = self._message_visible_deadline(message, registration, now)
|
|
1325
|
+
if deadline is None:
|
|
1326
|
+
continue
|
|
1327
|
+
if deadline <= now:
|
|
1328
|
+
return 0.0
|
|
1329
|
+
next_deadline = _min_datetime(next_deadline, deadline)
|
|
1330
|
+
if next_deadline is None:
|
|
1331
|
+
return None
|
|
1332
|
+
return max(0.0, (next_deadline - now).total_seconds())
|
|
1333
|
+
|
|
1334
|
+
def _message_visible_deadline(
|
|
1335
|
+
self,
|
|
1336
|
+
message: StoredMessage,
|
|
1337
|
+
registration: _DispatcherRegistration,
|
|
1338
|
+
now: datetime,
|
|
1339
|
+
) -> datetime | None:
|
|
1340
|
+
if not registration.matches_topic(message.topic):
|
|
1341
|
+
return None
|
|
1342
|
+
if message.duplicate_of is not None or message.expires_at <= now:
|
|
1343
|
+
return None
|
|
1344
|
+
if message.acknowledged_for(registration.consumer_group):
|
|
1345
|
+
return None
|
|
1346
|
+
deadline = message.lease_deadline_by_consumer.get(registration.consumer_group)
|
|
1347
|
+
if deadline is not None and deadline > now:
|
|
1348
|
+
return max(message.available_at, deadline)
|
|
1349
|
+
return message.available_at
|
|
1350
|
+
|
|
1351
|
+
def _registration_ready(self, registration: _DispatcherRegistration) -> bool:
|
|
1352
|
+
if registration.initial_delay_seconds is None:
|
|
1353
|
+
return True
|
|
1354
|
+
ready_at = registration.registered_at + timedelta(
|
|
1355
|
+
seconds=registration.initial_delay_seconds
|
|
1356
|
+
)
|
|
1357
|
+
return self._server.now >= ready_at
|
|
1358
|
+
|
|
1359
|
+
def _topics_for_registration(self, registration: _DispatcherRegistration) -> list[str]:
|
|
1360
|
+
topics = [
|
|
1361
|
+
topic
|
|
1362
|
+
for topic in self._server.state.by_topic
|
|
1363
|
+
if registration.matches_topic(topic) and self._can_schedule(registration)
|
|
1364
|
+
]
|
|
1365
|
+
return list(dict.fromkeys(topics))
|
|
1366
|
+
|
|
1367
|
+
def _can_schedule(self, registration: _DispatcherRegistration) -> bool:
|
|
1368
|
+
if registration.concurrency_limit is None:
|
|
1369
|
+
return True
|
|
1370
|
+
return self._inflight_counts.get(registration.key(), 0) < registration.concurrency_limit
|
|
1371
|
+
|
|
1372
|
+
async def _deliver(
|
|
1373
|
+
self,
|
|
1374
|
+
registration: _DispatcherRegistration,
|
|
1375
|
+
topic: str,
|
|
1376
|
+
message_id: str,
|
|
1377
|
+
delivery: PushDelivery,
|
|
1378
|
+
) -> None:
|
|
1379
|
+
try:
|
|
1380
|
+
semaphore = registration.semaphore
|
|
1381
|
+
if semaphore is None:
|
|
1382
|
+
await self._deliver_once(registration, delivery)
|
|
1383
|
+
else:
|
|
1384
|
+
async with semaphore:
|
|
1385
|
+
await self._deliver_once(registration, delivery)
|
|
1386
|
+
finally:
|
|
1387
|
+
del topic
|
|
1388
|
+
registration_key = registration.key()
|
|
1389
|
+
self._inflight.discard((registration_key, message_id))
|
|
1390
|
+
count = self._inflight_counts.get(registration_key, 0)
|
|
1391
|
+
if count <= 1:
|
|
1392
|
+
self._inflight_counts.pop(registration_key, None)
|
|
1393
|
+
else:
|
|
1394
|
+
self._inflight_counts[registration_key] = count - 1
|
|
1395
|
+
self.wake()
|
|
1396
|
+
|
|
1397
|
+
async def _deliver_once(
|
|
1398
|
+
self,
|
|
1399
|
+
registration: _DispatcherRegistration,
|
|
1400
|
+
delivery: PushDelivery,
|
|
1401
|
+
) -> None:
|
|
1402
|
+
try:
|
|
1403
|
+
client = self._client_factory()
|
|
1404
|
+
await client.accept_and_handle(delivery.body, delivery.headers)
|
|
1405
|
+
debug_log(
|
|
1406
|
+
"embedded.delivery_success",
|
|
1407
|
+
message_id=delivery.headers[CLOUD_EVENT_HEADER_VQS_MESSAGE_ID],
|
|
1408
|
+
topic=delivery.headers[CLOUD_EVENT_HEADER_VQS_TOPIC],
|
|
1409
|
+
consumer_group=registration.consumer_group,
|
|
1410
|
+
)
|
|
1411
|
+
except Exception: # noqa: BLE001
|
|
1412
|
+
message = self._server.state.by_id.get(
|
|
1413
|
+
delivery.headers[CLOUD_EVENT_HEADER_VQS_MESSAGE_ID]
|
|
1414
|
+
)
|
|
1415
|
+
debug_log(
|
|
1416
|
+
"embedded.delivery_failure",
|
|
1417
|
+
message_id=delivery.headers[CLOUD_EVENT_HEADER_VQS_MESSAGE_ID],
|
|
1418
|
+
topic=delivery.headers[CLOUD_EVENT_HEADER_VQS_TOPIC],
|
|
1419
|
+
consumer_group=registration.consumer_group,
|
|
1420
|
+
)
|
|
1421
|
+
if message is None:
|
|
1422
|
+
return
|
|
1423
|
+
attempts = message.delivery_count_by_consumer.get(registration.consumer_group, 0)
|
|
1424
|
+
if registration.max_attempts is not None and attempts >= registration.max_attempts:
|
|
1425
|
+
message.acknowledge_for(registration.consumer_group)
|
|
1426
|
+
debug_log(
|
|
1427
|
+
"embedded.max_attempts_acknowledgement",
|
|
1428
|
+
message_id=message.message_id,
|
|
1429
|
+
topic=message.topic,
|
|
1430
|
+
consumer_group=registration.consumer_group,
|
|
1431
|
+
attempts=attempts,
|
|
1432
|
+
)
|
|
1433
|
+
return
|
|
1434
|
+
retry_after_seconds = registration.retry_after_seconds or DEFAULT_RETRY_AFTER_SECONDS
|
|
1435
|
+
message.lease_deadline_by_consumer[registration.consumer_group] = (
|
|
1436
|
+
self._server.now + timedelta(seconds=retry_after_seconds)
|
|
1437
|
+
)
|
|
1438
|
+
self._server.requeue_push_message(
|
|
1439
|
+
message,
|
|
1440
|
+
registration.consumer_group,
|
|
1441
|
+
message.lease_deadline_by_consumer[registration.consumer_group],
|
|
1442
|
+
)
|
|
1443
|
+
debug_log(
|
|
1444
|
+
"embedded.retry_after_applied",
|
|
1445
|
+
message_id=message.message_id,
|
|
1446
|
+
topic=message.topic,
|
|
1447
|
+
consumer_group=registration.consumer_group,
|
|
1448
|
+
retry_after_seconds=retry_after_seconds,
|
|
1449
|
+
)
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
@dataclass(frozen=True)
|
|
1453
|
+
class EmbeddedQueueDevServer:
|
|
1454
|
+
"""A running embedded queue dev server."""
|
|
1455
|
+
|
|
1456
|
+
state: EmbeddedQueueState
|
|
1457
|
+
base_url: str
|
|
1458
|
+
app: EmbeddedQueueAsgiApp
|
|
1459
|
+
_thread: threading.Thread | None = field(default=None, repr=False, compare=False)
|
|
1460
|
+
|
|
1461
|
+
def is_running(self) -> bool:
|
|
1462
|
+
"""Return whether the embedded HTTP server thread is still running."""
|
|
1463
|
+
return self._thread is None or self._thread.is_alive()
|
|
1464
|
+
|
|
1465
|
+
def reset(self) -> None:
|
|
1466
|
+
"""Reset server state and cached clients between tests."""
|
|
1467
|
+
self.app.reset_clients()
|
|
1468
|
+
self.state.reset()
|
|
1469
|
+
|
|
1470
|
+
def get_async_client(
|
|
1471
|
+
self,
|
|
1472
|
+
*,
|
|
1473
|
+
token: str | None = "local-token",
|
|
1474
|
+
region: str | None = "iad1",
|
|
1475
|
+
base_url: BaseUrl | None = "http://vqs.test",
|
|
1476
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
1477
|
+
headers: Mapping[str, str] | None = None,
|
|
1478
|
+
timeout: Duration | None = 10.0,
|
|
1479
|
+
) -> QueueClient:
|
|
1480
|
+
"""Create an async client backed by the embedded app."""
|
|
1481
|
+
return self.app.get_async_client(
|
|
1482
|
+
token=token,
|
|
1483
|
+
region=region,
|
|
1484
|
+
base_url=base_url,
|
|
1485
|
+
deployment=deployment,
|
|
1486
|
+
headers=headers,
|
|
1487
|
+
timeout=timeout,
|
|
1488
|
+
)
|
|
1489
|
+
|
|
1490
|
+
@property
|
|
1491
|
+
def client(self) -> QueueClient:
|
|
1492
|
+
"""Default async client backed by the embedded app."""
|
|
1493
|
+
return self.get_async_client()
|
|
1494
|
+
|
|
1495
|
+
@property
|
|
1496
|
+
def http(self) -> BaseQueueRuntime:
|
|
1497
|
+
"""Low-level async HTTP runtime backed by the embedded server."""
|
|
1498
|
+
return self.get_async_client(base_url=self.base_url).http
|
|
1499
|
+
|
|
1500
|
+
def get_sync_client(
|
|
1501
|
+
self,
|
|
1502
|
+
*,
|
|
1503
|
+
token: str | None = "local-token",
|
|
1504
|
+
region: str | None = "iad1",
|
|
1505
|
+
base_url: BaseUrl | None = None,
|
|
1506
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
1507
|
+
headers: Mapping[str, str] | None = None,
|
|
1508
|
+
timeout: Duration | None = 10.0,
|
|
1509
|
+
) -> SyncQueueClient:
|
|
1510
|
+
"""Create a sync client backed by the running embedded server."""
|
|
1511
|
+
return self.app.get_sync_client(
|
|
1512
|
+
token=token,
|
|
1513
|
+
region=region,
|
|
1514
|
+
base_url=self.base_url if base_url is None else base_url,
|
|
1515
|
+
deployment=deployment,
|
|
1516
|
+
headers=headers,
|
|
1517
|
+
timeout=timeout,
|
|
1518
|
+
)
|
|
1519
|
+
|
|
1520
|
+
def iter_push_deliveries(
|
|
1521
|
+
self,
|
|
1522
|
+
topic: str,
|
|
1523
|
+
consumer: str,
|
|
1524
|
+
*,
|
|
1525
|
+
lease_seconds: int = DEFAULT_PUSH_LEASE_SECONDS,
|
|
1526
|
+
deployment: str | None = None,
|
|
1527
|
+
region: str = "iad1",
|
|
1528
|
+
) -> Iterator[PushDelivery]:
|
|
1529
|
+
"""Lease visible messages as synthetic push deliveries."""
|
|
1530
|
+
return self.app.iter_push_deliveries(
|
|
1531
|
+
topic,
|
|
1532
|
+
consumer,
|
|
1533
|
+
lease_seconds=lease_seconds,
|
|
1534
|
+
deployment=deployment,
|
|
1535
|
+
region=region,
|
|
1536
|
+
)
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
@dataclass(frozen=True)
|
|
1540
|
+
class EmbeddedQueueService:
|
|
1541
|
+
server: EmbeddedQueueServer
|
|
1542
|
+
dispatcher: EmbeddedQueueDispatcher
|
|
1543
|
+
token: str
|
|
1544
|
+
region: str
|
|
1545
|
+
base_url: str
|
|
1546
|
+
deployment: DeploymentOption
|
|
1547
|
+
async_http_client_factory: Callable[..., httpx.AsyncClient]
|
|
1548
|
+
|
|
1549
|
+
def get_async_client(self) -> QueueClient:
|
|
1550
|
+
return QueueClient(
|
|
1551
|
+
token=self.token,
|
|
1552
|
+
region=self.region,
|
|
1553
|
+
base_url=self.base_url,
|
|
1554
|
+
deployment=self.deployment,
|
|
1555
|
+
http_client_factory=self.async_http_client_factory,
|
|
1556
|
+
)
|
|
1557
|
+
|
|
1558
|
+
async def aclose(self) -> None:
|
|
1559
|
+
pass
|
|
1560
|
+
|
|
1561
|
+
|
|
1562
|
+
@asynccontextmanager
|
|
1563
|
+
async def embedded_queue_service(
|
|
1564
|
+
*,
|
|
1565
|
+
token: str = "local-token",
|
|
1566
|
+
region: str = "iad1",
|
|
1567
|
+
base_url: str = "http://vqs.test",
|
|
1568
|
+
deployment: DeploymentOption = ALL_DEPLOYMENTS,
|
|
1569
|
+
manual_clock: bool = False,
|
|
1570
|
+
) -> AsyncIterator[EmbeddedQueueService]:
|
|
1571
|
+
app = create_embedded_queue_app(manual_clock=manual_clock)
|
|
1572
|
+
server = app.server
|
|
1573
|
+
service: EmbeddedQueueService
|
|
1574
|
+
|
|
1575
|
+
def client_factory() -> QueueClient:
|
|
1576
|
+
return service.get_async_client()
|
|
1577
|
+
|
|
1578
|
+
dispatcher = EmbeddedQueueDispatcher(server, client_factory)
|
|
1579
|
+
service = EmbeddedQueueService(
|
|
1580
|
+
server=server,
|
|
1581
|
+
dispatcher=dispatcher,
|
|
1582
|
+
token=token,
|
|
1583
|
+
region=region,
|
|
1584
|
+
base_url=base_url,
|
|
1585
|
+
deployment=deployment,
|
|
1586
|
+
async_http_client_factory=app.create_async_client_factory(),
|
|
1587
|
+
)
|
|
1588
|
+
try:
|
|
1589
|
+
async with anyio.create_task_group() as task_group:
|
|
1590
|
+
task_group.start_soon(dispatcher.run, name="vercel-embedded-queue-dispatcher")
|
|
1591
|
+
dispatcher.wake()
|
|
1592
|
+
try:
|
|
1593
|
+
yield service
|
|
1594
|
+
finally:
|
|
1595
|
+
task_group.cancel_scope.cancel()
|
|
1596
|
+
finally:
|
|
1597
|
+
dispatcher.unregister()
|
|
1598
|
+
await dispatcher.aclose()
|
|
1599
|
+
await service.aclose()
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def create_embedded_queue_app(
|
|
1603
|
+
*,
|
|
1604
|
+
manual_clock: bool = False,
|
|
1605
|
+
) -> EmbeddedQueueAsgiApp:
|
|
1606
|
+
"""Create an embedded queue ASGI app and its backing server."""
|
|
1607
|
+
if manual_clock:
|
|
1608
|
+
state = EmbeddedQueueState(clock=ManualEmbeddedQueueClock())
|
|
1609
|
+
else:
|
|
1610
|
+
state = EmbeddedQueueState()
|
|
1611
|
+
server = EmbeddedQueueServer(state=state)
|
|
1612
|
+
return EmbeddedQueueAsgiApp(server)
|
|
1613
|
+
|
|
1614
|
+
|
|
1615
|
+
class _AsgiRequest:
|
|
1616
|
+
def __init__(self, scope: Mapping[str, Any], receive: Callable[[], Any]) -> None:
|
|
1617
|
+
self.method = str(scope["method"])
|
|
1618
|
+
self.path = str(scope["path"])
|
|
1619
|
+
self.path_parts = [unquote(part) for part in str(scope["path"]).split("/") if part]
|
|
1620
|
+
self.headers = httpx.Headers([
|
|
1621
|
+
(key.decode("latin-1"), value.decode("latin-1"))
|
|
1622
|
+
for key, value in scope.get("headers", [])
|
|
1623
|
+
])
|
|
1624
|
+
self._receive = receive
|
|
1625
|
+
self._body: bytes | None = None
|
|
1626
|
+
|
|
1627
|
+
async def body(self) -> bytes:
|
|
1628
|
+
if self._body is not None:
|
|
1629
|
+
return self._body
|
|
1630
|
+
chunks: list[bytes] = []
|
|
1631
|
+
more_body = True
|
|
1632
|
+
while more_body:
|
|
1633
|
+
event = await self._receive()
|
|
1634
|
+
chunks.append(event.get("body", b""))
|
|
1635
|
+
more_body = bool(event.get("more_body", False))
|
|
1636
|
+
self._body = b"".join(chunks)
|
|
1637
|
+
return self._body
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
@dataclass
|
|
1641
|
+
class _AsgiResponse:
|
|
1642
|
+
status_code: int
|
|
1643
|
+
body: bytes = b""
|
|
1644
|
+
headers: list[tuple[bytes, bytes]] = field(default_factory=list)
|
|
1645
|
+
|
|
1646
|
+
async def send(self, send: Callable[[MutableMapping[str, Any]], Any]) -> None:
|
|
1647
|
+
await send({
|
|
1648
|
+
"type": "http.response.start",
|
|
1649
|
+
"status": self.status_code,
|
|
1650
|
+
"headers": self.headers,
|
|
1651
|
+
})
|
|
1652
|
+
await send({"type": "http.response.body", "body": self.body})
|
|
1653
|
+
|
|
1654
|
+
|
|
1655
|
+
def _json_response(
|
|
1656
|
+
status_code: int,
|
|
1657
|
+
payload: Mapping[str, object],
|
|
1658
|
+
*,
|
|
1659
|
+
headers: list[tuple[bytes, bytes]] | None = None,
|
|
1660
|
+
) -> _AsgiResponse:
|
|
1661
|
+
return _AsgiResponse(
|
|
1662
|
+
status_code,
|
|
1663
|
+
json.dumps(payload).encode(),
|
|
1664
|
+
[*JSON_RESPONSE_HEADERS, *(headers or [])],
|
|
1665
|
+
)
|
|
1666
|
+
|
|
1667
|
+
|
|
1668
|
+
def _bad_request(message: str) -> _AsgiResponse:
|
|
1669
|
+
return _json_response(400, {"error": message})
|
|
1670
|
+
|
|
1671
|
+
|
|
1672
|
+
def _visibility_timeout_seconds_from_body(body: object) -> int:
|
|
1673
|
+
if not isinstance(body, dict):
|
|
1674
|
+
raise TypeError("JSON body must be an object")
|
|
1675
|
+
seconds_value = body.get("visibilityTimeoutSeconds")
|
|
1676
|
+
if not isinstance(seconds_value, int) or isinstance(seconds_value, bool):
|
|
1677
|
+
raise TypeError("Invalid visibilityTimeoutSeconds - must be a non-negative integer")
|
|
1678
|
+
return _int_header_value(
|
|
1679
|
+
seconds_value,
|
|
1680
|
+
DEFAULT_PULL_LEASE_SECONDS,
|
|
1681
|
+
minimum=0,
|
|
1682
|
+
maximum=MAX_VISIBILITY_TIMEOUT_SECONDS,
|
|
1683
|
+
name="visibilityTimeoutSeconds",
|
|
1684
|
+
)
|
|
1685
|
+
|
|
1686
|
+
|
|
1687
|
+
def _message_id_from_receipt_handle(receipt_handle: str) -> str | None:
|
|
1688
|
+
parts = receipt_handle.split(":", 2)
|
|
1689
|
+
if len(parts) < 3 or not parts[0].startswith("rh_"):
|
|
1690
|
+
return None
|
|
1691
|
+
return parts[1] or None
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
def _parse_queue_route(parts: list[str]) -> _QueueRoute | None:
|
|
1695
|
+
prefix_length = len(QUEUE_PATH_PREFIX)
|
|
1696
|
+
prefix_index = next(
|
|
1697
|
+
(
|
|
1698
|
+
index
|
|
1699
|
+
for index in range(len(parts) - prefix_length + 1)
|
|
1700
|
+
if tuple(parts[index : index + prefix_length]) == QUEUE_PATH_PREFIX
|
|
1701
|
+
),
|
|
1702
|
+
None,
|
|
1703
|
+
)
|
|
1704
|
+
if prefix_index is None:
|
|
1705
|
+
return None
|
|
1706
|
+
parts = parts[prefix_index:]
|
|
1707
|
+
if len(parts) <= prefix_length:
|
|
1708
|
+
return None
|
|
1709
|
+
|
|
1710
|
+
topic = parts[prefix_length]
|
|
1711
|
+
tail = parts[prefix_length + 1 :]
|
|
1712
|
+
if tail == []:
|
|
1713
|
+
return _QueueRoute("topic", topic)
|
|
1714
|
+
if len(tail) < 2 or tail[0] != "consumer":
|
|
1715
|
+
return None
|
|
1716
|
+
|
|
1717
|
+
consumer = tail[1]
|
|
1718
|
+
consumer_tail = tail[2:]
|
|
1719
|
+
if consumer_tail == []:
|
|
1720
|
+
return _QueueRoute("consumer", topic, consumer=consumer)
|
|
1721
|
+
if len(consumer_tail) == 2 and consumer_tail[0] == "id":
|
|
1722
|
+
return _QueueRoute(
|
|
1723
|
+
"message_id",
|
|
1724
|
+
topic,
|
|
1725
|
+
consumer=consumer,
|
|
1726
|
+
message_id=consumer_tail[1],
|
|
1727
|
+
)
|
|
1728
|
+
if len(consumer_tail) >= 2 and consumer_tail[0] == "lease":
|
|
1729
|
+
lease_tail = consumer_tail[1:]
|
|
1730
|
+
action: QueueRouteAction = "lease"
|
|
1731
|
+
if lease_tail[-1:] == ["visibility"]:
|
|
1732
|
+
lease_tail = lease_tail[:-1]
|
|
1733
|
+
action = "lease_visibility"
|
|
1734
|
+
if not lease_tail:
|
|
1735
|
+
return None
|
|
1736
|
+
return _QueueRoute(
|
|
1737
|
+
action,
|
|
1738
|
+
topic,
|
|
1739
|
+
consumer=consumer,
|
|
1740
|
+
receipt_handle="/".join(lease_tail),
|
|
1741
|
+
)
|
|
1742
|
+
return None
|
|
1743
|
+
|
|
1744
|
+
|
|
1745
|
+
def _validate_route(route: _QueueRoute) -> None:
|
|
1746
|
+
validate_topic_name(route.topic)
|
|
1747
|
+
if route.consumer is not None:
|
|
1748
|
+
validate_name(route.consumer, field="consumer_group")
|
|
1749
|
+
|
|
1750
|
+
|
|
1751
|
+
def _deployment_header(headers: Mapping[str, str]) -> str | None:
|
|
1752
|
+
deployment = headers.get(VQS_HEADER_DEPLOYMENT_ID)
|
|
1753
|
+
if deployment is not None:
|
|
1754
|
+
_validate_deployment_id(deployment)
|
|
1755
|
+
return deployment
|
|
1756
|
+
|
|
1757
|
+
|
|
1758
|
+
def _validate_deployment_id(deployment: str) -> None:
|
|
1759
|
+
if not deployment or DEPLOYMENT_ID_PATTERN.fullmatch(deployment) is None:
|
|
1760
|
+
raise ValueError(f"Invalid deployment id: {deployment!r}; must match {VQS_NAME_PATTERN}")
|
|
1761
|
+
|
|
1762
|
+
|
|
1763
|
+
def _receive_response_format(request: _AsgiRequest) -> ReceiveResponseFormat | _AsgiResponse:
|
|
1764
|
+
accept = request.headers.get(HEADER_ACCEPT)
|
|
1765
|
+
if accept is None:
|
|
1766
|
+
return _bad_request(
|
|
1767
|
+
"Accept header required. Supported: multipart/mixed, application/x-ndjson"
|
|
1768
|
+
)
|
|
1769
|
+
|
|
1770
|
+
accepted_types = {_media_type(value) for value in accept.split(",")}
|
|
1771
|
+
if "*/*" in accepted_types:
|
|
1772
|
+
return _bad_request(
|
|
1773
|
+
"Accept header required. Supported: multipart/mixed, application/x-ndjson"
|
|
1774
|
+
)
|
|
1775
|
+
if CONTENT_TYPE_MULTIPART_MIXED in accepted_types:
|
|
1776
|
+
return "multipart"
|
|
1777
|
+
if CONTENT_TYPE_NDJSON in accepted_types:
|
|
1778
|
+
return "ndjson"
|
|
1779
|
+
return _bad_request("Unsupported Accept header. Use multipart/mixed or application/x-ndjson")
|
|
1780
|
+
|
|
1781
|
+
|
|
1782
|
+
def _media_type(value: str) -> str:
|
|
1783
|
+
return value.split(";", 1)[0].strip().lower()
|
|
1784
|
+
|
|
1785
|
+
|
|
1786
|
+
def async_client_factory(app: EmbeddedQueueAsgiApp) -> Any:
|
|
1787
|
+
return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://vqs.test")
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def _receive_response(
|
|
1791
|
+
response_format: ReceiveResponseFormat,
|
|
1792
|
+
messages: list[_LeasedMessage],
|
|
1793
|
+
) -> _AsgiResponse:
|
|
1794
|
+
if response_format == "ndjson":
|
|
1795
|
+
return _ndjson_response(messages)
|
|
1796
|
+
return _multipart_response([
|
|
1797
|
+
_multipart_part(message.message, message.receipt_handle, message.consumer)
|
|
1798
|
+
for message in messages
|
|
1799
|
+
])
|
|
1800
|
+
|
|
1801
|
+
|
|
1802
|
+
def _ndjson_response(messages: list[_LeasedMessage]) -> _AsgiResponse:
|
|
1803
|
+
lines = [
|
|
1804
|
+
json.dumps(
|
|
1805
|
+
{
|
|
1806
|
+
"messageId": leased.message.message_id,
|
|
1807
|
+
"receiptHandle": leased.receipt_handle,
|
|
1808
|
+
"deliveryCount": leased.message.delivery_count_by_consumer[leased.consumer],
|
|
1809
|
+
"timestamp": _format_dt(leased.message.created_at),
|
|
1810
|
+
"expiresAt": _format_dt(leased.message.expires_at),
|
|
1811
|
+
"contentType": leased.message.content_type,
|
|
1812
|
+
"body": base64.b64encode(leased.message.payload.read()).decode("ascii"),
|
|
1813
|
+
},
|
|
1814
|
+
separators=(",", ":"),
|
|
1815
|
+
).encode()
|
|
1816
|
+
for leased in messages
|
|
1817
|
+
]
|
|
1818
|
+
return _AsgiResponse(
|
|
1819
|
+
200,
|
|
1820
|
+
b"\n".join(lines) + b"\n",
|
|
1821
|
+
[(b"content-type", CONTENT_TYPE_NDJSON.encode())],
|
|
1822
|
+
)
|
|
1823
|
+
|
|
1824
|
+
|
|
1825
|
+
def _multipart_response(parts: list[bytes]) -> _AsgiResponse:
|
|
1826
|
+
body = b"".join(parts) + f"--{BOUNDARY}--\r\n".encode()
|
|
1827
|
+
return _AsgiResponse(
|
|
1828
|
+
200,
|
|
1829
|
+
body,
|
|
1830
|
+
[(b"content-type", f"{CONTENT_TYPE_MULTIPART_MIXED}; boundary={BOUNDARY}".encode())],
|
|
1831
|
+
)
|
|
1832
|
+
|
|
1833
|
+
|
|
1834
|
+
def _multipart_part(message: StoredMessage, receipt_handle: str, consumer: str) -> bytes:
|
|
1835
|
+
headers = [
|
|
1836
|
+
f"--{BOUNDARY}".encode(),
|
|
1837
|
+
f"{HEADER_CONTENT_TYPE}: {message.content_type}".encode(),
|
|
1838
|
+
f"{VQS_HEADER_MESSAGE_ID}: {message.message_id}".encode(),
|
|
1839
|
+
f"{VQS_HEADER_RECEIPT_HANDLE}: {receipt_handle}".encode(),
|
|
1840
|
+
f"{VQS_HEADER_DELIVERY_COUNT}: {message.delivery_count_by_consumer[consumer]}".encode(),
|
|
1841
|
+
f"{VQS_HEADER_TIMESTAMP}: {_format_dt(message.created_at)}".encode(),
|
|
1842
|
+
f"{VQS_HEADER_EXPIRES_AT}: {_format_dt(message.expires_at)}".encode(),
|
|
1843
|
+
]
|
|
1844
|
+
return b"\r\n".join([*headers, b"", message.payload.read(), b""])
|
|
1845
|
+
|
|
1846
|
+
|
|
1847
|
+
def _format_dt(value: datetime) -> str:
|
|
1848
|
+
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
1849
|
+
|
|
1850
|
+
|
|
1851
|
+
def _min_datetime(current: datetime | None, candidate: datetime) -> datetime:
|
|
1852
|
+
if current is None or candidate < current:
|
|
1853
|
+
return candidate
|
|
1854
|
+
return current
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
def _int_header(
|
|
1858
|
+
value: str | None,
|
|
1859
|
+
default: int,
|
|
1860
|
+
*,
|
|
1861
|
+
minimum: int,
|
|
1862
|
+
maximum: int,
|
|
1863
|
+
name: str,
|
|
1864
|
+
) -> int:
|
|
1865
|
+
return _int_header_value(
|
|
1866
|
+
value,
|
|
1867
|
+
default,
|
|
1868
|
+
minimum=minimum,
|
|
1869
|
+
maximum=maximum,
|
|
1870
|
+
name=name,
|
|
1871
|
+
)
|
|
1872
|
+
|
|
1873
|
+
|
|
1874
|
+
def _int_header_value(
|
|
1875
|
+
value: object,
|
|
1876
|
+
default: int,
|
|
1877
|
+
*,
|
|
1878
|
+
minimum: int,
|
|
1879
|
+
maximum: int,
|
|
1880
|
+
name: str,
|
|
1881
|
+
) -> int:
|
|
1882
|
+
if value is None:
|
|
1883
|
+
return default
|
|
1884
|
+
if isinstance(value, bool):
|
|
1885
|
+
raise TypeError(f"{name} must be an integer")
|
|
1886
|
+
if isinstance(value, int):
|
|
1887
|
+
parsed = value
|
|
1888
|
+
elif isinstance(value, str):
|
|
1889
|
+
try:
|
|
1890
|
+
numeric = float(value)
|
|
1891
|
+
except ValueError as exc:
|
|
1892
|
+
raise ValueError(f"{name} must be an integer") from exc
|
|
1893
|
+
if not math.isfinite(numeric) or not numeric.is_integer():
|
|
1894
|
+
raise ValueError(f"{name} must be an integer")
|
|
1895
|
+
parsed = int(numeric)
|
|
1896
|
+
else:
|
|
1897
|
+
raise TypeError(f"{name} must be an integer")
|
|
1898
|
+
if parsed < minimum or parsed > maximum:
|
|
1899
|
+
raise ValueError(f"{name} must be between {minimum} and {maximum}")
|
|
1900
|
+
return parsed
|