aioartemis 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aioartemis/__init__.py +32 -0
- aioartemis/_engine.py +640 -0
- aioartemis/_policy.py +46 -0
- aioartemis/client.py +561 -0
- aioartemis/errors.py +76 -0
- aioartemis/py.typed +0 -0
- aioartemis-0.1.0.dist-info/METADATA +99 -0
- aioartemis-0.1.0.dist-info/RECORD +11 -0
- aioartemis-0.1.0.dist-info/WHEEL +5 -0
- aioartemis-0.1.0.dist-info/licenses/LICENSE +202 -0
- aioartemis-0.1.0.dist-info/top_level.txt +1 -0
aioartemis/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""asyncio AMQP 1.0 client for ActiveMQ Artemis with failover across nodes."""
|
|
2
|
+
|
|
3
|
+
from .client import Client, IncomingMessage, Subscription
|
|
4
|
+
from .errors import (
|
|
5
|
+
ArtemisError,
|
|
6
|
+
AuthenticationFailed,
|
|
7
|
+
ClientClosed,
|
|
8
|
+
ConnectionLost,
|
|
9
|
+
DeliveryLost,
|
|
10
|
+
EngineFailed,
|
|
11
|
+
LinkRefused,
|
|
12
|
+
SendRejected,
|
|
13
|
+
SendReleased,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ArtemisError",
|
|
20
|
+
"AuthenticationFailed",
|
|
21
|
+
"Client",
|
|
22
|
+
"ClientClosed",
|
|
23
|
+
"ConnectionLost",
|
|
24
|
+
"DeliveryLost",
|
|
25
|
+
"EngineFailed",
|
|
26
|
+
"IncomingMessage",
|
|
27
|
+
"LinkRefused",
|
|
28
|
+
"SendRejected",
|
|
29
|
+
"SendReleased",
|
|
30
|
+
"Subscription",
|
|
31
|
+
"__version__",
|
|
32
|
+
]
|
aioartemis/_engine.py
ADDED
|
@@ -0,0 +1,640 @@
|
|
|
1
|
+
"""Proton side of the client: everything here runs on the container's own thread.
|
|
2
|
+
|
|
3
|
+
No proton object leaves this thread: the asyncio side only ever holds plain
|
|
4
|
+
data and integer ids, because proton-c is not thread-safe and even dropping
|
|
5
|
+
the last reference to a wrapper (``pn_decref``) touches its state.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import itertools
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import queue
|
|
14
|
+
import random
|
|
15
|
+
import threading
|
|
16
|
+
import uuid
|
|
17
|
+
from collections import deque
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from contextlib import suppress
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from typing import Any, Literal, Protocol
|
|
22
|
+
|
|
23
|
+
from proton import Condition, Connection, Data, Delivery, Event, Handler, Link, Message, symbol
|
|
24
|
+
from proton._reactor import Task
|
|
25
|
+
from proton.handlers import MessagingHandler
|
|
26
|
+
from proton.reactor import ApplicationEvent, Container, EventInjector, LinkOption
|
|
27
|
+
|
|
28
|
+
from ._policy import Attempts, after_failed_attempt, after_lost_connection, backoff_delay, credit_to_grant
|
|
29
|
+
from .errors import (
|
|
30
|
+
AuthenticationFailed,
|
|
31
|
+
ClientClosed,
|
|
32
|
+
ConnectionLost,
|
|
33
|
+
DeliveryLost,
|
|
34
|
+
LinkRefused,
|
|
35
|
+
SendRejected,
|
|
36
|
+
SendReleased,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
log = logging.getLogger("aioartemis")
|
|
40
|
+
|
|
41
|
+
Routing = Literal["queue", "topic"]
|
|
42
|
+
Settlement = Literal["accept", "reject", "release", "nack"]
|
|
43
|
+
|
|
44
|
+
_FATAL_CONDITIONS = frozenset({"amqp:unauthorized-access"})
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Outcome(Protocol):
|
|
48
|
+
"""Settles one asyncio future from the proton thread."""
|
|
49
|
+
|
|
50
|
+
def ok(self, value: Any = None) -> None: ...
|
|
51
|
+
def fail(self, exc: BaseException) -> None: ...
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class Received:
|
|
56
|
+
body: Any
|
|
57
|
+
properties: dict[str, Any]
|
|
58
|
+
message_id: Any
|
|
59
|
+
delivery_count: int
|
|
60
|
+
subscription_id: int
|
|
61
|
+
delivery_id: int
|
|
62
|
+
epoch: int
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class EngineConfig:
|
|
67
|
+
urls: list[str]
|
|
68
|
+
user: str | None
|
|
69
|
+
password: str | None
|
|
70
|
+
heartbeat: float | None
|
|
71
|
+
open_timeout: float
|
|
72
|
+
close_timeout: float
|
|
73
|
+
reconnect_base: float
|
|
74
|
+
reconnect_max: float
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Listener(Protocol):
|
|
78
|
+
"""How the engine reports to the asyncio side; called on the proton thread."""
|
|
79
|
+
|
|
80
|
+
def connected(self, url: str, epoch: int) -> None: ...
|
|
81
|
+
def disconnected(self) -> None: ...
|
|
82
|
+
def fatal(self, exc: BaseException) -> None: ...
|
|
83
|
+
def error(self, reason: str) -> None: ...
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class PendingSend:
|
|
88
|
+
encoded: bytes
|
|
89
|
+
message_id: Any
|
|
90
|
+
outcome: Outcome
|
|
91
|
+
# Written by the asyncio thread on cancellation; a plain attribute store is atomic.
|
|
92
|
+
abandoned: bool = False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass
|
|
96
|
+
class _Sender:
|
|
97
|
+
address: str
|
|
98
|
+
routing: Routing
|
|
99
|
+
link: Link | None = None
|
|
100
|
+
pending: deque[PendingSend] = field(default_factory=deque)
|
|
101
|
+
in_flight: dict[str, PendingSend] = field(default_factory=dict)
|
|
102
|
+
# True once the broker has granted credit on the current link; credit seen
|
|
103
|
+
# before that may be stale, and a transfer sent on it is lost unsettled.
|
|
104
|
+
sendable: bool = False
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass
|
|
108
|
+
class _Subscription:
|
|
109
|
+
id: int
|
|
110
|
+
address: str
|
|
111
|
+
routing: Routing
|
|
112
|
+
prefetch: int
|
|
113
|
+
deliver: Callable[[Received], None]
|
|
114
|
+
fail: Callable[[BaseException], None]
|
|
115
|
+
opened: Outcome | None
|
|
116
|
+
link: Link | None = None
|
|
117
|
+
deliveries: dict[int, Delivery] = field(default_factory=dict)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _Routing(LinkOption):
|
|
121
|
+
"""Without an explicit capability Artemis auto-creates a MULTICAST address,
|
|
122
|
+
turning a "queue" into a topic: messages sent before a subscriber exist are
|
|
123
|
+
dropped and every subscriber gets every message."""
|
|
124
|
+
|
|
125
|
+
def __init__(self, routing: Routing) -> None:
|
|
126
|
+
self._routing = routing
|
|
127
|
+
|
|
128
|
+
def apply(self, link: Link) -> None:
|
|
129
|
+
terminus = link.target if link.is_sender else link.source
|
|
130
|
+
terminus.capabilities.put_array(False, Data.SYMBOL)
|
|
131
|
+
terminus.capabilities.enter()
|
|
132
|
+
terminus.capabilities.put_symbol(symbol(self._routing))
|
|
133
|
+
terminus.capabilities.exit()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class _Timer(Handler):
|
|
137
|
+
def __init__(self, callback: Callable[[], None]) -> None:
|
|
138
|
+
self._callback = callback
|
|
139
|
+
|
|
140
|
+
def on_timer_task(self, event: Event) -> None:
|
|
141
|
+
self._callback()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
_RUN_COMMANDS = "run_commands"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class Engine(MessagingHandler):
|
|
148
|
+
def __init__(self, config: EngineConfig, listener: Listener, *, shuffle: bool) -> None:
|
|
149
|
+
super().__init__(prefetch=0, auto_accept=False)
|
|
150
|
+
self._config = config
|
|
151
|
+
self._listener = listener
|
|
152
|
+
self._urls = list(config.urls)
|
|
153
|
+
if shuffle:
|
|
154
|
+
random.shuffle(self._urls)
|
|
155
|
+
self._injector = EventInjector()
|
|
156
|
+
self._injector_lock = threading.Lock()
|
|
157
|
+
self._injector_released = False
|
|
158
|
+
self._commands: queue.SimpleQueue[tuple[Callable[[], None], Outcome | None]] = queue.SimpleQueue()
|
|
159
|
+
self._container: Container | None = None
|
|
160
|
+
|
|
161
|
+
self._connection: Connection | None = None
|
|
162
|
+
self._connected_url: str | None = None
|
|
163
|
+
self._attempts = Attempts(tuple(self._urls))
|
|
164
|
+
self._stable = False
|
|
165
|
+
self._open_timer: Task | None = None
|
|
166
|
+
self._stable_timer: Task | None = None
|
|
167
|
+
self._retry_timer: Task | None = None
|
|
168
|
+
self._wake_pending = False
|
|
169
|
+
self._online = False
|
|
170
|
+
self._epoch = 0
|
|
171
|
+
self._closing = False
|
|
172
|
+
self._fatal: BaseException | None = None
|
|
173
|
+
|
|
174
|
+
self._senders: dict[tuple[str, Routing], _Sender] = {}
|
|
175
|
+
self._subscriptions: dict[int, _Subscription] = {}
|
|
176
|
+
self._by_link: dict[Link, _Sender | _Subscription] = {}
|
|
177
|
+
self._delivery_ids = itertools.count()
|
|
178
|
+
|
|
179
|
+
# -- any thread ---------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
def submit(self, command: Callable[[], None], outcome: Outcome | None = None) -> None:
|
|
182
|
+
# The lock keeps a late submit from writing into a pipe fd that
|
|
183
|
+
# release_fds has closed and the OS may have handed to another file.
|
|
184
|
+
with self._injector_lock:
|
|
185
|
+
if self._injector_released:
|
|
186
|
+
raise ClientClosed("client closed")
|
|
187
|
+
self._commands.put((command, outcome))
|
|
188
|
+
# The pipe is blocking and this runs on the event loop: write one
|
|
189
|
+
# wake-up per batch, not one per command, or a burst of sends
|
|
190
|
+
# stalls the loop whenever the proton thread is busy.
|
|
191
|
+
if not self._wake_pending:
|
|
192
|
+
self._wake_pending = True
|
|
193
|
+
self._injector.trigger(ApplicationEvent(_RUN_COMMANDS))
|
|
194
|
+
|
|
195
|
+
def release_fds(self) -> None:
|
|
196
|
+
"""EventInjector.close() leaves its pipe open; call once the container has exited."""
|
|
197
|
+
with self._injector_lock:
|
|
198
|
+
self._injector_released = True
|
|
199
|
+
for fd in self._injector.pipe:
|
|
200
|
+
os.close(fd)
|
|
201
|
+
|
|
202
|
+
# -- commands -----------------------------------------------------------------
|
|
203
|
+
|
|
204
|
+
def send(self, address: str, routing: Routing, item: PendingSend) -> None:
|
|
205
|
+
if self._refuse_when_closing(item.outcome):
|
|
206
|
+
return
|
|
207
|
+
sender = self._senders.get((address, routing))
|
|
208
|
+
if sender is None:
|
|
209
|
+
sender = self._senders[(address, routing)] = _Sender(address, routing)
|
|
210
|
+
if self._online:
|
|
211
|
+
self._attach_sender(sender)
|
|
212
|
+
sender.pending.append(item)
|
|
213
|
+
self._flush(sender)
|
|
214
|
+
|
|
215
|
+
def forget(self, address: str, routing: Routing, item: PendingSend) -> None:
|
|
216
|
+
sender = self._senders.get((address, routing))
|
|
217
|
+
if sender is not None and item in sender.pending:
|
|
218
|
+
sender.pending.remove(item)
|
|
219
|
+
|
|
220
|
+
def subscribe(self, subscription: _Subscription) -> None:
|
|
221
|
+
if subscription.opened is not None and self._refuse_when_closing(subscription.opened):
|
|
222
|
+
return
|
|
223
|
+
self._subscriptions[subscription.id] = subscription
|
|
224
|
+
if self._online:
|
|
225
|
+
self._attach_subscription(subscription)
|
|
226
|
+
|
|
227
|
+
def unsubscribe(self, subscription_id: int) -> None:
|
|
228
|
+
subscription = self._subscriptions.pop(subscription_id, None)
|
|
229
|
+
if subscription is None:
|
|
230
|
+
return
|
|
231
|
+
if subscription.opened is not None:
|
|
232
|
+
subscription.opened.fail(ClientClosed("unsubscribed before the link opened"))
|
|
233
|
+
if subscription.link is not None:
|
|
234
|
+
self._by_link.pop(subscription.link, None)
|
|
235
|
+
subscription.link.close()
|
|
236
|
+
|
|
237
|
+
def settle_received(
|
|
238
|
+
self, subscription_id: int, delivery_id: int, epoch: int, how: Settlement, outcome: Outcome
|
|
239
|
+
) -> None:
|
|
240
|
+
subscription = self._subscriptions.get(subscription_id)
|
|
241
|
+
if subscription is None or epoch != self._epoch or not self._online:
|
|
242
|
+
raise DeliveryLost("the connection or subscription this message came over is gone")
|
|
243
|
+
delivery = subscription.deliveries.pop(delivery_id, None)
|
|
244
|
+
if delivery is None:
|
|
245
|
+
raise DeliveryLost("the message was already settled")
|
|
246
|
+
if how == "accept":
|
|
247
|
+
delivery.update(Delivery.ACCEPTED)
|
|
248
|
+
elif how == "reject":
|
|
249
|
+
delivery.update(Delivery.REJECTED)
|
|
250
|
+
elif how == "release":
|
|
251
|
+
delivery.update(Delivery.RELEASED)
|
|
252
|
+
else:
|
|
253
|
+
# MODIFIED with delivery-failed: unlike RELEASED, Artemis counts it as a
|
|
254
|
+
# delivery attempt, so max-delivery-attempts and the DLQ still apply.
|
|
255
|
+
delivery.local.failed = True
|
|
256
|
+
delivery.update(Delivery.MODIFIED)
|
|
257
|
+
delivery.settle()
|
|
258
|
+
self._top_up(subscription)
|
|
259
|
+
outcome.ok()
|
|
260
|
+
|
|
261
|
+
def shutdown(self) -> None:
|
|
262
|
+
if self._closing:
|
|
263
|
+
return
|
|
264
|
+
self._closing = True
|
|
265
|
+
self._fail_everything(ClientClosed("client closed"))
|
|
266
|
+
connection = self._connection
|
|
267
|
+
if self._online and connection is not None:
|
|
268
|
+
connection.close()
|
|
269
|
+
self._schedule(self._config.close_timeout, self._stop)
|
|
270
|
+
else:
|
|
271
|
+
self._stop()
|
|
272
|
+
|
|
273
|
+
# -- proton callbacks ---------------------------------------------------------
|
|
274
|
+
|
|
275
|
+
def on_start(self, event: Event) -> None:
|
|
276
|
+
self._container = event.container
|
|
277
|
+
event.container.selectable(self._injector)
|
|
278
|
+
self._connect()
|
|
279
|
+
|
|
280
|
+
def on_run_commands(self, event: Event) -> None:
|
|
281
|
+
with self._injector_lock:
|
|
282
|
+
self._wake_pending = False
|
|
283
|
+
while True:
|
|
284
|
+
try:
|
|
285
|
+
command, outcome = self._commands.get_nowait()
|
|
286
|
+
except queue.Empty:
|
|
287
|
+
return
|
|
288
|
+
try:
|
|
289
|
+
command()
|
|
290
|
+
except Exception as exc:
|
|
291
|
+
if outcome is None:
|
|
292
|
+
raise
|
|
293
|
+
outcome.fail(_detached(exc))
|
|
294
|
+
|
|
295
|
+
def on_connection_opened(self, event: Event) -> None:
|
|
296
|
+
if not _same(event.connection, self._connection):
|
|
297
|
+
return
|
|
298
|
+
self._cancel(self._open_timer)
|
|
299
|
+
self._online = True
|
|
300
|
+
self._epoch += 1
|
|
301
|
+
self._connected_url = self._attempts.url
|
|
302
|
+
self._stable = False
|
|
303
|
+
self._stable_timer = self._schedule(self._config.reconnect_max, self._mark_stable)
|
|
304
|
+
for sender in self._senders.values():
|
|
305
|
+
self._attach_sender(sender)
|
|
306
|
+
for subscription in self._subscriptions.values():
|
|
307
|
+
self._attach_subscription(subscription)
|
|
308
|
+
self._listener.connected(self._connected_url, self._epoch)
|
|
309
|
+
|
|
310
|
+
def on_disconnected(self, event: Event) -> None:
|
|
311
|
+
self._lost(event.connection, "transport closed")
|
|
312
|
+
|
|
313
|
+
def on_connection_error(self, event: Event) -> None:
|
|
314
|
+
self._lost(event.connection, f"closed by broker: {_describe(_remote_condition(event.connection))}")
|
|
315
|
+
|
|
316
|
+
def on_connection_closing(self, event: Event) -> None:
|
|
317
|
+
self._lost(event.connection, "closed by broker")
|
|
318
|
+
|
|
319
|
+
def on_connection_closed(self, event: Event) -> None:
|
|
320
|
+
if self._closing:
|
|
321
|
+
self._stop()
|
|
322
|
+
|
|
323
|
+
def on_session_error(self, event: Event) -> None:
|
|
324
|
+
self._lost(event.connection, "session closed by broker")
|
|
325
|
+
|
|
326
|
+
def on_session_closing(self, event: Event) -> None:
|
|
327
|
+
self._lost(event.connection, "session closed by broker")
|
|
328
|
+
|
|
329
|
+
def on_transport_error(self, event: Event) -> None:
|
|
330
|
+
# Proton usually reports the transport closed before its error, so the
|
|
331
|
+
# condition is read in _lost; overriding also stops MessagingHandler
|
|
332
|
+
# from closing the connection itself on an authentication failure.
|
|
333
|
+
pass
|
|
334
|
+
|
|
335
|
+
def on_link_opened(self, event: Event) -> None:
|
|
336
|
+
link = event.link
|
|
337
|
+
owner = self._by_link.get(link) if link is not None else None
|
|
338
|
+
if not isinstance(owner, _Subscription) or link is None:
|
|
339
|
+
return
|
|
340
|
+
if link.remote_source.address is None:
|
|
341
|
+
return # refused: Artemis follows up with a detach carrying the reason
|
|
342
|
+
if owner.opened is not None:
|
|
343
|
+
owner.opened.ok(owner.id)
|
|
344
|
+
owner.opened = None
|
|
345
|
+
self._top_up(owner)
|
|
346
|
+
|
|
347
|
+
def on_link_error(self, event: Event) -> None:
|
|
348
|
+
self._link_gone(event.link, LinkRefused(_describe(_remote_condition(event.link))))
|
|
349
|
+
|
|
350
|
+
def on_link_closing(self, event: Event) -> None:
|
|
351
|
+
self._link_gone(event.link, LinkRefused("link detached by broker"))
|
|
352
|
+
|
|
353
|
+
def on_sendable(self, event: Event) -> None:
|
|
354
|
+
owner = self._by_link.get(event.link) if event.link is not None else None
|
|
355
|
+
if isinstance(owner, _Sender):
|
|
356
|
+
owner.sendable = True
|
|
357
|
+
self._flush(owner)
|
|
358
|
+
|
|
359
|
+
def on_accepted(self, event: Event) -> None:
|
|
360
|
+
self._resolve(event.delivery, None)
|
|
361
|
+
|
|
362
|
+
def on_rejected(self, event: Event) -> None:
|
|
363
|
+
delivery = event.delivery
|
|
364
|
+
# Only the rejected variant of RemoteDisposition carries a condition.
|
|
365
|
+
condition = getattr(delivery.remote, "condition", None) if delivery is not None else None
|
|
366
|
+
self._resolve(delivery, SendRejected(_describe(condition) if condition else "rejected"))
|
|
367
|
+
|
|
368
|
+
def on_released(self, event: Event) -> None:
|
|
369
|
+
self._resolve(event.delivery, SendReleased("broker released the message"))
|
|
370
|
+
|
|
371
|
+
def on_message(self, event: Event) -> None:
|
|
372
|
+
owner = self._by_link.get(event.link) if event.link is not None else None
|
|
373
|
+
delivery = event.delivery
|
|
374
|
+
if not isinstance(owner, _Subscription) or delivery is None:
|
|
375
|
+
return
|
|
376
|
+
delivery_id = next(self._delivery_ids)
|
|
377
|
+
owner.deliveries[delivery_id] = delivery
|
|
378
|
+
message: Message = event.message
|
|
379
|
+
owner.deliver(
|
|
380
|
+
Received(
|
|
381
|
+
body=message.body,
|
|
382
|
+
properties=dict(message.properties or {}),
|
|
383
|
+
message_id=message.id,
|
|
384
|
+
delivery_count=message.delivery_count,
|
|
385
|
+
subscription_id=owner.id,
|
|
386
|
+
delivery_id=delivery_id,
|
|
387
|
+
epoch=self._epoch,
|
|
388
|
+
)
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# -- connection lifecycle -----------------------------------------------------
|
|
392
|
+
|
|
393
|
+
def _connect(self) -> None:
|
|
394
|
+
assert self._container is not None
|
|
395
|
+
url = self._attempts.url
|
|
396
|
+
connection = self._container.connect(
|
|
397
|
+
url=url,
|
|
398
|
+
reconnect=False,
|
|
399
|
+
user=self._config.user,
|
|
400
|
+
password=self._config.password,
|
|
401
|
+
heartbeat=self._config.heartbeat,
|
|
402
|
+
)
|
|
403
|
+
self._connection = connection
|
|
404
|
+
self._open_timer = self._schedule(
|
|
405
|
+
self._config.open_timeout, lambda: self._lost(connection, "open timed out")
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
def _lost(self, connection: Connection | None, reason: str) -> None:
|
|
409
|
+
"""Every way of losing a connection ends up here; events for connections
|
|
410
|
+
we have already given up on are ignored."""
|
|
411
|
+
if connection is None or not _same(connection, self._connection):
|
|
412
|
+
return
|
|
413
|
+
self._connection = None
|
|
414
|
+
self._cancel(self._open_timer)
|
|
415
|
+
self._cancel(self._stable_timer)
|
|
416
|
+
condition = connection.transport.condition if connection.transport is not None else None
|
|
417
|
+
_abort(connection)
|
|
418
|
+
was_online, self._online = self._online, False
|
|
419
|
+
self._drop_links(reason)
|
|
420
|
+
if was_online:
|
|
421
|
+
self._listener.disconnected()
|
|
422
|
+
if condition is not None:
|
|
423
|
+
reason = f"{reason}: {_describe(condition)}"
|
|
424
|
+
if condition is not None and condition.name in _FATAL_CONDITIONS:
|
|
425
|
+
if self._epoch == 0:
|
|
426
|
+
# Wrong credentials before we ever got in: retrying cannot help.
|
|
427
|
+
self._fatal = AuthenticationFailed(_describe(condition))
|
|
428
|
+
self._fail_everything(self._fatal)
|
|
429
|
+
self._listener.fatal(self._fatal)
|
|
430
|
+
else:
|
|
431
|
+
# Once we have logged in, a node refusing us is a misconfigured node,
|
|
432
|
+
# not broken credentials, and is failed over like any other.
|
|
433
|
+
log.warning("%s refused the credentials: %s", self._attempts.url, _describe(condition))
|
|
434
|
+
if self._closing or self._fatal is not None:
|
|
435
|
+
self._stop()
|
|
436
|
+
return
|
|
437
|
+
self._listener.error(f"{self._attempts.url}: {reason}")
|
|
438
|
+
log.info("connection to %s lost: %s", self._attempts.url, reason)
|
|
439
|
+
self._schedule_reconnect(was_online)
|
|
440
|
+
|
|
441
|
+
def _schedule_reconnect(self, was_online: bool) -> None:
|
|
442
|
+
if was_online and self._connected_url is not None:
|
|
443
|
+
self._attempts = after_lost_connection(
|
|
444
|
+
self._urls, self._connected_url, self._attempts, stable=self._stable
|
|
445
|
+
)
|
|
446
|
+
else:
|
|
447
|
+
self._attempts = after_failed_attempt(self._attempts)
|
|
448
|
+
delay = backoff_delay(
|
|
449
|
+
self._attempts.round,
|
|
450
|
+
base=self._config.reconnect_base,
|
|
451
|
+
cap=self._config.reconnect_max,
|
|
452
|
+
rand=random.random,
|
|
453
|
+
)
|
|
454
|
+
self._retry_timer = self._schedule(delay, self._connect)
|
|
455
|
+
|
|
456
|
+
def _mark_stable(self) -> None:
|
|
457
|
+
self._stable = True
|
|
458
|
+
|
|
459
|
+
def _drop_links(self, reason: str) -> None:
|
|
460
|
+
for sender in self._senders.values():
|
|
461
|
+
sender.link = None
|
|
462
|
+
for item in sender.in_flight.values():
|
|
463
|
+
item.outcome.fail(ConnectionLost(f"outcome unknown: {reason}", item.message_id))
|
|
464
|
+
sender.in_flight.clear()
|
|
465
|
+
for subscription in self._subscriptions.values():
|
|
466
|
+
subscription.link = None
|
|
467
|
+
subscription.deliveries.clear()
|
|
468
|
+
self._by_link.clear()
|
|
469
|
+
|
|
470
|
+
def _stop(self) -> None:
|
|
471
|
+
self._cancel(self._open_timer)
|
|
472
|
+
self._cancel(self._stable_timer)
|
|
473
|
+
self._cancel(self._retry_timer)
|
|
474
|
+
if self._connection is not None:
|
|
475
|
+
_abort(self._connection)
|
|
476
|
+
self._connection = None
|
|
477
|
+
self._injector.close()
|
|
478
|
+
assert self._container is not None
|
|
479
|
+
self._container.stop()
|
|
480
|
+
|
|
481
|
+
def crash_cleanup(self, container: Container) -> None:
|
|
482
|
+
"""After an exception escaped the event loop: close the socket instead of
|
|
483
|
+
leaving the broker holding the connection and its unacked messages.
|
|
484
|
+
|
|
485
|
+
The engine is unhooked first so the bug that crashed it cannot fire
|
|
486
|
+
again; the remaining events then only reach proton's IO handler, which
|
|
487
|
+
closes the sockets."""
|
|
488
|
+
container.handler = Handler()
|
|
489
|
+
with suppress(Exception):
|
|
490
|
+
if self._connection is not None:
|
|
491
|
+
_abort(self._connection)
|
|
492
|
+
self._injector.close()
|
|
493
|
+
container.stop()
|
|
494
|
+
while container.process():
|
|
495
|
+
pass
|
|
496
|
+
|
|
497
|
+
# -- links --------------------------------------------------------------------
|
|
498
|
+
|
|
499
|
+
def _attach_sender(self, sender: _Sender) -> None:
|
|
500
|
+
assert self._container is not None
|
|
501
|
+
assert self._connection is not None
|
|
502
|
+
link = self._container.create_sender(
|
|
503
|
+
self._connection,
|
|
504
|
+
sender.address,
|
|
505
|
+
name=_link_name(sender.address),
|
|
506
|
+
options=_Routing(sender.routing),
|
|
507
|
+
)
|
|
508
|
+
sender.link, sender.sendable = link, False
|
|
509
|
+
self._by_link[link] = sender
|
|
510
|
+
|
|
511
|
+
def _attach_subscription(self, subscription: _Subscription) -> None:
|
|
512
|
+
assert self._container is not None
|
|
513
|
+
assert self._connection is not None
|
|
514
|
+
link = self._container.create_receiver(
|
|
515
|
+
self._connection,
|
|
516
|
+
subscription.address,
|
|
517
|
+
name=_link_name(subscription.address),
|
|
518
|
+
options=_Routing(subscription.routing),
|
|
519
|
+
)
|
|
520
|
+
subscription.link = link
|
|
521
|
+
self._by_link[link] = subscription
|
|
522
|
+
|
|
523
|
+
def _link_gone(self, link: Link | None, error: LinkRefused) -> None:
|
|
524
|
+
owner = self._by_link.pop(link, None) if link is not None else None
|
|
525
|
+
if isinstance(owner, _Sender):
|
|
526
|
+
self._senders.pop((owner.address, owner.routing), None)
|
|
527
|
+
for item in owner.pending:
|
|
528
|
+
item.outcome.fail(error)
|
|
529
|
+
for item in owner.in_flight.values():
|
|
530
|
+
item.outcome.fail(ConnectionLost(f"outcome unknown: {error}", item.message_id))
|
|
531
|
+
elif isinstance(owner, _Subscription):
|
|
532
|
+
self._subscriptions.pop(owner.id, None)
|
|
533
|
+
if owner.opened is not None:
|
|
534
|
+
owner.opened.fail(error)
|
|
535
|
+
else:
|
|
536
|
+
owner.fail(error)
|
|
537
|
+
|
|
538
|
+
def _flush(self, sender: _Sender) -> None:
|
|
539
|
+
link = sender.link
|
|
540
|
+
while link is not None and sender.sendable and sender.pending and link.credit > 0:
|
|
541
|
+
item = sender.pending.popleft()
|
|
542
|
+
if item.abandoned:
|
|
543
|
+
continue
|
|
544
|
+
delivery = link.delivery(link.delivery_tag())
|
|
545
|
+
link.stream(item.encoded)
|
|
546
|
+
link.advance()
|
|
547
|
+
sender.in_flight[delivery.tag] = item
|
|
548
|
+
|
|
549
|
+
def _resolve(self, delivery: Delivery | None, error: Exception | None) -> None:
|
|
550
|
+
if delivery is None:
|
|
551
|
+
return
|
|
552
|
+
owner = self._by_link.get(delivery.link)
|
|
553
|
+
item = owner.in_flight.pop(delivery.tag, None) if isinstance(owner, _Sender) else None
|
|
554
|
+
if item is None:
|
|
555
|
+
return
|
|
556
|
+
if error is None:
|
|
557
|
+
item.outcome.ok(item.message_id)
|
|
558
|
+
else:
|
|
559
|
+
item.outcome.fail(error)
|
|
560
|
+
|
|
561
|
+
def _top_up(self, subscription: _Subscription) -> None:
|
|
562
|
+
link = subscription.link
|
|
563
|
+
if link is None:
|
|
564
|
+
return
|
|
565
|
+
grant = credit_to_grant(subscription.prefetch, link.credit, len(subscription.deliveries))
|
|
566
|
+
if grant:
|
|
567
|
+
link.flow(grant)
|
|
568
|
+
|
|
569
|
+
# -- helpers ------------------------------------------------------------------
|
|
570
|
+
|
|
571
|
+
def _refuse_when_closing(self, outcome: Outcome) -> bool:
|
|
572
|
+
if self._fatal is not None:
|
|
573
|
+
outcome.fail(self._fatal)
|
|
574
|
+
return True
|
|
575
|
+
if self._closing:
|
|
576
|
+
outcome.fail(ClientClosed("client closed"))
|
|
577
|
+
return True
|
|
578
|
+
return False
|
|
579
|
+
|
|
580
|
+
def _fail_everything(self, error: BaseException) -> None:
|
|
581
|
+
for sender in self._senders.values():
|
|
582
|
+
for item in sender.pending:
|
|
583
|
+
item.outcome.fail(error)
|
|
584
|
+
for item in sender.in_flight.values():
|
|
585
|
+
item.outcome.fail(ConnectionLost(f"outcome unknown: {error}", item.message_id))
|
|
586
|
+
sender.pending.clear()
|
|
587
|
+
sender.in_flight.clear()
|
|
588
|
+
for subscription in self._subscriptions.values():
|
|
589
|
+
if subscription.opened is not None:
|
|
590
|
+
subscription.opened.fail(error)
|
|
591
|
+
else:
|
|
592
|
+
subscription.fail(error)
|
|
593
|
+
self._subscriptions.clear()
|
|
594
|
+
|
|
595
|
+
def _schedule(self, delay: float, callback: Callable[[], None]) -> Task:
|
|
596
|
+
assert self._container is not None
|
|
597
|
+
return self._container.schedule(delay, _Timer(callback))
|
|
598
|
+
|
|
599
|
+
@staticmethod
|
|
600
|
+
def _cancel(task: Task | None) -> None:
|
|
601
|
+
if task is not None:
|
|
602
|
+
task.cancel()
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _detached(exc: BaseException) -> BaseException:
|
|
606
|
+
# The traceback holds this thread's frames and, through their locals,
|
|
607
|
+
# proton objects; the asyncio side must not end up owning those.
|
|
608
|
+
exc.__context__ = None
|
|
609
|
+
exc.__cause__ = None
|
|
610
|
+
return exc.with_traceback(None)
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _same(a: Connection | None, b: Connection | None) -> bool:
|
|
614
|
+
# Proton hands out a fresh wrapper per event, so identity never matches;
|
|
615
|
+
# wrappers compare equal when they wrap the same C object.
|
|
616
|
+
return a is not None and b is not None and a == b
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _link_name(address: str) -> str:
|
|
620
|
+
# Proton's default name is container-id + address, so a second link to the
|
|
621
|
+
# same address on one connection collides with the first and never attaches.
|
|
622
|
+
return f"{address}-{uuid.uuid4().hex}"
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _abort(connection: Connection) -> None:
|
|
626
|
+
connection.close()
|
|
627
|
+
transport = connection.transport
|
|
628
|
+
if transport is not None:
|
|
629
|
+
transport.close_head()
|
|
630
|
+
transport.close_tail()
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _remote_condition(endpoint: Connection | Link | None) -> Condition | None:
|
|
634
|
+
return endpoint.remote_condition if endpoint is not None else None
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _describe(condition: Condition | None) -> str:
|
|
638
|
+
if condition is None:
|
|
639
|
+
return "no reason given"
|
|
640
|
+
return f"{condition.name}: {condition.description}" if condition.description else str(condition.name)
|