phoenix-channels-python-client 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phoenix_channels_python_client/__init__.py +18 -2
- phoenix_channels_python_client/client.py +12 -6
- phoenix_channels_python_client/client_types.py +56 -3
- phoenix_channels_python_client/default_decider.py +228 -0
- phoenix_channels_python_client/reconnect_controller.py +41 -303
- phoenix_channels_python_client/supervisor.py +49 -78
- {phoenix_channels_python_client-0.4.2.dist-info → phoenix_channels_python_client-0.4.4.dist-info}/METADATA +30 -1
- {phoenix_channels_python_client-0.4.2.dist-info → phoenix_channels_python_client-0.4.4.dist-info}/RECORD +11 -10
- {phoenix_channels_python_client-0.4.2.dist-info → phoenix_channels_python_client-0.4.4.dist-info}/WHEEL +0 -0
- {phoenix_channels_python_client-0.4.2.dist-info → phoenix_channels_python_client-0.4.4.dist-info}/licenses/LICENSE +0 -0
- {phoenix_channels_python_client-0.4.2.dist-info → phoenix_channels_python_client-0.4.4.dist-info}/top_level.txt +0 -0
|
@@ -13,24 +13,40 @@ from phoenix_channels_python_client.client import (
|
|
|
13
13
|
ReconnectPolicy,
|
|
14
14
|
TopicLostCallback,
|
|
15
15
|
)
|
|
16
|
-
from phoenix_channels_python_client.client_types import
|
|
16
|
+
from phoenix_channels_python_client.client_types import (
|
|
17
|
+
Connected,
|
|
18
|
+
ConnectFailed,
|
|
19
|
+
Connecting,
|
|
20
|
+
Disconnected,
|
|
21
|
+
JoinResponseCallback,
|
|
22
|
+
ReconnectDecider,
|
|
23
|
+
ReconnectDecision,
|
|
24
|
+
ReconnectEvent,
|
|
25
|
+
)
|
|
17
26
|
from phoenix_channels_python_client.protocol_handler import (
|
|
18
27
|
PhoenixChannelsProtocolVersion,
|
|
19
28
|
PHXProtocolHandler,
|
|
20
29
|
)
|
|
21
30
|
from phoenix_channels_python_client.utils import setup_logging
|
|
22
31
|
|
|
23
|
-
__version__ = "0.4.
|
|
32
|
+
__version__ = "0.4.4"
|
|
24
33
|
__author__ = "Phoenix Channels Python Client"
|
|
25
34
|
|
|
26
35
|
__all__ = [
|
|
36
|
+
"ConnectFailed",
|
|
37
|
+
"Connected",
|
|
38
|
+
"Connecting",
|
|
27
39
|
"DisconnectCallback",
|
|
40
|
+
"Disconnected",
|
|
28
41
|
"HeartbeatAckCallback",
|
|
29
42
|
"JoinResponseCallback",
|
|
30
43
|
"PHXChannelsClient",
|
|
31
44
|
"PHXProtocolHandler",
|
|
32
45
|
"PhoenixChannelsProtocolVersion",
|
|
33
46
|
"ReconnectCallback",
|
|
47
|
+
"ReconnectDecider",
|
|
48
|
+
"ReconnectDecision",
|
|
49
|
+
"ReconnectEvent",
|
|
34
50
|
"ReconnectPolicy",
|
|
35
51
|
"TopicLostCallback",
|
|
36
52
|
"setup_logging",
|
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import logging
|
|
5
5
|
import math
|
|
6
|
-
from collections import deque
|
|
7
6
|
from collections.abc import Awaitable, Callable
|
|
8
7
|
from types import TracebackType
|
|
9
8
|
from typing import Self, cast
|
|
@@ -19,7 +18,11 @@ from phoenix_channels_python_client.authentication import (
|
|
|
19
18
|
select_authentication_header,
|
|
20
19
|
)
|
|
21
20
|
from phoenix_channels_python_client.client_state_machine import transition_client_state
|
|
22
|
-
from phoenix_channels_python_client.client_types import
|
|
21
|
+
from phoenix_channels_python_client.client_types import (
|
|
22
|
+
ClientState,
|
|
23
|
+
ReconnectDecider,
|
|
24
|
+
ReconnectPolicy,
|
|
25
|
+
)
|
|
23
26
|
from phoenix_channels_python_client.diagnostics import (
|
|
24
27
|
ConnectionLogger,
|
|
25
28
|
diagnostic_secrets,
|
|
@@ -114,7 +117,7 @@ def _validate_heartbeat_threshold(
|
|
|
114
117
|
|
|
115
118
|
|
|
116
119
|
class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerMixin):
|
|
117
|
-
def __init__( # noqa: PLR0913 # the public options
|
|
120
|
+
def __init__( # noqa: PLR0913, PLR0915 # the public options
|
|
118
121
|
self,
|
|
119
122
|
websocket_url: str,
|
|
120
123
|
api_key: str,
|
|
@@ -133,6 +136,7 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
133
136
|
on_disconnect: DisconnectCallback | None = None,
|
|
134
137
|
on_heartbeat_ack: HeartbeatAckCallback | None = None,
|
|
135
138
|
on_topic_lost: TopicLostCallback | None = None,
|
|
139
|
+
reconnect_decider: ReconnectDecider | None = None,
|
|
136
140
|
additional_headers: dict[str, str] | None = None,
|
|
137
141
|
) -> None:
|
|
138
142
|
self.additional_headers = normalize_headers(additional_headers)
|
|
@@ -185,7 +189,6 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
185
189
|
self._shutdown_task: asyncio.Task[None] | None = None
|
|
186
190
|
self._message_routing_task: asyncio.Task[None] | None = None
|
|
187
191
|
self._initial_connection_future: asyncio.Future[None] | None = None
|
|
188
|
-
self._rapid_disconnects: deque[float] = deque()
|
|
189
192
|
self._terminal_error: Exception | None = None
|
|
190
193
|
self._heartbeat_dead_threshold_s = heartbeat_dead_threshold_s
|
|
191
194
|
self._heartbeat_deadline: float | None = None
|
|
@@ -197,6 +200,8 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
197
200
|
self._on_disconnect = on_disconnect
|
|
198
201
|
self._on_heartbeat_ack = on_heartbeat_ack
|
|
199
202
|
self._on_topic_lost = on_topic_lost
|
|
203
|
+
self._reconnect_decider = reconnect_decider
|
|
204
|
+
self._decider = self._new_decider()
|
|
200
205
|
self._lifecycle_tasks: set[asyncio.Task[None]] = set()
|
|
201
206
|
self._cleanup_tasks: set[asyncio.Task[None]] = set()
|
|
202
207
|
self._forced_close_pending = False
|
|
@@ -226,7 +231,7 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
226
231
|
|
|
227
232
|
self._shutdown_event.clear()
|
|
228
233
|
self._connected_event.clear()
|
|
229
|
-
self.
|
|
234
|
+
self._decider = self._new_decider()
|
|
230
235
|
self._terminal_error = None
|
|
231
236
|
self._conn_generation = 0
|
|
232
237
|
self._initial_connection_future = asyncio.get_running_loop().create_future()
|
|
@@ -236,7 +241,8 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
236
241
|
|
|
237
242
|
try:
|
|
238
243
|
await self._initial_connection_future
|
|
239
|
-
except Exception:
|
|
244
|
+
except (Exception, asyncio.CancelledError):
|
|
245
|
+
# Not BaseException: awaiting cleanup on GeneratorExit raises RuntimeError.
|
|
240
246
|
await self.shutdown("Initial connection failed")
|
|
241
247
|
raise
|
|
242
248
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import math
|
|
3
4
|
from collections.abc import Callable
|
|
4
5
|
from dataclasses import dataclass
|
|
5
6
|
from enum import StrEnum
|
|
@@ -71,9 +72,61 @@ class ReconnectPolicy(BaseModel):
|
|
|
71
72
|
return self
|
|
72
73
|
|
|
73
74
|
|
|
74
|
-
@dataclass(frozen=True)
|
|
75
|
+
@dataclass(frozen=True, kw_only=True)
|
|
75
76
|
class ReconnectDecision:
|
|
77
|
+
"""Whether the client goes on, and how long it waits before the next attempt.
|
|
78
|
+
|
|
79
|
+
``delay_s`` applies after ``Disconnected`` and ``ConnectFailed``; a gate
|
|
80
|
+
(``Connecting``, ``Connected``) only proceeds or stops. A stop with a
|
|
81
|
+
``terminal_error`` ends the client with that error.
|
|
82
|
+
"""
|
|
83
|
+
|
|
76
84
|
should_reconnect: bool
|
|
77
|
-
|
|
78
|
-
max_delay_s: float | None = None
|
|
85
|
+
delay_s: float = 0.0
|
|
79
86
|
terminal_error: PHXConnectionError | None = None
|
|
87
|
+
|
|
88
|
+
def __post_init__(self) -> None:
|
|
89
|
+
if not math.isfinite(self.delay_s) or self.delay_s < 0:
|
|
90
|
+
raise ValueError("delay_s must be finite and >= 0")
|
|
91
|
+
if not self.should_reconnect and self.delay_s:
|
|
92
|
+
raise ValueError("a stop has no delay_s")
|
|
93
|
+
if self.should_reconnect and self.terminal_error is not None:
|
|
94
|
+
raise ValueError("a terminal_error requires should_reconnect=False")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
PROCEED = ReconnectDecision(should_reconnect=True)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class Connecting:
|
|
102
|
+
"""A connection attempt is about to start."""
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass(frozen=True)
|
|
106
|
+
class Connected:
|
|
107
|
+
"""The socket for ``generation`` is up; topics rejoin only if this proceeds."""
|
|
108
|
+
|
|
109
|
+
generation: int
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(frozen=True)
|
|
113
|
+
class Disconnected:
|
|
114
|
+
"""The connection closed. ``close_code`` is ``None`` for a forced close."""
|
|
115
|
+
|
|
116
|
+
close_code: int | None
|
|
117
|
+
close_reason: str
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(frozen=True)
|
|
121
|
+
class ConnectFailed:
|
|
122
|
+
"""A connection attempt failed with ``error``, e.g. a refused handshake."""
|
|
123
|
+
|
|
124
|
+
error: Exception
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
ReconnectEvent = Connecting | Connected | Disconnected | ConnectFailed
|
|
128
|
+
|
|
129
|
+
# Decision points that only proceed or stop; no attempt follows them to delay.
|
|
130
|
+
GATE_EVENTS = (Connecting, Connected)
|
|
131
|
+
|
|
132
|
+
ReconnectDecider = Callable[[ReconnectEvent], ReconnectDecision]
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The built-in reconnect policy, as a ``reconnect_decider``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import random
|
|
8
|
+
from collections import deque
|
|
9
|
+
from http import HTTPStatus
|
|
10
|
+
|
|
11
|
+
from websockets.exceptions import InvalidStatus
|
|
12
|
+
from websockets.frames import CloseCode
|
|
13
|
+
|
|
14
|
+
from phoenix_channels_python_client.client_types import (
|
|
15
|
+
PROCEED,
|
|
16
|
+
Connected,
|
|
17
|
+
ConnectFailed,
|
|
18
|
+
Disconnected,
|
|
19
|
+
ReconnectDecision,
|
|
20
|
+
ReconnectEvent,
|
|
21
|
+
ReconnectPolicy,
|
|
22
|
+
)
|
|
23
|
+
from phoenix_channels_python_client.exceptions import PHXConnectionError
|
|
24
|
+
|
|
25
|
+
# The Band platform answers every auth refusal with 403, and its websocket spec
|
|
26
|
+
# (thenvoi-platform spec/websocket-api.draft.md) makes a rejected reconnect terminal.
|
|
27
|
+
# A 401 could only come from an edge layer, so it and every other status retry.
|
|
28
|
+
TERMINAL_HANDSHAKE_STATUS = HTTPStatus.FORBIDDEN
|
|
29
|
+
|
|
30
|
+
# A (low, high) range a close's backoff is raised into.
|
|
31
|
+
DelayRange = tuple[float, float]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def random_between(min_delay_s: float, max_delay_s: float) -> float:
|
|
35
|
+
low = max(0.0, min_delay_s)
|
|
36
|
+
high = max(low, max_delay_s)
|
|
37
|
+
if high == low:
|
|
38
|
+
return low
|
|
39
|
+
return low + (random.random() * (high - low)) # noqa: S311 # jitter
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def exponential_delay(policy: ReconnectPolicy, attempt: int) -> float:
|
|
43
|
+
"""The policy's capped exponential backoff for ``attempt``, unjittered."""
|
|
44
|
+
return min(
|
|
45
|
+
policy.max_delay_s,
|
|
46
|
+
policy.base_delay_s * (policy.factor ** max(attempt, 0)),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def equal_jitter(delay: float) -> float:
|
|
51
|
+
# Equal jitter avoids synchronization while keeping a meaningful minimum delay.
|
|
52
|
+
return random_between(delay / 2, delay)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _now() -> float:
|
|
56
|
+
return asyncio.get_running_loop().time()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _terminal_decision(cause: str) -> ReconnectDecision:
|
|
60
|
+
return ReconnectDecision(
|
|
61
|
+
should_reconnect=False,
|
|
62
|
+
terminal_error=PHXConnectionError(
|
|
63
|
+
f"Reconnect disabled due to terminal {cause}"
|
|
64
|
+
),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class DefaultReconnectDecider:
|
|
69
|
+
"""Classifies closes and refusals, backs off, and stops on rapid churn.
|
|
70
|
+
|
|
71
|
+
Holds one run's backoff attempt and rapid-disconnect history, so the client
|
|
72
|
+
builds a fresh one each time it is entered.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(self, policy: ReconnectPolicy, logger: logging.Logger) -> None:
|
|
76
|
+
self._policy = policy
|
|
77
|
+
self._logger = logger
|
|
78
|
+
self._attempt = 0
|
|
79
|
+
self._rapid_disconnects: deque[float] = deque()
|
|
80
|
+
self._connected_at = 0.0
|
|
81
|
+
|
|
82
|
+
def __call__(self, event: ReconnectEvent) -> ReconnectDecision:
|
|
83
|
+
match event:
|
|
84
|
+
case Connected():
|
|
85
|
+
self._connected_at = _now()
|
|
86
|
+
return PROCEED
|
|
87
|
+
case Disconnected():
|
|
88
|
+
return self._after_close(event.close_code, event.close_reason)
|
|
89
|
+
case ConnectFailed():
|
|
90
|
+
return self._after_connect_failure(event.error)
|
|
91
|
+
case _: # Connecting: always attempt.
|
|
92
|
+
return PROCEED
|
|
93
|
+
|
|
94
|
+
def _after_close(
|
|
95
|
+
self, close_code: int | None, close_reason: str
|
|
96
|
+
) -> ReconnectDecision:
|
|
97
|
+
stop = self._stop_for_close(close_code, close_reason)
|
|
98
|
+
if stop is not None:
|
|
99
|
+
return stop
|
|
100
|
+
uptime_s = _now() - self._connected_at
|
|
101
|
+
if self._suppressed(uptime_s):
|
|
102
|
+
return self._suppression()
|
|
103
|
+
if uptime_s >= self._policy.stable_reset_s:
|
|
104
|
+
self._attempt = 0
|
|
105
|
+
self._rapid_disconnects.clear()
|
|
106
|
+
else:
|
|
107
|
+
self._attempt += 1
|
|
108
|
+
delay = self._backoff(self._attempt)
|
|
109
|
+
delay_range = self._delay_range_for_close(close_code)
|
|
110
|
+
if delay_range is not None:
|
|
111
|
+
delay = max(delay, random_between(*delay_range))
|
|
112
|
+
return ReconnectDecision(should_reconnect=True, delay_s=delay)
|
|
113
|
+
|
|
114
|
+
def _after_connect_failure(self, error: Exception) -> ReconnectDecision:
|
|
115
|
+
if (
|
|
116
|
+
isinstance(error, InvalidStatus)
|
|
117
|
+
and error.response.status_code == TERMINAL_HANDSHAKE_STATUS
|
|
118
|
+
):
|
|
119
|
+
return _terminal_decision(
|
|
120
|
+
f"handshake status {TERMINAL_HANDSHAKE_STATUS} "
|
|
121
|
+
f"({TERMINAL_HANDSHAKE_STATUS.phrase})"
|
|
122
|
+
)
|
|
123
|
+
if self._suppressed(0.0):
|
|
124
|
+
return self._suppression()
|
|
125
|
+
delay = self._backoff(self._attempt)
|
|
126
|
+
self._attempt += 1
|
|
127
|
+
return ReconnectDecision(should_reconnect=True, delay_s=delay)
|
|
128
|
+
|
|
129
|
+
def _stop_for_close(
|
|
130
|
+
self, close_code: int | None, close_reason: str
|
|
131
|
+
) -> ReconnectDecision | None:
|
|
132
|
+
if (
|
|
133
|
+
close_code in {CloseCode.NORMAL_CLOSURE, CloseCode.GOING_AWAY}
|
|
134
|
+
and not self._policy.reconnect_on_normal_close
|
|
135
|
+
):
|
|
136
|
+
return ReconnectDecision(should_reconnect=False)
|
|
137
|
+
if (
|
|
138
|
+
close_code == CloseCode.POLICY_VIOLATION
|
|
139
|
+
and self._policy.policy_violation_is_terminal
|
|
140
|
+
):
|
|
141
|
+
reason = close_reason or "policy violation"
|
|
142
|
+
return _terminal_decision(
|
|
143
|
+
f"close code {CloseCode.POLICY_VIOLATION} ({reason})"
|
|
144
|
+
)
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
def _delay_range_for_close(self, close_code: int | None) -> DelayRange | None:
|
|
148
|
+
policy = self._policy
|
|
149
|
+
match close_code:
|
|
150
|
+
case CloseCode.SERVICE_RESTART:
|
|
151
|
+
return (
|
|
152
|
+
policy.service_restart_min_delay_s,
|
|
153
|
+
policy.service_restart_max_delay_s,
|
|
154
|
+
)
|
|
155
|
+
case CloseCode.TRY_AGAIN_LATER:
|
|
156
|
+
return (
|
|
157
|
+
policy.try_again_later_min_delay_s,
|
|
158
|
+
policy.try_again_later_max_delay_s,
|
|
159
|
+
)
|
|
160
|
+
case _:
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
def _suppressed(self, uptime_s: float) -> bool:
|
|
164
|
+
"""Record a disconnect after ``uptime_s``; whether churn now stops us."""
|
|
165
|
+
now = _now()
|
|
166
|
+
if uptime_s < self._policy.rapid_disconnect_uptime_s:
|
|
167
|
+
self._rapid_disconnects.append(now)
|
|
168
|
+
while (
|
|
169
|
+
self._rapid_disconnects
|
|
170
|
+
and now - self._rapid_disconnects[0] > self._policy.rapid_window_s
|
|
171
|
+
):
|
|
172
|
+
self._rapid_disconnects.popleft()
|
|
173
|
+
threshold = self._policy.rapid_suppress_disconnect_count
|
|
174
|
+
rapid_count = len(self._rapid_disconnects)
|
|
175
|
+
self._logger.debug(
|
|
176
|
+
"Recorded disconnect. uptime_s=%s rapid_count=%s threshold=%s",
|
|
177
|
+
uptime_s,
|
|
178
|
+
rapid_count,
|
|
179
|
+
threshold,
|
|
180
|
+
)
|
|
181
|
+
return threshold > 0 and rapid_count >= threshold
|
|
182
|
+
|
|
183
|
+
def _suppression(self) -> ReconnectDecision:
|
|
184
|
+
return ReconnectDecision(
|
|
185
|
+
should_reconnect=False,
|
|
186
|
+
terminal_error=PHXConnectionError(
|
|
187
|
+
"Reconnect suppressed after repeated rapid disconnects. "
|
|
188
|
+
"Likely duplicate connection or unstable endpoint."
|
|
189
|
+
),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def _rapid_floors(self) -> tuple[float, ...]:
|
|
193
|
+
"""The floor for each rapid-disconnect count before the cooldown starts."""
|
|
194
|
+
return (
|
|
195
|
+
0.0,
|
|
196
|
+
self._policy.rapid_first_min_delay_s,
|
|
197
|
+
self._policy.rapid_second_min_delay_s,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
def _backoff(self, attempt: int) -> float:
|
|
201
|
+
"""The jittered wait for ``attempt``, raised to the rapid-disconnect floor."""
|
|
202
|
+
rapid_count = len(self._rapid_disconnects)
|
|
203
|
+
cooldown_steps = rapid_count - len(self._rapid_floors())
|
|
204
|
+
if cooldown_steps < 0:
|
|
205
|
+
floor = self._rapid_floors()[rapid_count]
|
|
206
|
+
else:
|
|
207
|
+
floor = min(
|
|
208
|
+
self._policy.rapid_cooldown_base_s
|
|
209
|
+
+ (self._policy.rapid_cooldown_step_s * cooldown_steps),
|
|
210
|
+
self._policy.rapid_cooldown_max_s,
|
|
211
|
+
)
|
|
212
|
+
delay = max(exponential_delay(self._policy, attempt), floor)
|
|
213
|
+
if delay <= 0:
|
|
214
|
+
return 0.0
|
|
215
|
+
if cooldown_steps < 0:
|
|
216
|
+
computed = equal_jitter(delay)
|
|
217
|
+
else:
|
|
218
|
+
computed = random_between(
|
|
219
|
+
delay * self._policy.rapid_hold_down_jitter_low_ratio,
|
|
220
|
+
delay * self._policy.rapid_hold_down_jitter_high_ratio,
|
|
221
|
+
)
|
|
222
|
+
self._logger.debug(
|
|
223
|
+
"Computed reconnect delay. attempt=%s rapid_count=%s computed_delay_s=%s",
|
|
224
|
+
attempt,
|
|
225
|
+
rapid_count,
|
|
226
|
+
computed,
|
|
227
|
+
)
|
|
228
|
+
return computed
|
|
@@ -1,34 +1,31 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import inspect
|
|
4
4
|
import logging
|
|
5
|
-
import random
|
|
6
|
-
from collections import deque
|
|
7
|
-
from http import HTTPStatus
|
|
8
5
|
|
|
9
6
|
from websockets import ClientConnection
|
|
10
|
-
from websockets.exceptions import ConnectionClosed
|
|
11
|
-
from websockets.frames import CloseCode
|
|
7
|
+
from websockets.exceptions import ConnectionClosed
|
|
12
8
|
|
|
13
9
|
from phoenix_channels_python_client.client_types import (
|
|
10
|
+
GATE_EVENTS,
|
|
11
|
+
ReconnectDecider,
|
|
14
12
|
ReconnectDecision,
|
|
13
|
+
ReconnectEvent,
|
|
15
14
|
ReconnectPolicy,
|
|
16
15
|
)
|
|
16
|
+
from phoenix_channels_python_client.default_decider import (
|
|
17
|
+
DefaultReconnectDecider,
|
|
18
|
+
equal_jitter,
|
|
19
|
+
exponential_delay,
|
|
20
|
+
)
|
|
17
21
|
from phoenix_channels_python_client.exceptions import PHXConnectionError
|
|
18
22
|
|
|
19
|
-
# The Band platform answers every auth refusal with 403, and its websocket spec
|
|
20
|
-
# (thenvoi-platform spec/websocket-api.draft.md) makes a rejected reconnect terminal.
|
|
21
|
-
# A 401 could only come from an edge layer, so it and every other status retry.
|
|
22
|
-
TERMINAL_HANDSHAKE_STATUS = HTTPStatus.FORBIDDEN
|
|
23
|
-
|
|
24
23
|
|
|
25
|
-
def
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
),
|
|
31
|
-
)
|
|
24
|
+
def _decider_failure(message: str, cause: Exception | None = None) -> ReconnectDecision:
|
|
25
|
+
"""A stop for a broken decider; never the built-in policy deciding instead."""
|
|
26
|
+
error = PHXConnectionError(message)
|
|
27
|
+
error.__cause__ = cause
|
|
28
|
+
return ReconnectDecision(should_reconnect=False, terminal_error=error)
|
|
32
29
|
|
|
33
30
|
|
|
34
31
|
class ReconnectControllerMixin:
|
|
@@ -38,137 +35,35 @@ class ReconnectControllerMixin:
|
|
|
38
35
|
raise NotImplementedError
|
|
39
36
|
|
|
40
37
|
reconnect_policy: ReconnectPolicy
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
self.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
"
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def _should_suppress_reconnect(self) -> bool:
|
|
68
|
-
threshold = self.reconnect_policy.rapid_suppress_disconnect_count
|
|
69
|
-
should_suppress = threshold > 0 and len(self._rapid_disconnects) >= threshold
|
|
70
|
-
if should_suppress:
|
|
71
|
-
self.logger.debug(
|
|
72
|
-
"Reconnect suppression threshold met. rapid_count=%s threshold=%s",
|
|
73
|
-
len(self._rapid_disconnects),
|
|
74
|
-
threshold,
|
|
75
|
-
extra={
|
|
76
|
-
"rapid_count": len(self._rapid_disconnects),
|
|
77
|
-
"threshold": threshold,
|
|
78
|
-
},
|
|
79
|
-
)
|
|
80
|
-
return should_suppress
|
|
81
|
-
|
|
82
|
-
def _rapid_floors(self) -> tuple[float, ...]:
|
|
83
|
-
"""The floor for each rapid-disconnect count before the cooldown starts."""
|
|
84
|
-
return (
|
|
85
|
-
0.0,
|
|
86
|
-
self.reconnect_policy.rapid_first_min_delay_s,
|
|
87
|
-
self.reconnect_policy.rapid_second_min_delay_s,
|
|
88
|
-
)
|
|
89
|
-
|
|
90
|
-
def _rapid_cooldown_steps(self, rapid_count: int) -> int:
|
|
91
|
-
"""Steps into the cooldown; negative while a floor still applies."""
|
|
92
|
-
return rapid_count - len(self._rapid_floors())
|
|
93
|
-
|
|
94
|
-
def _rapid_min_delay(self, rapid_count: int) -> float:
|
|
95
|
-
steps = self._rapid_cooldown_steps(rapid_count)
|
|
96
|
-
if steps < 0:
|
|
97
|
-
return self._rapid_floors()[rapid_count]
|
|
98
|
-
return min(
|
|
99
|
-
self.reconnect_policy.rapid_cooldown_base_s
|
|
100
|
-
+ (self.reconnect_policy.rapid_cooldown_step_s * steps),
|
|
101
|
-
self.reconnect_policy.rapid_cooldown_max_s,
|
|
102
|
-
)
|
|
103
|
-
|
|
104
|
-
def _exponential_delay(self, attempt: int) -> float:
|
|
105
|
-
"""The policy's capped exponential backoff for ``attempt``, unjittered."""
|
|
106
|
-
return min(
|
|
107
|
-
self.reconnect_policy.max_delay_s,
|
|
108
|
-
self.reconnect_policy.base_delay_s
|
|
109
|
-
* (self.reconnect_policy.factor ** max(attempt, 0)),
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
def _equal_jitter(self, delay: float) -> float:
|
|
113
|
-
# Equal jitter avoids synchronization while keeping a meaningful minimum delay.
|
|
114
|
-
return self._random_between(delay / 2, delay)
|
|
38
|
+
_reconnect_decider: ReconnectDecider | None
|
|
39
|
+
_decider: ReconnectDecider
|
|
40
|
+
|
|
41
|
+
def _new_decider(self) -> ReconnectDecider:
|
|
42
|
+
"""The consumer's decider, or a fresh built-in one for this run."""
|
|
43
|
+
if self._reconnect_decider is not None:
|
|
44
|
+
return self._reconnect_decider
|
|
45
|
+
return DefaultReconnectDecider(self.reconnect_policy, self.logger)
|
|
46
|
+
|
|
47
|
+
def _decide(self, event: ReconnectEvent) -> ReconnectDecision:
|
|
48
|
+
"""The decider's answer to ``event``; a broken answer stops the client."""
|
|
49
|
+
try:
|
|
50
|
+
decision = self._decider(event)
|
|
51
|
+
except Exception as exc:
|
|
52
|
+
self.logger.exception("reconnect_decider failed; stopping the client")
|
|
53
|
+
return _decider_failure("reconnect_decider failed", exc)
|
|
54
|
+
if inspect.iscoroutine(decision):
|
|
55
|
+
decision.close() # Never awaited; closed so it raises no warning.
|
|
56
|
+
return _decider_failure("reconnect_decider must be synchronous")
|
|
57
|
+
if not isinstance(decision, ReconnectDecision):
|
|
58
|
+
return _decider_failure("reconnect_decider returned no ReconnectDecision")
|
|
59
|
+
if decision.delay_s and isinstance(event, GATE_EVENTS):
|
|
60
|
+
# No attempt follows a gate, so its delay would be silently dropped.
|
|
61
|
+
return _decider_failure("reconnect_decider returned a delay for a gate")
|
|
62
|
+
return decision
|
|
115
63
|
|
|
116
64
|
def _channel_rejoin_delay(self, attempt: int) -> float:
|
|
117
65
|
# No rapid-disconnect floors: those describe socket churn, not one channel.
|
|
118
|
-
return
|
|
119
|
-
|
|
120
|
-
def _compute_reconnect_delay(self, attempt: int) -> float:
|
|
121
|
-
base_delay = self._exponential_delay(attempt)
|
|
122
|
-
|
|
123
|
-
rapid_count = len(self._rapid_disconnects)
|
|
124
|
-
min_delay = self._rapid_min_delay(rapid_count)
|
|
125
|
-
|
|
126
|
-
delay = max(base_delay, min_delay)
|
|
127
|
-
if delay <= 0:
|
|
128
|
-
return 0.0
|
|
129
|
-
|
|
130
|
-
if self._rapid_cooldown_steps(rapid_count) >= 0:
|
|
131
|
-
low_ratio = self.reconnect_policy.rapid_hold_down_jitter_low_ratio
|
|
132
|
-
high_ratio = self.reconnect_policy.rapid_hold_down_jitter_high_ratio
|
|
133
|
-
min_jittered = delay * low_ratio
|
|
134
|
-
max_jittered = delay * high_ratio
|
|
135
|
-
computed = self._random_between(min_jittered, max_jittered)
|
|
136
|
-
self.logger.debug(
|
|
137
|
-
"Computed reconnect delay with hold-down jitter. attempt=%s "
|
|
138
|
-
"rapid_count=%s base_delay_s=%s min_delay_s=%s computed_delay_s=%s",
|
|
139
|
-
attempt,
|
|
140
|
-
rapid_count,
|
|
141
|
-
base_delay,
|
|
142
|
-
min_delay,
|
|
143
|
-
computed,
|
|
144
|
-
extra={
|
|
145
|
-
"attempt": attempt,
|
|
146
|
-
"rapid_count": rapid_count,
|
|
147
|
-
"base_delay_s": base_delay,
|
|
148
|
-
"min_delay_s": min_delay,
|
|
149
|
-
"computed_delay_s": computed,
|
|
150
|
-
},
|
|
151
|
-
)
|
|
152
|
-
return computed
|
|
153
|
-
|
|
154
|
-
computed = self._equal_jitter(delay)
|
|
155
|
-
self.logger.debug(
|
|
156
|
-
"Computed reconnect delay with equal jitter. attempt=%s rapid_count=%s "
|
|
157
|
-
"base_delay_s=%s min_delay_s=%s computed_delay_s=%s",
|
|
158
|
-
attempt,
|
|
159
|
-
rapid_count,
|
|
160
|
-
base_delay,
|
|
161
|
-
min_delay,
|
|
162
|
-
computed,
|
|
163
|
-
extra={
|
|
164
|
-
"attempt": attempt,
|
|
165
|
-
"rapid_count": rapid_count,
|
|
166
|
-
"base_delay_s": base_delay,
|
|
167
|
-
"min_delay_s": min_delay,
|
|
168
|
-
"computed_delay_s": computed,
|
|
169
|
-
},
|
|
170
|
-
)
|
|
171
|
-
return computed
|
|
66
|
+
return equal_jitter(exponential_delay(self.reconnect_policy, attempt))
|
|
172
67
|
|
|
173
68
|
def _extract_close_details(
|
|
174
69
|
self,
|
|
@@ -182,160 +77,3 @@ class ReconnectControllerMixin:
|
|
|
182
77
|
if routing_error.sent is not None:
|
|
183
78
|
return routing_error.sent.code, routing_error.sent.reason
|
|
184
79
|
return connection.close_code, connection.close_reason or ""
|
|
185
|
-
|
|
186
|
-
def _classify_disconnect(
|
|
187
|
-
self, close_code: int | None, close_reason: str
|
|
188
|
-
) -> ReconnectDecision:
|
|
189
|
-
if close_code is None:
|
|
190
|
-
decision = ReconnectDecision(should_reconnect=True)
|
|
191
|
-
self.logger.debug(
|
|
192
|
-
"Disconnect classification result. close_code=%s close_reason=%s "
|
|
193
|
-
"should_reconnect=%s",
|
|
194
|
-
close_code,
|
|
195
|
-
close_reason,
|
|
196
|
-
decision.should_reconnect,
|
|
197
|
-
extra={
|
|
198
|
-
"close_code": close_code,
|
|
199
|
-
"close_reason": close_reason,
|
|
200
|
-
"should_reconnect": decision.should_reconnect,
|
|
201
|
-
},
|
|
202
|
-
)
|
|
203
|
-
return decision
|
|
204
|
-
|
|
205
|
-
if (
|
|
206
|
-
close_code in {CloseCode.NORMAL_CLOSURE, CloseCode.GOING_AWAY}
|
|
207
|
-
and not self.reconnect_policy.reconnect_on_normal_close
|
|
208
|
-
):
|
|
209
|
-
decision = ReconnectDecision(should_reconnect=False)
|
|
210
|
-
self.logger.debug(
|
|
211
|
-
"Disconnect classification result. close_code=%s close_reason=%s "
|
|
212
|
-
"should_reconnect=%s",
|
|
213
|
-
close_code,
|
|
214
|
-
close_reason,
|
|
215
|
-
decision.should_reconnect,
|
|
216
|
-
extra={
|
|
217
|
-
"close_code": close_code,
|
|
218
|
-
"close_reason": close_reason,
|
|
219
|
-
"should_reconnect": decision.should_reconnect,
|
|
220
|
-
},
|
|
221
|
-
)
|
|
222
|
-
return decision
|
|
223
|
-
|
|
224
|
-
if (
|
|
225
|
-
close_code == CloseCode.POLICY_VIOLATION
|
|
226
|
-
and self.reconnect_policy.policy_violation_is_terminal
|
|
227
|
-
):
|
|
228
|
-
reason = close_reason or "policy violation"
|
|
229
|
-
decision = _terminal_decision(
|
|
230
|
-
f"close code {CloseCode.POLICY_VIOLATION} ({reason})"
|
|
231
|
-
)
|
|
232
|
-
self.logger.debug(
|
|
233
|
-
"Disconnect classification result. close_code=%s close_reason=%s "
|
|
234
|
-
"should_reconnect=%s terminal=%s",
|
|
235
|
-
close_code,
|
|
236
|
-
close_reason,
|
|
237
|
-
decision.should_reconnect,
|
|
238
|
-
decision.terminal_error,
|
|
239
|
-
extra={
|
|
240
|
-
"close_code": close_code,
|
|
241
|
-
"close_reason": close_reason,
|
|
242
|
-
"should_reconnect": decision.should_reconnect,
|
|
243
|
-
"terminal": True,
|
|
244
|
-
},
|
|
245
|
-
)
|
|
246
|
-
return decision
|
|
247
|
-
|
|
248
|
-
if close_code == CloseCode.SERVICE_RESTART:
|
|
249
|
-
decision = ReconnectDecision(
|
|
250
|
-
should_reconnect=True,
|
|
251
|
-
min_delay_s=self.reconnect_policy.service_restart_min_delay_s,
|
|
252
|
-
max_delay_s=self.reconnect_policy.service_restart_max_delay_s,
|
|
253
|
-
)
|
|
254
|
-
self.logger.debug(
|
|
255
|
-
"Disconnect classification result. close_code=%s close_reason=%s "
|
|
256
|
-
"should_reconnect=%s min_delay_s=%s max_delay_s=%s",
|
|
257
|
-
close_code,
|
|
258
|
-
close_reason,
|
|
259
|
-
decision.should_reconnect,
|
|
260
|
-
decision.min_delay_s,
|
|
261
|
-
decision.max_delay_s,
|
|
262
|
-
extra={
|
|
263
|
-
"close_code": close_code,
|
|
264
|
-
"close_reason": close_reason,
|
|
265
|
-
"should_reconnect": decision.should_reconnect,
|
|
266
|
-
"min_delay_s": decision.min_delay_s,
|
|
267
|
-
"max_delay_s": decision.max_delay_s,
|
|
268
|
-
},
|
|
269
|
-
)
|
|
270
|
-
return decision
|
|
271
|
-
|
|
272
|
-
if close_code == CloseCode.TRY_AGAIN_LATER:
|
|
273
|
-
decision = ReconnectDecision(
|
|
274
|
-
should_reconnect=True,
|
|
275
|
-
min_delay_s=self.reconnect_policy.try_again_later_min_delay_s,
|
|
276
|
-
max_delay_s=self.reconnect_policy.try_again_later_max_delay_s,
|
|
277
|
-
)
|
|
278
|
-
self.logger.debug(
|
|
279
|
-
"Disconnect classification result. close_code=%s close_reason=%s "
|
|
280
|
-
"should_reconnect=%s min_delay_s=%s max_delay_s=%s",
|
|
281
|
-
close_code,
|
|
282
|
-
close_reason,
|
|
283
|
-
decision.should_reconnect,
|
|
284
|
-
decision.min_delay_s,
|
|
285
|
-
decision.max_delay_s,
|
|
286
|
-
extra={
|
|
287
|
-
"close_code": close_code,
|
|
288
|
-
"close_reason": close_reason,
|
|
289
|
-
"should_reconnect": decision.should_reconnect,
|
|
290
|
-
"min_delay_s": decision.min_delay_s,
|
|
291
|
-
"max_delay_s": decision.max_delay_s,
|
|
292
|
-
},
|
|
293
|
-
)
|
|
294
|
-
return decision
|
|
295
|
-
|
|
296
|
-
decision = ReconnectDecision(should_reconnect=True)
|
|
297
|
-
self.logger.debug(
|
|
298
|
-
"Disconnect classification result. close_code=%s close_reason=%s "
|
|
299
|
-
"should_reconnect=%s",
|
|
300
|
-
close_code,
|
|
301
|
-
close_reason,
|
|
302
|
-
decision.should_reconnect,
|
|
303
|
-
extra={
|
|
304
|
-
"close_code": close_code,
|
|
305
|
-
"close_reason": close_reason,
|
|
306
|
-
"should_reconnect": decision.should_reconnect,
|
|
307
|
-
},
|
|
308
|
-
)
|
|
309
|
-
return decision
|
|
310
|
-
|
|
311
|
-
def _classify_connect_failure(self, error: Exception) -> ReconnectDecision:
|
|
312
|
-
if (
|
|
313
|
-
isinstance(error, InvalidStatus)
|
|
314
|
-
and error.response.status_code == TERMINAL_HANDSHAKE_STATUS
|
|
315
|
-
):
|
|
316
|
-
return _terminal_decision(
|
|
317
|
-
f"handshake status {TERMINAL_HANDSHAKE_STATUS} "
|
|
318
|
-
f"({TERMINAL_HANDSHAKE_STATUS.phrase})"
|
|
319
|
-
)
|
|
320
|
-
return ReconnectDecision(should_reconnect=True)
|
|
321
|
-
|
|
322
|
-
def _apply_disconnect_delay_override(
|
|
323
|
-
self,
|
|
324
|
-
computed_delay_s: float,
|
|
325
|
-
decision: ReconnectDecision,
|
|
326
|
-
) -> float:
|
|
327
|
-
if decision.min_delay_s is None:
|
|
328
|
-
return computed_delay_s
|
|
329
|
-
|
|
330
|
-
if decision.max_delay_s is None:
|
|
331
|
-
return max(computed_delay_s, decision.min_delay_s)
|
|
332
|
-
|
|
333
|
-
override = self._random_between(decision.min_delay_s, decision.max_delay_s)
|
|
334
|
-
return max(computed_delay_s, override)
|
|
335
|
-
|
|
336
|
-
def _random_between(self, min_delay_s: float, max_delay_s: float) -> float:
|
|
337
|
-
low = max(0.0, min_delay_s)
|
|
338
|
-
high = max(low, max_delay_s)
|
|
339
|
-
if high == low:
|
|
340
|
-
return low
|
|
341
|
-
return low + (random.random() * (high - low)) # noqa: S311 # jitter
|
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import inspect
|
|
5
5
|
import logging
|
|
6
|
-
from collections import deque
|
|
7
6
|
from collections.abc import Awaitable, Callable
|
|
8
7
|
from contextlib import nullcontext
|
|
9
8
|
from typing import NamedTuple, Protocol, cast
|
|
@@ -14,8 +13,12 @@ from websockets.protocol import State
|
|
|
14
13
|
|
|
15
14
|
from phoenix_channels_python_client.client_types import (
|
|
16
15
|
ClientState,
|
|
16
|
+
Connected,
|
|
17
|
+
ConnectFailed,
|
|
18
|
+
Connecting,
|
|
19
|
+
Disconnected,
|
|
17
20
|
ReconnectDecision,
|
|
18
|
-
|
|
21
|
+
ReconnectEvent,
|
|
19
22
|
)
|
|
20
23
|
from phoenix_channels_python_client.exceptions import PHXConnectionError
|
|
21
24
|
from phoenix_channels_python_client.phx_messages import (
|
|
@@ -28,9 +31,8 @@ from phoenix_channels_python_client.shutdown_signals import handle_shutdown_sign
|
|
|
28
31
|
from phoenix_channels_python_client.topic_subscription import TopicSubscription
|
|
29
32
|
from phoenix_channels_python_client.utils import cancel_and_wait, make_message
|
|
30
33
|
|
|
31
|
-
# RFC 6455 §7.4.2 private-use range (4000-4999, unregistrable)
|
|
32
|
-
#
|
|
33
|
-
# reconnects like any other unclassified disconnect.
|
|
34
|
+
# RFC 6455 §7.4.2 private-use range (4000-4999, unregistrable). The decider
|
|
35
|
+
# never sees it: a forced close is decided with close_code=None.
|
|
34
36
|
_FORCED_CLOSE_CODE = 4000
|
|
35
37
|
|
|
36
38
|
# RFC 6455 §5.5.1: control frames (including Close) cap at 125 bytes total;
|
|
@@ -58,19 +60,10 @@ class _SupervisorRuntimeDeps(Protocol):
|
|
|
58
60
|
async def _rejoin_topics(self, generation: int) -> None: ...
|
|
59
61
|
def _fail_pending_joins(self, error: Exception) -> None: ...
|
|
60
62
|
def _channel_recoveries(self) -> list[asyncio.Task[None]]: ...
|
|
61
|
-
def _record_disconnect(self, connection_uptime_s: float) -> None: ...
|
|
62
|
-
def _should_suppress_reconnect(self) -> bool: ...
|
|
63
|
-
def _compute_reconnect_delay(self, attempt: int) -> float: ...
|
|
64
63
|
def _extract_close_details(
|
|
65
64
|
self, *, connection: ClientConnection, routing_error: Exception | None
|
|
66
65
|
) -> tuple[int | None, str]: ...
|
|
67
|
-
def
|
|
68
|
-
self, close_code: int | None, close_reason: str
|
|
69
|
-
) -> ReconnectDecision: ...
|
|
70
|
-
def _classify_connect_failure(self, error: Exception) -> ReconnectDecision: ...
|
|
71
|
-
def _apply_disconnect_delay_override(
|
|
72
|
-
self, computed_delay_s: float, decision: ReconnectDecision
|
|
73
|
-
) -> float: ...
|
|
66
|
+
def _decide(self, event: ReconnectEvent) -> ReconnectDecision: ...
|
|
74
67
|
def _transition_state(self, new_state: ClientState) -> None: ...
|
|
75
68
|
async def shutdown(self, reason: str) -> None: ...
|
|
76
69
|
|
|
@@ -81,7 +74,6 @@ class SupervisorMixin:
|
|
|
81
74
|
channel_socket_url_redacted: str
|
|
82
75
|
additional_headers: dict[str, str]
|
|
83
76
|
auto_reconnect: bool
|
|
84
|
-
reconnect_policy: ReconnectPolicy
|
|
85
77
|
connection: ClientConnection | None
|
|
86
78
|
_topic_subscriptions: dict[str, TopicSubscription]
|
|
87
79
|
_protocol_handler: PHXProtocolHandler
|
|
@@ -92,7 +84,6 @@ class SupervisorMixin:
|
|
|
92
84
|
_supervisor_task: asyncio.Task[None] | None
|
|
93
85
|
_message_routing_task: asyncio.Task[None] | None
|
|
94
86
|
_initial_connection_future: asyncio.Future[None] | None
|
|
95
|
-
_rapid_disconnects: deque[float]
|
|
96
87
|
_terminal_error: Exception | None
|
|
97
88
|
_heartbeat_dead_threshold_s: float | None
|
|
98
89
|
_heartbeat_deadline: float | None
|
|
@@ -239,42 +230,29 @@ class SupervisorMixin:
|
|
|
239
230
|
self.logger.debug("Sent heartbeat (ref=%s)", ref)
|
|
240
231
|
|
|
241
232
|
async def _supervisor_loop(self) -> None:
|
|
242
|
-
attempt = 0
|
|
243
233
|
try:
|
|
244
234
|
while not self._shutdown_event.is_set():
|
|
235
|
+
if self.auto_reconnect and self._proceeds(Connecting()) is None:
|
|
236
|
+
break
|
|
245
237
|
try:
|
|
246
238
|
connection = await self._connect()
|
|
247
239
|
except Exception as exc: # noqa: BLE001 # each failure is retried
|
|
248
|
-
|
|
240
|
+
decision = self._retry_after_connect_failure(exc)
|
|
241
|
+
if decision is None:
|
|
249
242
|
break
|
|
250
|
-
|
|
251
|
-
attempt += 1
|
|
252
|
-
await self._wait_for_shutdown_or_timeout(delay)
|
|
243
|
+
await self._wait_for_shutdown_or_timeout(decision.delay_s)
|
|
253
244
|
continue
|
|
254
245
|
|
|
255
|
-
if not self._adopt_connection(connection):
|
|
246
|
+
if not await self._adopt_connection(connection):
|
|
256
247
|
break
|
|
257
|
-
connected_since = asyncio.get_running_loop().time()
|
|
258
248
|
routing = await self._start_session(connection)
|
|
259
249
|
routing_error = await self._await_routing(routing)
|
|
260
250
|
close = await self._end_session(connection, routing_error)
|
|
261
251
|
decision = self._decide_reconnect(close)
|
|
262
252
|
if decision is None:
|
|
263
253
|
break
|
|
264
|
-
|
|
265
|
-
uptime = asyncio.get_running_loop().time() - connected_since
|
|
266
|
-
if self._suppressed_after_disconnect(uptime):
|
|
267
|
-
break
|
|
268
|
-
if uptime >= self.reconnect_policy.stable_reset_s:
|
|
269
|
-
attempt = 0
|
|
270
|
-
self._rapid_disconnects.clear()
|
|
271
|
-
else:
|
|
272
|
-
attempt += 1
|
|
273
|
-
|
|
274
254
|
self._deps._transition_state(ClientState.RECONNECTING)
|
|
275
|
-
|
|
276
|
-
delay = self._deps._apply_disconnect_delay_override(delay, decision)
|
|
277
|
-
await self._wait_for_shutdown_or_timeout(delay)
|
|
255
|
+
await self._wait_for_shutdown_or_timeout(decision.delay_s)
|
|
278
256
|
finally:
|
|
279
257
|
self._settle_supervisor_exit()
|
|
280
258
|
|
|
@@ -285,7 +263,8 @@ class SupervisorMixin:
|
|
|
285
263
|
logger=self.logger,
|
|
286
264
|
)
|
|
287
265
|
|
|
288
|
-
def _retry_after_connect_failure(self, exc: Exception) ->
|
|
266
|
+
def _retry_after_connect_failure(self, exc: Exception) -> ReconnectDecision | None:
|
|
267
|
+
"""The decision to retry after ``exc``, or ``None`` to stop."""
|
|
289
268
|
if not self.auto_reconnect:
|
|
290
269
|
self._settle_initial_connection(
|
|
291
270
|
PHXConnectionError(
|
|
@@ -296,35 +275,38 @@ class SupervisorMixin:
|
|
|
296
275
|
)
|
|
297
276
|
)
|
|
298
277
|
self.logger.error("Connection failed and auto_reconnect=False: %s", exc)
|
|
299
|
-
return
|
|
278
|
+
return None
|
|
300
279
|
# A requested stop outranks a refusal's terminal error.
|
|
301
280
|
if self._shutdown_event.is_set():
|
|
302
281
|
self.logger.debug(
|
|
303
282
|
"Ignoring a connect failure after shutdown began: %s", exc
|
|
304
283
|
)
|
|
305
|
-
return
|
|
306
|
-
decision = self.
|
|
307
|
-
if decision
|
|
308
|
-
self.
|
|
309
|
-
|
|
310
|
-
if self._suppressed_after_disconnect(0.0):
|
|
311
|
-
return False
|
|
312
|
-
self._deps._transition_state(ClientState.RECONNECTING)
|
|
313
|
-
return True
|
|
284
|
+
return None
|
|
285
|
+
decision = self._proceeds(ConnectFailed(exc))
|
|
286
|
+
if decision is not None:
|
|
287
|
+
self._deps._transition_state(ClientState.RECONNECTING)
|
|
288
|
+
return decision
|
|
314
289
|
|
|
315
|
-
def _adopt_connection(self, connection: ClientConnection) -> bool:
|
|
290
|
+
async def _adopt_connection(self, connection: ClientConnection) -> bool:
|
|
316
291
|
self.connection = connection
|
|
317
292
|
if self._shutdown_event.is_set():
|
|
318
293
|
# shutdown() began during the handshake; its own cleanup closes
|
|
319
294
|
# this socket.
|
|
320
295
|
return False
|
|
296
|
+
self._conn_generation += 1
|
|
297
|
+
if (
|
|
298
|
+
self.auto_reconnect
|
|
299
|
+
and self._proceeds(Connected(self._conn_generation)) is None
|
|
300
|
+
):
|
|
301
|
+
# Refused before any topic rejoins on it.
|
|
302
|
+
await self._cleanup_connection()
|
|
303
|
+
return False
|
|
321
304
|
self._heartbeat_deadline = (
|
|
322
305
|
asyncio.get_running_loop().time() + self._heartbeat_dead_threshold_s
|
|
323
306
|
if self._heartbeat_interval_s is not None
|
|
324
307
|
and self._heartbeat_dead_threshold_s is not None
|
|
325
308
|
else None
|
|
326
309
|
)
|
|
327
|
-
self._conn_generation += 1
|
|
328
310
|
self._connected_event.set()
|
|
329
311
|
self._deps._transition_state(ClientState.CONNECTED)
|
|
330
312
|
return True
|
|
@@ -392,36 +374,25 @@ class SupervisorMixin:
|
|
|
392
374
|
if self._shutdown_event.is_set() or not self.auto_reconnect:
|
|
393
375
|
return None
|
|
394
376
|
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
377
|
+
# websockets reports the close code it received, not the one we sent,
|
|
378
|
+
# so a forced close carries no code to decide on.
|
|
379
|
+
event = (
|
|
380
|
+
Disconnected(close_code=None, close_reason="")
|
|
381
|
+
if close.forced
|
|
382
|
+
else Disconnected(close_code=close.code, close_reason=close.reason)
|
|
383
|
+
)
|
|
384
|
+
return self._proceeds(event)
|
|
385
|
+
|
|
386
|
+
def _proceeds(self, event: ReconnectEvent) -> ReconnectDecision | None:
|
|
387
|
+
"""The decision to go on after ``event``, or ``None`` when it stops."""
|
|
388
|
+
decision = self._deps._decide(event)
|
|
389
|
+
if decision.should_reconnect:
|
|
390
|
+
return decision
|
|
401
391
|
if decision.terminal_error is not None:
|
|
402
392
|
self._record_terminal_error(decision.terminal_error)
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
self.logger.info(
|
|
407
|
-
"Reconnect disabled for close code %s reason=%s",
|
|
408
|
-
close.code,
|
|
409
|
-
close.reason,
|
|
410
|
-
)
|
|
411
|
-
return None
|
|
412
|
-
return decision
|
|
413
|
-
|
|
414
|
-
def _suppressed_after_disconnect(self, uptime_s: float) -> bool:
|
|
415
|
-
self._deps._record_disconnect(connection_uptime_s=uptime_s)
|
|
416
|
-
if not self._deps._should_suppress_reconnect():
|
|
417
|
-
return False
|
|
418
|
-
self._record_terminal_error(
|
|
419
|
-
PHXConnectionError(
|
|
420
|
-
"Reconnect suppressed after repeated rapid disconnects. "
|
|
421
|
-
"Likely duplicate connection or unstable endpoint."
|
|
422
|
-
)
|
|
423
|
-
)
|
|
424
|
-
return True
|
|
393
|
+
else:
|
|
394
|
+
self.logger.info("Reconnect stopped by the reconnect policy")
|
|
395
|
+
return None
|
|
425
396
|
|
|
426
397
|
def _record_terminal_error(self, error: PHXConnectionError) -> None:
|
|
427
398
|
self._terminal_error = error
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phoenix-channels-python-client
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.4
|
|
4
4
|
Summary: A Python client library for connecting to Phoenix Channels
|
|
5
5
|
Author: Phoenix Channels Python Client
|
|
6
6
|
License-Expression: MIT
|
|
@@ -307,6 +307,7 @@ Everything after `api_key` is keyword-only:
|
|
|
307
307
|
| `on_disconnect` | `None` | `async (error: Exception \| None) -> None`; `None` on a clean close |
|
|
308
308
|
| `on_heartbeat_ack` | `None` | Synchronous `() -> None`; runs on the message path, so keep it fast |
|
|
309
309
|
| `on_topic_lost` | `None` | `async (topic: str, error: Exception) -> None`, called when the client drops a topic you didn't unsubscribe |
|
|
310
|
+
| `reconnect_decider` | `None` | Synchronous `(event: ReconnectEvent) -> ReconnectDecision`; replaces the built-in reconnect policy (see [Owning the reconnect policy](#owning-the-reconnect-policy)) |
|
|
310
311
|
| `additional_headers` | `None` | Extra handshake headers sent on every (re)connect |
|
|
311
312
|
|
|
312
313
|
Exceptions raised in ordinary message and lifecycle callbacks are logged, not raised.
|
|
@@ -340,6 +341,34 @@ base = ReconnectPolicy()
|
|
|
340
341
|
policy = ReconnectPolicy.model_validate(base.model_dump() | {"max_delay_s": 10.0})
|
|
341
342
|
```
|
|
342
343
|
|
|
344
|
+
### Owning the reconnect policy
|
|
345
|
+
|
|
346
|
+
Pass `reconnect_decider` to make every reconnect decision yourself. While `auto_reconnect` is on, the client calls it with:
|
|
347
|
+
|
|
348
|
+
- `Connecting()` before every connection attempt, the first included;
|
|
349
|
+
- `Connected(generation)` when a socket is up, before any topic rejoins on it;
|
|
350
|
+
- `Disconnected(close_code, close_reason)` after every close. A forced close (heartbeat expiry, `close_connection()`) has `close_code=None`, since the received code is unreliable then;
|
|
351
|
+
- `ConnectFailed(error)` after every failed attempt, with the raw exception (a refused handshake is a `websockets` `InvalidStatus` carrying the response).
|
|
352
|
+
|
|
353
|
+
It returns a `ReconnectDecision`:
|
|
354
|
+
|
|
355
|
+
- `ReconnectDecision(should_reconnect=True, delay_s=2.0)` after `Disconnected` or `ConnectFailed` waits exactly 2 s before the next attempt; `delay_s` defaults to `0`. `Connecting` and `Connected` only proceed or stop, and a delay there stops the client with `PHXConnectionError`.
|
|
356
|
+
- `ReconnectDecision(should_reconnect=False)` stops; `run_forever()` returns `None`, and before the first connection `async with` raises `PHXConnectionError`. A stop at `Connected` closes that socket before any rejoin.
|
|
357
|
+
- `ReconnectDecision(should_reconnect=False, terminal_error=PHXConnectionError(...))` stops; `run_forever()` (or `async with`, before the first connection) raises that error.
|
|
358
|
+
|
|
359
|
+
The decider replaces the whole built-in policy above, rapid-disconnect suppression included; the built-in policy is itself a decider over the same events. It runs inline on the client's supervisor task, so keep it fast and never block. The client calls the same decider on every `async with` entry, so reset any per-run state yourself; the built-in one is rebuilt each time. If it raises, the client stops with a `PHXConnectionError` whose `__cause__` is that exception; if it returns anything but a `ReconnectDecision` (an `async def` decider returns a coroutine), the client stops with a `PHXConnectionError` too. It is not asked once `shutdown()` has begun. To end a live connection, call `close_connection()` and stop at the next decision.
|
|
360
|
+
|
|
361
|
+
```python
|
|
362
|
+
def decide(event: ReconnectEvent) -> ReconnectDecision:
|
|
363
|
+
match event:
|
|
364
|
+
case ConnectFailed(error=InvalidStatus(response=response)) if response.status_code == 403:
|
|
365
|
+
return ReconnectDecision(should_reconnect=False, terminal_error=PHXConnectionError("refused"))
|
|
366
|
+
case Disconnected() | ConnectFailed():
|
|
367
|
+
return ReconnectDecision(should_reconnect=True, delay_s=1.0)
|
|
368
|
+
case _:
|
|
369
|
+
return ReconnectDecision(should_reconnect=True)
|
|
370
|
+
```
|
|
371
|
+
|
|
343
372
|
### Channel recovery
|
|
344
373
|
|
|
345
374
|
A channel can fail while the socket stays up. When its process crashes, the server sends `phx_error`, and the client rejoins that topic on the same socket, waiting the `ReconnectPolicy` backoff (`base_delay_s`, `factor`, `max_delay_s`, with equal jitter) before each attempt. A rejoin that times out is retried the same way, including one after a socket reconnect. A drop of the socket hands the topic to the socket's own rejoin.
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
phoenix_channels_python_client/__init__.py,sha256=
|
|
1
|
+
phoenix_channels_python_client/__init__.py,sha256=sskkEUbwcMux5BkaBeH6X7CZlLBC_VoIW1QYRSlL-5E,1233
|
|
2
2
|
phoenix_channels_python_client/authentication.py,sha256=r_UtblJDKr0ynUyxf_WDoMdErDcqvKD8y_iip-PLR-0,1353
|
|
3
|
-
phoenix_channels_python_client/client.py,sha256=
|
|
3
|
+
phoenix_channels_python_client/client.py,sha256=IlF_vF399lNkPZMTl7VL67glJ7NCLJuWNv3pvCXPtv8,12644
|
|
4
4
|
phoenix_channels_python_client/client_state_machine.py,sha256=pjIHK55C4MZII_xnEOZXfiJ6drNrLcC0EPtWXRRGUuY,1090
|
|
5
|
-
phoenix_channels_python_client/client_types.py,sha256=
|
|
5
|
+
phoenix_channels_python_client/client_types.py,sha256=yvayfL9ynDsx1kXG976ZFR8pWY64SFYYJUbC550x08U,4304
|
|
6
|
+
phoenix_channels_python_client/default_decider.py,sha256=R_EuLtatl19rqNfkGB0N0bnSA_oTSGaxm2f894fsKd0,8191
|
|
6
7
|
phoenix_channels_python_client/diagnostics.py,sha256=vaf3pDIZvRTNh1QZCVgPKPAlQaonl9of1JQ5Ow7Cvf8,3987
|
|
7
8
|
phoenix_channels_python_client/exceptions.py,sha256=_fS9Zfn751WmykTnDCcJjGpVCRrUqgqPUpTI3B-xzF0,217
|
|
8
9
|
phoenix_channels_python_client/phx_messages.py,sha256=F_uW79oNaJm6Mpm9wtSrLIJL0KVPQMRA7E5l0-S6j0Q,1614
|
|
9
10
|
phoenix_channels_python_client/protocol_handler.py,sha256=J5kHCcFVltwQH-4PJZxsQA0NTQE7jqjvZ8jmP5tWN54,8967
|
|
10
|
-
phoenix_channels_python_client/reconnect_controller.py,sha256=
|
|
11
|
+
phoenix_channels_python_client/reconnect_controller.py,sha256=a1ER5D-AKviY3fEJiIzUJ_whE5Y-eUugfnLeDtP6xGQ,3149
|
|
11
12
|
phoenix_channels_python_client/shutdown_signals.py,sha256=qbcfJnp7pBcu07WolZyJyqNarFaUb0f7GaasqZGvi9E,4067
|
|
12
|
-
phoenix_channels_python_client/supervisor.py,sha256=
|
|
13
|
+
phoenix_channels_python_client/supervisor.py,sha256=TzdQq8K6PLja1soBU1esDHVR3kMLX190ZFCPCuoSnZA,20742
|
|
13
14
|
phoenix_channels_python_client/topic_runtime.py,sha256=zSccUxUZ1UVcawHR--OTNaXCR64Wn0WQhXuhyNnoizo,36414
|
|
14
15
|
phoenix_channels_python_client/topic_subscription.py,sha256=CMlBbXB6fLiFe3XQYmrrvrKVQ6T4YVmSSieL-oYFLY8,2783
|
|
15
16
|
phoenix_channels_python_client/utils.py,sha256=YFo2N2h9UJ9vcSYUCGfbElsEVgyc3vM2103t8bZdEd8,2204
|
|
16
|
-
phoenix_channels_python_client-0.4.
|
|
17
|
-
phoenix_channels_python_client-0.4.
|
|
18
|
-
phoenix_channels_python_client-0.4.
|
|
19
|
-
phoenix_channels_python_client-0.4.
|
|
20
|
-
phoenix_channels_python_client-0.4.
|
|
17
|
+
phoenix_channels_python_client-0.4.4.dist-info/licenses/LICENSE,sha256=zwqkIClV_fMWOC0Auvd0DYqZMLL0-hIqeKd6tygE39g,1064
|
|
18
|
+
phoenix_channels_python_client-0.4.4.dist-info/METADATA,sha256=NKxcBucgeI2bI-dsS2oeKEZ7Anb4lUOclhXrKYlSD4s,25238
|
|
19
|
+
phoenix_channels_python_client-0.4.4.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
20
|
+
phoenix_channels_python_client-0.4.4.dist-info/top_level.txt,sha256=ZYXMGCOeev4cyNU9XvNGHKt1wOgVkFHaS9TlwDa4eDU,31
|
|
21
|
+
phoenix_channels_python_client-0.4.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|