phoenix-channels-python-client 0.4.2__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {phoenix_channels_python_client-0.4.2/phoenix_channels_python_client.egg-info → phoenix_channels_python_client-0.4.3}/PKG-INFO +30 -1
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/README.md +29 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/__init__.py +18 -2
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/client.py +10 -5
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/client_types.py +56 -3
- phoenix_channels_python_client-0.4.3/phoenix_channels_python_client/default_decider.py +228 -0
- phoenix_channels_python_client-0.4.3/phoenix_channels_python_client/reconnect_controller.py +79 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/supervisor.py +49 -78
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3/phoenix_channels_python_client.egg-info}/PKG-INFO +30 -1
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client.egg-info/SOURCES.txt +2 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/pyproject.toml +1 -1
- phoenix_channels_python_client-0.4.3/tests/test_reconnect_decider.py +506 -0
- phoenix_channels_python_client-0.4.3/tests/test_reconnect_policy_invariants.py +319 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_shutdown_races.py +1 -1
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_supervisor_internals.py +8 -31
- phoenix_channels_python_client-0.4.2/phoenix_channels_python_client/reconnect_controller.py +0 -341
- phoenix_channels_python_client-0.4.2/tests/test_reconnect_policy_invariants.py +0 -228
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/LICENSE +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/MANIFEST.in +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/authentication.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/client_state_machine.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/diagnostics.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/exceptions.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/phx_messages.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/protocol_handler.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/shutdown_signals.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/topic_runtime.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/topic_subscription.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client/utils.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client.egg-info/dependency_links.txt +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client.egg-info/requires.txt +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/phoenix_channels_python_client.egg-info/top_level.txt +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/setup.cfg +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_cancel_and_wait.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_channel_recovery.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_cleanup_ownership.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_client_state.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_connection.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_heartbeat.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_heartbeat_timeout.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_join_installation.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_join_lifecycle.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_join_responses.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_optional_lifecycle.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_optional_reconnect.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_protocol_handler.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_reconnect.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_reconnect_stress.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_run_forever_signals.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_security.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_subscriptions.py +0 -0
- {phoenix_channels_python_client-0.4.2 → phoenix_channels_python_client-0.4.3}/tests/test_topic_runtime_internals.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phoenix-channels-python-client
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.3
|
|
4
4
|
Summary: A Python client library for connecting to Phoenix Channels
|
|
5
5
|
Author: Phoenix Channels Python Client
|
|
6
6
|
License-Expression: MIT
|
|
@@ -307,6 +307,7 @@ Everything after `api_key` is keyword-only:
|
|
|
307
307
|
| `on_disconnect` | `None` | `async (error: Exception \| None) -> None`; `None` on a clean close |
|
|
308
308
|
| `on_heartbeat_ack` | `None` | Synchronous `() -> None`; runs on the message path, so keep it fast |
|
|
309
309
|
| `on_topic_lost` | `None` | `async (topic: str, error: Exception) -> None`, called when the client drops a topic you didn't unsubscribe |
|
|
310
|
+
| `reconnect_decider` | `None` | Synchronous `(event: ReconnectEvent) -> ReconnectDecision`; replaces the built-in reconnect policy (see [Owning the reconnect policy](#owning-the-reconnect-policy)) |
|
|
310
311
|
| `additional_headers` | `None` | Extra handshake headers sent on every (re)connect |
|
|
311
312
|
|
|
312
313
|
Exceptions raised in ordinary message and lifecycle callbacks are logged, not raised.
|
|
@@ -340,6 +341,34 @@ base = ReconnectPolicy()
|
|
|
340
341
|
policy = ReconnectPolicy.model_validate(base.model_dump() | {"max_delay_s": 10.0})
|
|
341
342
|
```
|
|
342
343
|
|
|
344
|
+
### Owning the reconnect policy
|
|
345
|
+
|
|
346
|
+
Pass `reconnect_decider` to make every reconnect decision yourself. While `auto_reconnect` is on, the client calls it with:
|
|
347
|
+
|
|
348
|
+
- `Connecting()` before every connection attempt, the first included;
|
|
349
|
+
- `Connected(generation)` when a socket is up, before any topic rejoins on it;
|
|
350
|
+
- `Disconnected(close_code, close_reason)` after every close. A forced close (heartbeat expiry, `close_connection()`) has `close_code=None`, since the received code is unreliable then;
|
|
351
|
+
- `ConnectFailed(error)` after every failed attempt, with the raw exception (a refused handshake is a `websockets` `InvalidStatus` carrying the response).
|
|
352
|
+
|
|
353
|
+
It returns a `ReconnectDecision`:
|
|
354
|
+
|
|
355
|
+
- `ReconnectDecision(should_reconnect=True, delay_s=2.0)` after `Disconnected` or `ConnectFailed` waits exactly 2 s before the next attempt; `delay_s` defaults to `0`. `Connecting` and `Connected` only proceed or stop, and a delay there stops the client with `PHXConnectionError`.
|
|
356
|
+
- `ReconnectDecision(should_reconnect=False)` stops; `run_forever()` returns `None`, and before the first connection `async with` raises `PHXConnectionError`. A stop at `Connected` closes that socket before any rejoin.
|
|
357
|
+
- `ReconnectDecision(should_reconnect=False, terminal_error=PHXConnectionError(...))` stops; `run_forever()` (or `async with`, before the first connection) raises that error.
|
|
358
|
+
|
|
359
|
+
The decider replaces the whole built-in policy above, rapid-disconnect suppression included; the built-in policy is itself a decider over the same events. It runs inline on the client's supervisor task, so keep it fast and never block. The client calls the same decider on every `async with` entry, so reset any per-run state yourself; the built-in one is rebuilt each time. If it raises, the client stops with a `PHXConnectionError` whose `__cause__` is that exception; if it returns anything but a `ReconnectDecision` (an `async def` decider returns a coroutine), the client stops with a `PHXConnectionError` too. It is not asked once `shutdown()` has begun. To end a live connection, call `close_connection()` and stop at the next decision.
|
|
360
|
+
|
|
361
|
+
```python
|
|
362
|
+
def decide(event: ReconnectEvent) -> ReconnectDecision:
|
|
363
|
+
match event:
|
|
364
|
+
case ConnectFailed(error=InvalidStatus(response=response)) if response.status_code == 403:
|
|
365
|
+
return ReconnectDecision(should_reconnect=False, terminal_error=PHXConnectionError("refused"))
|
|
366
|
+
case Disconnected() | ConnectFailed():
|
|
367
|
+
return ReconnectDecision(should_reconnect=True, delay_s=1.0)
|
|
368
|
+
case _:
|
|
369
|
+
return ReconnectDecision(should_reconnect=True)
|
|
370
|
+
```
|
|
371
|
+
|
|
343
372
|
### Channel recovery
|
|
344
373
|
|
|
345
374
|
A channel can fail while the socket stays up. When its process crashes, the server sends `phx_error`, and the client rejoins that topic on the same socket, waiting the `ReconnectPolicy` backoff (`base_delay_s`, `factor`, `max_delay_s`, with equal jitter) before each attempt. A rejoin that times out is retried the same way, including one after a socket reconnect. A drop of the socket hands the topic to the socket's own rejoin.
|
|
@@ -274,6 +274,7 @@ Everything after `api_key` is keyword-only:
|
|
|
274
274
|
| `on_disconnect` | `None` | `async (error: Exception \| None) -> None`; `None` on a clean close |
|
|
275
275
|
| `on_heartbeat_ack` | `None` | Synchronous `() -> None`; runs on the message path, so keep it fast |
|
|
276
276
|
| `on_topic_lost` | `None` | `async (topic: str, error: Exception) -> None`, called when the client drops a topic you didn't unsubscribe |
|
|
277
|
+
| `reconnect_decider` | `None` | Synchronous `(event: ReconnectEvent) -> ReconnectDecision`; replaces the built-in reconnect policy (see [Owning the reconnect policy](#owning-the-reconnect-policy)) |
|
|
277
278
|
| `additional_headers` | `None` | Extra handshake headers sent on every (re)connect |
|
|
278
279
|
|
|
279
280
|
Exceptions raised in ordinary message and lifecycle callbacks are logged, not raised.
|
|
@@ -307,6 +308,34 @@ base = ReconnectPolicy()
|
|
|
307
308
|
policy = ReconnectPolicy.model_validate(base.model_dump() | {"max_delay_s": 10.0})
|
|
308
309
|
```
|
|
309
310
|
|
|
311
|
+
### Owning the reconnect policy
|
|
312
|
+
|
|
313
|
+
Pass `reconnect_decider` to make every reconnect decision yourself. While `auto_reconnect` is on, the client calls it with:
|
|
314
|
+
|
|
315
|
+
- `Connecting()` before every connection attempt, the first included;
|
|
316
|
+
- `Connected(generation)` when a socket is up, before any topic rejoins on it;
|
|
317
|
+
- `Disconnected(close_code, close_reason)` after every close. A forced close (heartbeat expiry, `close_connection()`) has `close_code=None`, since the received code is unreliable then;
|
|
318
|
+
- `ConnectFailed(error)` after every failed attempt, with the raw exception (a refused handshake is a `websockets` `InvalidStatus` carrying the response).
|
|
319
|
+
|
|
320
|
+
It returns a `ReconnectDecision`:
|
|
321
|
+
|
|
322
|
+
- `ReconnectDecision(should_reconnect=True, delay_s=2.0)` after `Disconnected` or `ConnectFailed` waits exactly 2 s before the next attempt; `delay_s` defaults to `0`. `Connecting` and `Connected` only proceed or stop, and a delay there stops the client with `PHXConnectionError`.
|
|
323
|
+
- `ReconnectDecision(should_reconnect=False)` stops; `run_forever()` returns `None`, and before the first connection `async with` raises `PHXConnectionError`. A stop at `Connected` closes that socket before any rejoin.
|
|
324
|
+
- `ReconnectDecision(should_reconnect=False, terminal_error=PHXConnectionError(...))` stops; `run_forever()` (or `async with`, before the first connection) raises that error.
|
|
325
|
+
|
|
326
|
+
The decider replaces the whole built-in policy above, rapid-disconnect suppression included; the built-in policy is itself a decider over the same events. It runs inline on the client's supervisor task, so keep it fast and never block. The client calls the same decider on every `async with` entry, so reset any per-run state yourself; the built-in one is rebuilt each time. If it raises, the client stops with a `PHXConnectionError` whose `__cause__` is that exception; if it returns anything but a `ReconnectDecision` (an `async def` decider returns a coroutine), the client stops with a `PHXConnectionError` too. It is not asked once `shutdown()` has begun. To end a live connection, call `close_connection()` and stop at the next decision.
|
|
327
|
+
|
|
328
|
+
```python
|
|
329
|
+
def decide(event: ReconnectEvent) -> ReconnectDecision:
|
|
330
|
+
match event:
|
|
331
|
+
case ConnectFailed(error=InvalidStatus(response=response)) if response.status_code == 403:
|
|
332
|
+
return ReconnectDecision(should_reconnect=False, terminal_error=PHXConnectionError("refused"))
|
|
333
|
+
case Disconnected() | ConnectFailed():
|
|
334
|
+
return ReconnectDecision(should_reconnect=True, delay_s=1.0)
|
|
335
|
+
case _:
|
|
336
|
+
return ReconnectDecision(should_reconnect=True)
|
|
337
|
+
```
|
|
338
|
+
|
|
310
339
|
### Channel recovery
|
|
311
340
|
|
|
312
341
|
A channel can fail while the socket stays up. When its process crashes, the server sends `phx_error`, and the client rejoins that topic on the same socket, waiting the `ReconnectPolicy` backoff (`base_delay_s`, `factor`, `max_delay_s`, with equal jitter) before each attempt. A rejoin that times out is retried the same way, including one after a socket reconnect. A drop of the socket hands the topic to the socket's own rejoin.
|
|
@@ -13,24 +13,40 @@ from phoenix_channels_python_client.client import (
|
|
|
13
13
|
ReconnectPolicy,
|
|
14
14
|
TopicLostCallback,
|
|
15
15
|
)
|
|
16
|
-
from phoenix_channels_python_client.client_types import
|
|
16
|
+
from phoenix_channels_python_client.client_types import (
|
|
17
|
+
Connected,
|
|
18
|
+
ConnectFailed,
|
|
19
|
+
Connecting,
|
|
20
|
+
Disconnected,
|
|
21
|
+
JoinResponseCallback,
|
|
22
|
+
ReconnectDecider,
|
|
23
|
+
ReconnectDecision,
|
|
24
|
+
ReconnectEvent,
|
|
25
|
+
)
|
|
17
26
|
from phoenix_channels_python_client.protocol_handler import (
|
|
18
27
|
PhoenixChannelsProtocolVersion,
|
|
19
28
|
PHXProtocolHandler,
|
|
20
29
|
)
|
|
21
30
|
from phoenix_channels_python_client.utils import setup_logging
|
|
22
31
|
|
|
23
|
-
__version__ = "0.4.
|
|
32
|
+
__version__ = "0.4.3"
|
|
24
33
|
__author__ = "Phoenix Channels Python Client"
|
|
25
34
|
|
|
26
35
|
__all__ = [
|
|
36
|
+
"ConnectFailed",
|
|
37
|
+
"Connected",
|
|
38
|
+
"Connecting",
|
|
27
39
|
"DisconnectCallback",
|
|
40
|
+
"Disconnected",
|
|
28
41
|
"HeartbeatAckCallback",
|
|
29
42
|
"JoinResponseCallback",
|
|
30
43
|
"PHXChannelsClient",
|
|
31
44
|
"PHXProtocolHandler",
|
|
32
45
|
"PhoenixChannelsProtocolVersion",
|
|
33
46
|
"ReconnectCallback",
|
|
47
|
+
"ReconnectDecider",
|
|
48
|
+
"ReconnectDecision",
|
|
49
|
+
"ReconnectEvent",
|
|
34
50
|
"ReconnectPolicy",
|
|
35
51
|
"TopicLostCallback",
|
|
36
52
|
"setup_logging",
|
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import logging
|
|
5
5
|
import math
|
|
6
|
-
from collections import deque
|
|
7
6
|
from collections.abc import Awaitable, Callable
|
|
8
7
|
from types import TracebackType
|
|
9
8
|
from typing import Self, cast
|
|
@@ -19,7 +18,11 @@ from phoenix_channels_python_client.authentication import (
|
|
|
19
18
|
select_authentication_header,
|
|
20
19
|
)
|
|
21
20
|
from phoenix_channels_python_client.client_state_machine import transition_client_state
|
|
22
|
-
from phoenix_channels_python_client.client_types import
|
|
21
|
+
from phoenix_channels_python_client.client_types import (
|
|
22
|
+
ClientState,
|
|
23
|
+
ReconnectDecider,
|
|
24
|
+
ReconnectPolicy,
|
|
25
|
+
)
|
|
23
26
|
from phoenix_channels_python_client.diagnostics import (
|
|
24
27
|
ConnectionLogger,
|
|
25
28
|
diagnostic_secrets,
|
|
@@ -114,7 +117,7 @@ def _validate_heartbeat_threshold(
|
|
|
114
117
|
|
|
115
118
|
|
|
116
119
|
class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerMixin):
|
|
117
|
-
def __init__( # noqa: PLR0913 # the public options
|
|
120
|
+
def __init__( # noqa: PLR0913, PLR0915 # the public options
|
|
118
121
|
self,
|
|
119
122
|
websocket_url: str,
|
|
120
123
|
api_key: str,
|
|
@@ -133,6 +136,7 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
133
136
|
on_disconnect: DisconnectCallback | None = None,
|
|
134
137
|
on_heartbeat_ack: HeartbeatAckCallback | None = None,
|
|
135
138
|
on_topic_lost: TopicLostCallback | None = None,
|
|
139
|
+
reconnect_decider: ReconnectDecider | None = None,
|
|
136
140
|
additional_headers: dict[str, str] | None = None,
|
|
137
141
|
) -> None:
|
|
138
142
|
self.additional_headers = normalize_headers(additional_headers)
|
|
@@ -185,7 +189,6 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
185
189
|
self._shutdown_task: asyncio.Task[None] | None = None
|
|
186
190
|
self._message_routing_task: asyncio.Task[None] | None = None
|
|
187
191
|
self._initial_connection_future: asyncio.Future[None] | None = None
|
|
188
|
-
self._rapid_disconnects: deque[float] = deque()
|
|
189
192
|
self._terminal_error: Exception | None = None
|
|
190
193
|
self._heartbeat_dead_threshold_s = heartbeat_dead_threshold_s
|
|
191
194
|
self._heartbeat_deadline: float | None = None
|
|
@@ -197,6 +200,8 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
197
200
|
self._on_disconnect = on_disconnect
|
|
198
201
|
self._on_heartbeat_ack = on_heartbeat_ack
|
|
199
202
|
self._on_topic_lost = on_topic_lost
|
|
203
|
+
self._reconnect_decider = reconnect_decider
|
|
204
|
+
self._decider = self._new_decider()
|
|
200
205
|
self._lifecycle_tasks: set[asyncio.Task[None]] = set()
|
|
201
206
|
self._cleanup_tasks: set[asyncio.Task[None]] = set()
|
|
202
207
|
self._forced_close_pending = False
|
|
@@ -226,7 +231,7 @@ class PHXChannelsClient(SupervisorMixin, TopicRuntimeMixin, ReconnectControllerM
|
|
|
226
231
|
|
|
227
232
|
self._shutdown_event.clear()
|
|
228
233
|
self._connected_event.clear()
|
|
229
|
-
self.
|
|
234
|
+
self._decider = self._new_decider()
|
|
230
235
|
self._terminal_error = None
|
|
231
236
|
self._conn_generation = 0
|
|
232
237
|
self._initial_connection_future = asyncio.get_running_loop().create_future()
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import math
|
|
3
4
|
from collections.abc import Callable
|
|
4
5
|
from dataclasses import dataclass
|
|
5
6
|
from enum import StrEnum
|
|
@@ -71,9 +72,61 @@ class ReconnectPolicy(BaseModel):
|
|
|
71
72
|
return self
|
|
72
73
|
|
|
73
74
|
|
|
74
|
-
@dataclass(frozen=True)
|
|
75
|
+
@dataclass(frozen=True, kw_only=True)
|
|
75
76
|
class ReconnectDecision:
|
|
77
|
+
"""Whether the client goes on, and how long it waits before the next attempt.
|
|
78
|
+
|
|
79
|
+
``delay_s`` applies after ``Disconnected`` and ``ConnectFailed``; a gate
|
|
80
|
+
(``Connecting``, ``Connected``) only proceeds or stops. A stop with a
|
|
81
|
+
``terminal_error`` ends the client with that error.
|
|
82
|
+
"""
|
|
83
|
+
|
|
76
84
|
should_reconnect: bool
|
|
77
|
-
|
|
78
|
-
max_delay_s: float | None = None
|
|
85
|
+
delay_s: float = 0.0
|
|
79
86
|
terminal_error: PHXConnectionError | None = None
|
|
87
|
+
|
|
88
|
+
def __post_init__(self) -> None:
|
|
89
|
+
if not math.isfinite(self.delay_s) or self.delay_s < 0:
|
|
90
|
+
raise ValueError("delay_s must be finite and >= 0")
|
|
91
|
+
if not self.should_reconnect and self.delay_s:
|
|
92
|
+
raise ValueError("a stop has no delay_s")
|
|
93
|
+
if self.should_reconnect and self.terminal_error is not None:
|
|
94
|
+
raise ValueError("a terminal_error requires should_reconnect=False")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
PROCEED = ReconnectDecision(should_reconnect=True)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class Connecting:
|
|
102
|
+
"""A connection attempt is about to start."""
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass(frozen=True)
|
|
106
|
+
class Connected:
|
|
107
|
+
"""The socket for ``generation`` is up; topics rejoin only if this proceeds."""
|
|
108
|
+
|
|
109
|
+
generation: int
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(frozen=True)
|
|
113
|
+
class Disconnected:
|
|
114
|
+
"""The connection closed. ``close_code`` is ``None`` for a forced close."""
|
|
115
|
+
|
|
116
|
+
close_code: int | None
|
|
117
|
+
close_reason: str
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(frozen=True)
|
|
121
|
+
class ConnectFailed:
|
|
122
|
+
"""A connection attempt failed with ``error``, e.g. a refused handshake."""
|
|
123
|
+
|
|
124
|
+
error: Exception
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
ReconnectEvent = Connecting | Connected | Disconnected | ConnectFailed
|
|
128
|
+
|
|
129
|
+
# Decision points that only proceed or stop; no attempt follows them to delay.
|
|
130
|
+
GATE_EVENTS = (Connecting, Connected)
|
|
131
|
+
|
|
132
|
+
ReconnectDecider = Callable[[ReconnectEvent], ReconnectDecision]
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The built-in reconnect policy, as a ``reconnect_decider``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import random
|
|
8
|
+
from collections import deque
|
|
9
|
+
from http import HTTPStatus
|
|
10
|
+
|
|
11
|
+
from websockets.exceptions import InvalidStatus
|
|
12
|
+
from websockets.frames import CloseCode
|
|
13
|
+
|
|
14
|
+
from phoenix_channels_python_client.client_types import (
|
|
15
|
+
PROCEED,
|
|
16
|
+
Connected,
|
|
17
|
+
ConnectFailed,
|
|
18
|
+
Disconnected,
|
|
19
|
+
ReconnectDecision,
|
|
20
|
+
ReconnectEvent,
|
|
21
|
+
ReconnectPolicy,
|
|
22
|
+
)
|
|
23
|
+
from phoenix_channels_python_client.exceptions import PHXConnectionError
|
|
24
|
+
|
|
25
|
+
# The Band platform answers every auth refusal with 403, and its websocket spec
|
|
26
|
+
# (thenvoi-platform spec/websocket-api.draft.md) makes a rejected reconnect terminal.
|
|
27
|
+
# A 401 could only come from an edge layer, so it and every other status retry.
|
|
28
|
+
TERMINAL_HANDSHAKE_STATUS = HTTPStatus.FORBIDDEN
|
|
29
|
+
|
|
30
|
+
# A (low, high) range a close's backoff is raised into.
|
|
31
|
+
DelayRange = tuple[float, float]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def random_between(min_delay_s: float, max_delay_s: float) -> float:
|
|
35
|
+
low = max(0.0, min_delay_s)
|
|
36
|
+
high = max(low, max_delay_s)
|
|
37
|
+
if high == low:
|
|
38
|
+
return low
|
|
39
|
+
return low + (random.random() * (high - low)) # noqa: S311 # jitter
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def exponential_delay(policy: ReconnectPolicy, attempt: int) -> float:
|
|
43
|
+
"""The policy's capped exponential backoff for ``attempt``, unjittered."""
|
|
44
|
+
return min(
|
|
45
|
+
policy.max_delay_s,
|
|
46
|
+
policy.base_delay_s * (policy.factor ** max(attempt, 0)),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def equal_jitter(delay: float) -> float:
|
|
51
|
+
# Equal jitter avoids synchronization while keeping a meaningful minimum delay.
|
|
52
|
+
return random_between(delay / 2, delay)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _now() -> float:
|
|
56
|
+
return asyncio.get_running_loop().time()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _terminal_decision(cause: str) -> ReconnectDecision:
|
|
60
|
+
return ReconnectDecision(
|
|
61
|
+
should_reconnect=False,
|
|
62
|
+
terminal_error=PHXConnectionError(
|
|
63
|
+
f"Reconnect disabled due to terminal {cause}"
|
|
64
|
+
),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class DefaultReconnectDecider:
|
|
69
|
+
"""Classifies closes and refusals, backs off, and stops on rapid churn.
|
|
70
|
+
|
|
71
|
+
Holds one run's backoff attempt and rapid-disconnect history, so the client
|
|
72
|
+
builds a fresh one each time it is entered.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(self, policy: ReconnectPolicy, logger: logging.Logger) -> None:
|
|
76
|
+
self._policy = policy
|
|
77
|
+
self._logger = logger
|
|
78
|
+
self._attempt = 0
|
|
79
|
+
self._rapid_disconnects: deque[float] = deque()
|
|
80
|
+
self._connected_at = 0.0
|
|
81
|
+
|
|
82
|
+
def __call__(self, event: ReconnectEvent) -> ReconnectDecision:
|
|
83
|
+
match event:
|
|
84
|
+
case Connected():
|
|
85
|
+
self._connected_at = _now()
|
|
86
|
+
return PROCEED
|
|
87
|
+
case Disconnected():
|
|
88
|
+
return self._after_close(event.close_code, event.close_reason)
|
|
89
|
+
case ConnectFailed():
|
|
90
|
+
return self._after_connect_failure(event.error)
|
|
91
|
+
case _: # Connecting: always attempt.
|
|
92
|
+
return PROCEED
|
|
93
|
+
|
|
94
|
+
def _after_close(
|
|
95
|
+
self, close_code: int | None, close_reason: str
|
|
96
|
+
) -> ReconnectDecision:
|
|
97
|
+
stop = self._stop_for_close(close_code, close_reason)
|
|
98
|
+
if stop is not None:
|
|
99
|
+
return stop
|
|
100
|
+
uptime_s = _now() - self._connected_at
|
|
101
|
+
if self._suppressed(uptime_s):
|
|
102
|
+
return self._suppression()
|
|
103
|
+
if uptime_s >= self._policy.stable_reset_s:
|
|
104
|
+
self._attempt = 0
|
|
105
|
+
self._rapid_disconnects.clear()
|
|
106
|
+
else:
|
|
107
|
+
self._attempt += 1
|
|
108
|
+
delay = self._backoff(self._attempt)
|
|
109
|
+
delay_range = self._delay_range_for_close(close_code)
|
|
110
|
+
if delay_range is not None:
|
|
111
|
+
delay = max(delay, random_between(*delay_range))
|
|
112
|
+
return ReconnectDecision(should_reconnect=True, delay_s=delay)
|
|
113
|
+
|
|
114
|
+
def _after_connect_failure(self, error: Exception) -> ReconnectDecision:
|
|
115
|
+
if (
|
|
116
|
+
isinstance(error, InvalidStatus)
|
|
117
|
+
and error.response.status_code == TERMINAL_HANDSHAKE_STATUS
|
|
118
|
+
):
|
|
119
|
+
return _terminal_decision(
|
|
120
|
+
f"handshake status {TERMINAL_HANDSHAKE_STATUS} "
|
|
121
|
+
f"({TERMINAL_HANDSHAKE_STATUS.phrase})"
|
|
122
|
+
)
|
|
123
|
+
if self._suppressed(0.0):
|
|
124
|
+
return self._suppression()
|
|
125
|
+
delay = self._backoff(self._attempt)
|
|
126
|
+
self._attempt += 1
|
|
127
|
+
return ReconnectDecision(should_reconnect=True, delay_s=delay)
|
|
128
|
+
|
|
129
|
+
def _stop_for_close(
|
|
130
|
+
self, close_code: int | None, close_reason: str
|
|
131
|
+
) -> ReconnectDecision | None:
|
|
132
|
+
if (
|
|
133
|
+
close_code in {CloseCode.NORMAL_CLOSURE, CloseCode.GOING_AWAY}
|
|
134
|
+
and not self._policy.reconnect_on_normal_close
|
|
135
|
+
):
|
|
136
|
+
return ReconnectDecision(should_reconnect=False)
|
|
137
|
+
if (
|
|
138
|
+
close_code == CloseCode.POLICY_VIOLATION
|
|
139
|
+
and self._policy.policy_violation_is_terminal
|
|
140
|
+
):
|
|
141
|
+
reason = close_reason or "policy violation"
|
|
142
|
+
return _terminal_decision(
|
|
143
|
+
f"close code {CloseCode.POLICY_VIOLATION} ({reason})"
|
|
144
|
+
)
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
def _delay_range_for_close(self, close_code: int | None) -> DelayRange | None:
|
|
148
|
+
policy = self._policy
|
|
149
|
+
match close_code:
|
|
150
|
+
case CloseCode.SERVICE_RESTART:
|
|
151
|
+
return (
|
|
152
|
+
policy.service_restart_min_delay_s,
|
|
153
|
+
policy.service_restart_max_delay_s,
|
|
154
|
+
)
|
|
155
|
+
case CloseCode.TRY_AGAIN_LATER:
|
|
156
|
+
return (
|
|
157
|
+
policy.try_again_later_min_delay_s,
|
|
158
|
+
policy.try_again_later_max_delay_s,
|
|
159
|
+
)
|
|
160
|
+
case _:
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
def _suppressed(self, uptime_s: float) -> bool:
|
|
164
|
+
"""Record a disconnect after ``uptime_s``; whether churn now stops us."""
|
|
165
|
+
now = _now()
|
|
166
|
+
if uptime_s < self._policy.rapid_disconnect_uptime_s:
|
|
167
|
+
self._rapid_disconnects.append(now)
|
|
168
|
+
while (
|
|
169
|
+
self._rapid_disconnects
|
|
170
|
+
and now - self._rapid_disconnects[0] > self._policy.rapid_window_s
|
|
171
|
+
):
|
|
172
|
+
self._rapid_disconnects.popleft()
|
|
173
|
+
threshold = self._policy.rapid_suppress_disconnect_count
|
|
174
|
+
rapid_count = len(self._rapid_disconnects)
|
|
175
|
+
self._logger.debug(
|
|
176
|
+
"Recorded disconnect. uptime_s=%s rapid_count=%s threshold=%s",
|
|
177
|
+
uptime_s,
|
|
178
|
+
rapid_count,
|
|
179
|
+
threshold,
|
|
180
|
+
)
|
|
181
|
+
return threshold > 0 and rapid_count >= threshold
|
|
182
|
+
|
|
183
|
+
def _suppression(self) -> ReconnectDecision:
|
|
184
|
+
return ReconnectDecision(
|
|
185
|
+
should_reconnect=False,
|
|
186
|
+
terminal_error=PHXConnectionError(
|
|
187
|
+
"Reconnect suppressed after repeated rapid disconnects. "
|
|
188
|
+
"Likely duplicate connection or unstable endpoint."
|
|
189
|
+
),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def _rapid_floors(self) -> tuple[float, ...]:
|
|
193
|
+
"""The floor for each rapid-disconnect count before the cooldown starts."""
|
|
194
|
+
return (
|
|
195
|
+
0.0,
|
|
196
|
+
self._policy.rapid_first_min_delay_s,
|
|
197
|
+
self._policy.rapid_second_min_delay_s,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
def _backoff(self, attempt: int) -> float:
|
|
201
|
+
"""The jittered wait for ``attempt``, raised to the rapid-disconnect floor."""
|
|
202
|
+
rapid_count = len(self._rapid_disconnects)
|
|
203
|
+
cooldown_steps = rapid_count - len(self._rapid_floors())
|
|
204
|
+
if cooldown_steps < 0:
|
|
205
|
+
floor = self._rapid_floors()[rapid_count]
|
|
206
|
+
else:
|
|
207
|
+
floor = min(
|
|
208
|
+
self._policy.rapid_cooldown_base_s
|
|
209
|
+
+ (self._policy.rapid_cooldown_step_s * cooldown_steps),
|
|
210
|
+
self._policy.rapid_cooldown_max_s,
|
|
211
|
+
)
|
|
212
|
+
delay = max(exponential_delay(self._policy, attempt), floor)
|
|
213
|
+
if delay <= 0:
|
|
214
|
+
return 0.0
|
|
215
|
+
if cooldown_steps < 0:
|
|
216
|
+
computed = equal_jitter(delay)
|
|
217
|
+
else:
|
|
218
|
+
computed = random_between(
|
|
219
|
+
delay * self._policy.rapid_hold_down_jitter_low_ratio,
|
|
220
|
+
delay * self._policy.rapid_hold_down_jitter_high_ratio,
|
|
221
|
+
)
|
|
222
|
+
self._logger.debug(
|
|
223
|
+
"Computed reconnect delay. attempt=%s rapid_count=%s computed_delay_s=%s",
|
|
224
|
+
attempt,
|
|
225
|
+
rapid_count,
|
|
226
|
+
computed,
|
|
227
|
+
)
|
|
228
|
+
return computed
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
from websockets import ClientConnection
|
|
7
|
+
from websockets.exceptions import ConnectionClosed
|
|
8
|
+
|
|
9
|
+
from phoenix_channels_python_client.client_types import (
|
|
10
|
+
GATE_EVENTS,
|
|
11
|
+
ReconnectDecider,
|
|
12
|
+
ReconnectDecision,
|
|
13
|
+
ReconnectEvent,
|
|
14
|
+
ReconnectPolicy,
|
|
15
|
+
)
|
|
16
|
+
from phoenix_channels_python_client.default_decider import (
|
|
17
|
+
DefaultReconnectDecider,
|
|
18
|
+
equal_jitter,
|
|
19
|
+
exponential_delay,
|
|
20
|
+
)
|
|
21
|
+
from phoenix_channels_python_client.exceptions import PHXConnectionError
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _decider_failure(message: str, cause: Exception | None = None) -> ReconnectDecision:
|
|
25
|
+
"""A stop for a broken decider; never the built-in policy deciding instead."""
|
|
26
|
+
error = PHXConnectionError(message)
|
|
27
|
+
error.__cause__ = cause
|
|
28
|
+
return ReconnectDecision(should_reconnect=False, terminal_error=error)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ReconnectControllerMixin:
|
|
32
|
+
logger: logging.Logger
|
|
33
|
+
|
|
34
|
+
def redact_diagnostic(self, text: str) -> str:
|
|
35
|
+
raise NotImplementedError
|
|
36
|
+
|
|
37
|
+
reconnect_policy: ReconnectPolicy
|
|
38
|
+
_reconnect_decider: ReconnectDecider | None
|
|
39
|
+
_decider: ReconnectDecider
|
|
40
|
+
|
|
41
|
+
def _new_decider(self) -> ReconnectDecider:
|
|
42
|
+
"""The consumer's decider, or a fresh built-in one for this run."""
|
|
43
|
+
if self._reconnect_decider is not None:
|
|
44
|
+
return self._reconnect_decider
|
|
45
|
+
return DefaultReconnectDecider(self.reconnect_policy, self.logger)
|
|
46
|
+
|
|
47
|
+
def _decide(self, event: ReconnectEvent) -> ReconnectDecision:
|
|
48
|
+
"""The decider's answer to ``event``; a broken answer stops the client."""
|
|
49
|
+
try:
|
|
50
|
+
decision = self._decider(event)
|
|
51
|
+
except Exception as exc:
|
|
52
|
+
self.logger.exception("reconnect_decider failed; stopping the client")
|
|
53
|
+
return _decider_failure("reconnect_decider failed", exc)
|
|
54
|
+
if inspect.iscoroutine(decision):
|
|
55
|
+
decision.close() # Never awaited; closed so it raises no warning.
|
|
56
|
+
return _decider_failure("reconnect_decider must be synchronous")
|
|
57
|
+
if not isinstance(decision, ReconnectDecision):
|
|
58
|
+
return _decider_failure("reconnect_decider returned no ReconnectDecision")
|
|
59
|
+
if decision.delay_s and isinstance(event, GATE_EVENTS):
|
|
60
|
+
# No attempt follows a gate, so its delay would be silently dropped.
|
|
61
|
+
return _decider_failure("reconnect_decider returned a delay for a gate")
|
|
62
|
+
return decision
|
|
63
|
+
|
|
64
|
+
def _channel_rejoin_delay(self, attempt: int) -> float:
|
|
65
|
+
# No rapid-disconnect floors: those describe socket churn, not one channel.
|
|
66
|
+
return equal_jitter(exponential_delay(self.reconnect_policy, attempt))
|
|
67
|
+
|
|
68
|
+
def _extract_close_details(
|
|
69
|
+
self,
|
|
70
|
+
*,
|
|
71
|
+
connection: ClientConnection,
|
|
72
|
+
routing_error: Exception | None,
|
|
73
|
+
) -> tuple[int | None, str]:
|
|
74
|
+
if isinstance(routing_error, ConnectionClosed):
|
|
75
|
+
if routing_error.rcvd is not None:
|
|
76
|
+
return routing_error.rcvd.code, routing_error.rcvd.reason
|
|
77
|
+
if routing_error.sent is not None:
|
|
78
|
+
return routing_error.sent.code, routing_error.sent.reason
|
|
79
|
+
return connection.close_code, connection.close_reason or ""
|