semiont 0.6.12.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- semiont/__init__.py +20 -0
- semiont/bus.py +236 -0
- semiont/cache.py +345 -0
- semiont/cached.py +152 -0
- semiont/channel.py +81 -0
- semiont/channels.py +1306 -0
- semiont/client.py +161 -0
- semiont/error_codes.py +232 -0
- semiont/errors.py +147 -0
- semiont/event_bus.py +108 -0
- semiont/events.py +101 -0
- semiont/http/__init__.py +24 -0
- semiont/http/agent.py +232 -0
- semiont/http/content.py +234 -0
- semiont/http/exchange.py +200 -0
- semiont/http/oauth.py +346 -0
- semiont/http/session.py +191 -0
- semiont/http/sse.py +71 -0
- semiont/http/stream.py +683 -0
- semiont/http/transport.py +326 -0
- semiont/identifiers.py +114 -0
- semiont/identity.py +59 -0
- semiont/media_types.py +69 -0
- semiont/media_types_table.py +89 -0
- semiont/model.py +33 -0
- semiont/namespaces/__init__.py +19 -0
- semiont/namespaces/auth.py +32 -0
- semiont/namespaces/beckon.py +47 -0
- semiont/namespaces/bind.py +38 -0
- semiont/namespaces/browse.py +338 -0
- semiont/namespaces/follow.py +238 -0
- semiont/namespaces/frame.py +35 -0
- semiont/namespaces/gather.py +82 -0
- semiont/namespaces/job.py +94 -0
- semiont/namespaces/links.py +66 -0
- semiont/namespaces/live.py +301 -0
- semiont/namespaces/mark.py +174 -0
- semiont/namespaces/match.py +67 -0
- semiont/namespaces/refresher.py +283 -0
- semiont/namespaces/system.py +24 -0
- semiont/namespaces/yield_.py +101 -0
- semiont/oauth_clients.py +27 -0
- semiont/operations.py +337 -0
- semiont/py.typed +0 -0
- semiont/refresh.py +188 -0
- semiont/resume.py +95 -0
- semiont/retry.py +164 -0
- semiont/running.py +110 -0
- semiont/session.py +493 -0
- semiont/sign_in.py +54 -0
- semiont/sign_in_store.py +283 -0
- semiont/storage.py +139 -0
- semiont/telemetry.py +116 -0
- semiont/telemetry_table.py +121 -0
- semiont/testing/__init__.py +62 -0
- semiont/testing/client.py +53 -0
- semiont/testing/content.py +124 -0
- semiont/testing/gateway.py +85 -0
- semiont/testing/transport.py +341 -0
- semiont/timing.py +187 -0
- semiont/transport.py +531 -0
- semiont/types.py +5902 -0
- semiont/watched.py +111 -0
- semiont-0.6.12.dev2.dist-info/METADATA +533 -0
- semiont-0.6.12.dev2.dist-info/RECORD +66 -0
- semiont-0.6.12.dev2.dist-info/WHEEL +4 -0
semiont/http/stream.py
ADDED
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
"""The stream (`docs/protocol/TRANSPORT-HTTP.md` § The client).
|
|
2
|
+
|
|
3
|
+
One task owns the subscription, the connections that carry it, and the state
|
|
4
|
+
they add up to.
|
|
5
|
+
|
|
6
|
+
A client holds one live stream. Two things replace it, and they are different
|
|
7
|
+
things:
|
|
8
|
+
|
|
9
|
+
- A **drop**: the stream ended, or a connect failed with none live. For a
|
|
10
|
+
while frames are not delivered, the state leaves `open`, and the stream is
|
|
11
|
+
opened again on a backoff, naming each scope's last position and the replies
|
|
12
|
+
still awaited.
|
|
13
|
+
- A **handoff**: the subscription changed. A second connection is opened beside
|
|
14
|
+
the live one and takes over once it is open. The old one keeps being read
|
|
15
|
+
for `linger_ms` and is closed then. Nothing is missed, so the state stays
|
|
16
|
+
`open`.
|
|
17
|
+
|
|
18
|
+
While both connections of a handoff are read, a frame is delivered once: the
|
|
19
|
+
ids of the last frames delivered are remembered, and a frame whose id is among
|
|
20
|
+
them is dropped.
|
|
21
|
+
|
|
22
|
+
Connections are tasks that only read. Everything they read comes back to this
|
|
23
|
+
one task, in order, so the state has one writer.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import asyncio
|
|
27
|
+
import logging
|
|
28
|
+
from collections import OrderedDict
|
|
29
|
+
from collections.abc import Mapping
|
|
30
|
+
from contextlib import suppress
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from typing import Final, Protocol, final
|
|
33
|
+
|
|
34
|
+
import httpx
|
|
35
|
+
from pydantic import JsonValue, ValidationError
|
|
36
|
+
|
|
37
|
+
from semiont import telemetry
|
|
38
|
+
from semiont.channels import RESOURCE_SCOPED_CHANNELS
|
|
39
|
+
from semiont.errors import SemiontError, TransportError
|
|
40
|
+
from semiont.events import Broadcast
|
|
41
|
+
from semiont.http.exchange import TokenRefresher
|
|
42
|
+
from semiont.http.sse import SseParser
|
|
43
|
+
from semiont.identifiers import ResourceId
|
|
44
|
+
from semiont.retry import RetryPolicy, equal_jitter, retry_after_ms
|
|
45
|
+
from semiont.timing import (
|
|
46
|
+
DEGRADED_THRESHOLD_MS,
|
|
47
|
+
EMIT_RETRY,
|
|
48
|
+
HTTP_REQUEST_TIMEOUT_MS,
|
|
49
|
+
LAZY_REMOVE_MS,
|
|
50
|
+
LINGER_MS,
|
|
51
|
+
MAX_RECONNECT_MS,
|
|
52
|
+
RECONNECT_DEBOUNCE_MS,
|
|
53
|
+
RECONNECT_MS,
|
|
54
|
+
SEEN_EVENT_IDS_COUNT,
|
|
55
|
+
)
|
|
56
|
+
from semiont.transport import CONNECTION_STATE_MAY_BECOME, ConnectionState, Frame, FrameHub, ReplyRouter, TraceContext
|
|
57
|
+
from semiont.types import BusFrame, BusSubscribeRequest, BusSubscribeRequestScopedItem, ErrorResponse
|
|
58
|
+
from semiont.watched import Variable, Watched
|
|
59
|
+
|
|
60
|
+
__all__ = ["Bookmarks", "SeenIds", "Stream", "Timing", "backoff_cap_ms"]
|
|
61
|
+
|
|
62
|
+
_LOG: Final = logging.getLogger("semiont.http")
|
|
63
|
+
|
|
64
|
+
# Where a frame's payload carries the trace it was sent under. It is the wire's, and no reader of the frame sees it.
|
|
65
|
+
_TRACE_FIELD: Final = "_trace"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _sent_under(carried: JsonValue) -> TraceContext | None:
|
|
69
|
+
"""The trace a frame's payload says it was sent under, when it says so."""
|
|
70
|
+
if not isinstance(carried, dict):
|
|
71
|
+
return None
|
|
72
|
+
traceparent, tracestate = carried.get("traceparent"), carried.get("tracestate")
|
|
73
|
+
if not isinstance(traceparent, str):
|
|
74
|
+
return None
|
|
75
|
+
return TraceContext(traceparent=traceparent, tracestate=tracestate if isinstance(tracestate, str) and tracestate else None)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@final
|
|
79
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
80
|
+
class Timing:
|
|
81
|
+
"""The timing a transport keeps: `specs/src/client/timing.json`'s, unless a caller that must not wait it out says otherwise."""
|
|
82
|
+
|
|
83
|
+
reconnect_ms: int = RECONNECT_MS
|
|
84
|
+
lazy_remove_ms: int = LAZY_REMOVE_MS
|
|
85
|
+
linger_ms: int = LINGER_MS
|
|
86
|
+
emit_retry: RetryPolicy = EMIT_RETRY
|
|
87
|
+
seen_event_ids_count: int = SEEN_EVENT_IDS_COUNT
|
|
88
|
+
http_request_ms: int = HTTP_REQUEST_TIMEOUT_MS
|
|
89
|
+
"""The deadline on one request that is neither the stream nor an emit."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def backoff_cap_ms(reconnect_ms: int, failures: int) -> int:
|
|
93
|
+
"""The longest wait before the next connect, after `failures` in a row: `reconnect_ms` doubled per failure, up to `MAX_RECONNECT_MS`."""
|
|
94
|
+
if reconnect_ms >= MAX_RECONNECT_MS:
|
|
95
|
+
return MAX_RECONNECT_MS
|
|
96
|
+
# Past this many doublings the cap is the ceiling whatever the base.
|
|
97
|
+
doublings = min(failures, MAX_RECONNECT_MS.bit_length())
|
|
98
|
+
return min(reconnect_ms << doublings, MAX_RECONNECT_MS)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@final
|
|
102
|
+
class _Carried(BusFrame, frozen=True, extra="ignore"):
|
|
103
|
+
"""A frame as a stream carries it, read by what it means.
|
|
104
|
+
|
|
105
|
+
One a later gateway adds a field to is still a frame. One whose channel,
|
|
106
|
+
payload or scope is not what a frame's is, is not.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@final
|
|
111
|
+
class SeenIds:
|
|
112
|
+
"""The ids of the frames delivered last."""
|
|
113
|
+
|
|
114
|
+
def __init__(self, capacity: int) -> None:
|
|
115
|
+
self._capacity = capacity
|
|
116
|
+
self._ids: OrderedDict[str, None] = OrderedDict()
|
|
117
|
+
|
|
118
|
+
def remember(self, event_id: str) -> bool:
|
|
119
|
+
"""Remember `event_id`. False when it was already remembered: its frame has been delivered."""
|
|
120
|
+
if event_id in self._ids:
|
|
121
|
+
return False
|
|
122
|
+
self._ids[event_id] = None
|
|
123
|
+
if len(self._ids) > self._capacity:
|
|
124
|
+
self._ids.popitem(last=False)
|
|
125
|
+
return True
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# What a connection, the transport's caller, or the token's owner reports to the stream's task.
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@final
|
|
132
|
+
@dataclass(frozen=True, slots=True)
|
|
133
|
+
class _Opened:
|
|
134
|
+
"""The subscribe response is streaming."""
|
|
135
|
+
|
|
136
|
+
conn: int
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@final
|
|
140
|
+
@dataclass(frozen=True, slots=True)
|
|
141
|
+
class _Framed:
|
|
142
|
+
"""An event of the stream, with the id it came under."""
|
|
143
|
+
|
|
144
|
+
conn: int
|
|
145
|
+
id: str | None
|
|
146
|
+
data: str
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@final
|
|
150
|
+
@dataclass(frozen=True, slots=True)
|
|
151
|
+
class _Refused:
|
|
152
|
+
"""The gateway answered the connect, and not with a stream."""
|
|
153
|
+
|
|
154
|
+
conn: int
|
|
155
|
+
error: TransportError
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@final
|
|
159
|
+
@dataclass(frozen=True, slots=True)
|
|
160
|
+
class _Ended:
|
|
161
|
+
"""The connect got no answer, or the stream ended."""
|
|
162
|
+
|
|
163
|
+
conn: int
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@final
|
|
167
|
+
@dataclass(frozen=True, slots=True)
|
|
168
|
+
class _ScopeTaken:
|
|
169
|
+
"""A resource's scope got its first hold."""
|
|
170
|
+
|
|
171
|
+
resource: ResourceId
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@final
|
|
175
|
+
@dataclass(frozen=True, slots=True)
|
|
176
|
+
class _ScopeLetGo:
|
|
177
|
+
"""A resource's scope lost its last hold."""
|
|
178
|
+
|
|
179
|
+
resource: ResourceId
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@final
|
|
183
|
+
@dataclass(frozen=True, slots=True)
|
|
184
|
+
class _TokenChanged:
|
|
185
|
+
"""The token is another one than it was."""
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@final
|
|
189
|
+
@dataclass(frozen=True, slots=True)
|
|
190
|
+
class _Renewed:
|
|
191
|
+
"""The refresher answered."""
|
|
192
|
+
|
|
193
|
+
keep_previous: bool
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class Bookmarks(Protocol):
|
|
197
|
+
"""Where a stream's place in each scope is kept across a client's lives.
|
|
198
|
+
|
|
199
|
+
A client that starts with the places its last life reached is sent what
|
|
200
|
+
was recorded since, and what it kept of those scopes is brought up to
|
|
201
|
+
date by replay.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
def load(self) -> Mapping[ResourceId, str]:
|
|
205
|
+
"""The id of the last recorded event delivered on each scope, as kept."""
|
|
206
|
+
...
|
|
207
|
+
|
|
208
|
+
def save(self, scope: ResourceId, event_id: str) -> None:
|
|
209
|
+
"""A recorded event was delivered on `scope`."""
|
|
210
|
+
...
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@final
|
|
214
|
+
@dataclass(frozen=True, slots=True)
|
|
215
|
+
class _Stop:
|
|
216
|
+
"""The transport is closing."""
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
type _Report = _Opened | _Framed | _Refused | _Ended | _ScopeTaken | _ScopeLetGo | _TokenChanged | _Renewed
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
@final
|
|
223
|
+
@dataclass(frozen=True, slots=True)
|
|
224
|
+
class _Connection:
|
|
225
|
+
task: asyncio.Task[None]
|
|
226
|
+
token: str
|
|
227
|
+
"""The token the connect sent: what is remembered if it is refused."""
|
|
228
|
+
keep_previous: bool
|
|
229
|
+
"""Whether it was opened beside a live stream, to take over from it."""
|
|
230
|
+
previous: tuple[int, ...]
|
|
231
|
+
"""The connections it retires once it is open."""
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@final
|
|
235
|
+
class Stream:
|
|
236
|
+
"""A transport's stream: what it is subscribed to, the connections that carry it, and its state."""
|
|
237
|
+
|
|
238
|
+
def __init__(
|
|
239
|
+
self,
|
|
240
|
+
*,
|
|
241
|
+
base_url: str,
|
|
242
|
+
http: httpx.AsyncClient,
|
|
243
|
+
token: Watched[str | None],
|
|
244
|
+
refresher: TokenRefresher | None,
|
|
245
|
+
client_id: str,
|
|
246
|
+
channels: tuple[str, ...],
|
|
247
|
+
timing: Timing,
|
|
248
|
+
hub: FrameHub,
|
|
249
|
+
router: ReplyRouter,
|
|
250
|
+
failures: Broadcast[SemiontError],
|
|
251
|
+
bookmarks: Bookmarks | None,
|
|
252
|
+
) -> None:
|
|
253
|
+
self._base_url = base_url
|
|
254
|
+
self._http = http
|
|
255
|
+
self._token = token
|
|
256
|
+
self._refresher = refresher
|
|
257
|
+
self._client_id = client_id
|
|
258
|
+
self._global = channels
|
|
259
|
+
self._timing = timing
|
|
260
|
+
self._hub = hub
|
|
261
|
+
self._router = router
|
|
262
|
+
self._failures = failures
|
|
263
|
+
self._bookmarks = bookmarks
|
|
264
|
+
|
|
265
|
+
self._state: Variable[ConnectionState] = Variable("initial")
|
|
266
|
+
self._reports: asyncio.Queue[_Report | _Stop] = asyncio.Queue()
|
|
267
|
+
self._running = False
|
|
268
|
+
|
|
269
|
+
self._scoped: dict[ResourceId, tuple[str, ...]] = {}
|
|
270
|
+
"""The subscription's scoped half: each scope held, and its channels."""
|
|
271
|
+
self._watermarks: dict[ResourceId, str] = {} if bookmarks is None else dict(bookmarks.load())
|
|
272
|
+
"""The last recorded event delivered on each scope. A scope keeps its
|
|
273
|
+
position after it is let go: taken again, it resumes from there. Each
|
|
274
|
+
begins where the client's last life left it, when that was kept."""
|
|
275
|
+
self._seen = SeenIds(timing.seen_event_ids_count)
|
|
276
|
+
|
|
277
|
+
self._next_conn = 0
|
|
278
|
+
self._connections: dict[int, _Connection] = {}
|
|
279
|
+
self._live: int | None = None
|
|
280
|
+
"""The connection whose stream is the client's own. While there is one the state is `open`."""
|
|
281
|
+
self._connecting: int | None = None
|
|
282
|
+
"""The connect not yet answered. Changes of subscription asked for
|
|
283
|
+
meanwhile are served by one follow-up once it opens."""
|
|
284
|
+
self._reconnect_owed = False
|
|
285
|
+
self._superseded: set[int] = set()
|
|
286
|
+
"""Connections a handoff retired: still read, until their linger ends."""
|
|
287
|
+
self._lingering: list[tuple[float, tuple[int, ...]]] = []
|
|
288
|
+
|
|
289
|
+
self._failed_connects = 0
|
|
290
|
+
"""How many connects have failed since the last open."""
|
|
291
|
+
self._refused_token: str | None = None
|
|
292
|
+
"""The token the gateway refused: never sent again."""
|
|
293
|
+
self._awaiting_credential = False
|
|
294
|
+
self._refresher_asked = False
|
|
295
|
+
"""Whether the refresher has been asked since the stream was last open: it is asked once per outage."""
|
|
296
|
+
|
|
297
|
+
self._retry_at: tuple[float, bool] | None = None
|
|
298
|
+
self._debounce_at: float | None = None
|
|
299
|
+
self._lazy_at: float | None = None
|
|
300
|
+
self._degraded_at: float | None = None
|
|
301
|
+
|
|
302
|
+
@property
|
|
303
|
+
def state(self) -> Watched[ConnectionState]:
|
|
304
|
+
"""The connection's state."""
|
|
305
|
+
return self._state
|
|
306
|
+
|
|
307
|
+
def take_scope(self, resource: ResourceId) -> None:
|
|
308
|
+
"""A resource's scope got its first hold."""
|
|
309
|
+
self._reports.put_nowait(_ScopeTaken(resource))
|
|
310
|
+
|
|
311
|
+
def let_go_of_scope(self, resource: ResourceId) -> None:
|
|
312
|
+
"""A resource's scope lost its last hold."""
|
|
313
|
+
self._reports.put_nowait(_ScopeLetGo(resource))
|
|
314
|
+
|
|
315
|
+
def stop(self) -> None:
|
|
316
|
+
"""Close: the state is `closed` from now, and the stream's task ends what it started."""
|
|
317
|
+
self._closed()
|
|
318
|
+
self._reports.put_nowait(_Stop())
|
|
319
|
+
|
|
320
|
+
def _closed(self) -> None:
|
|
321
|
+
"""Nothing more is delivered: the state is `closed`, every reader's frames end, and every reply awaited will not come."""
|
|
322
|
+
self._running = False
|
|
323
|
+
self._state.set("closed")
|
|
324
|
+
self._state.end()
|
|
325
|
+
self._hub.close()
|
|
326
|
+
self._router.close()
|
|
327
|
+
|
|
328
|
+
async def run(self) -> None:
|
|
329
|
+
"""Hold the client's stream until `stop`. Every task it started has ended when it returns."""
|
|
330
|
+
loop = asyncio.get_running_loop()
|
|
331
|
+
try:
|
|
332
|
+
async with asyncio.TaskGroup() as tasks:
|
|
333
|
+
watching = tasks.create_task(self._watch_token())
|
|
334
|
+
while True:
|
|
335
|
+
try:
|
|
336
|
+
async with asyncio.timeout_at(self._next_deadline()):
|
|
337
|
+
report = await self._reports.get()
|
|
338
|
+
except TimeoutError:
|
|
339
|
+
self._timers(tasks, loop.time())
|
|
340
|
+
continue
|
|
341
|
+
if isinstance(report, _Stop):
|
|
342
|
+
break
|
|
343
|
+
self._reported(tasks, report, loop.time())
|
|
344
|
+
watching.cancel()
|
|
345
|
+
for connection in self._connections.values():
|
|
346
|
+
connection.task.cancel()
|
|
347
|
+
finally:
|
|
348
|
+
# However its task ended, by `stop` or not, the stream is over.
|
|
349
|
+
self._closed()
|
|
350
|
+
self._connections.clear()
|
|
351
|
+
|
|
352
|
+
# ── what is reported ─────────────────────────────────────────────────
|
|
353
|
+
|
|
354
|
+
def _reported(self, tasks: asyncio.TaskGroup, report: _Report, now: float) -> None:
|
|
355
|
+
match report:
|
|
356
|
+
case _Opened(conn):
|
|
357
|
+
self._opened(tasks, conn, now)
|
|
358
|
+
case _Framed(conn, event_id, data):
|
|
359
|
+
self._framed(conn, event_id, data)
|
|
360
|
+
case _Refused(conn, error):
|
|
361
|
+
self._refused(tasks, conn, error, now)
|
|
362
|
+
case _Ended(conn):
|
|
363
|
+
self._ended(tasks, conn, now)
|
|
364
|
+
case _ScopeTaken(resource):
|
|
365
|
+
self._scoped[resource] = RESOURCE_SCOPED_CHANNELS
|
|
366
|
+
# A new scope needs to be live now: additions are gathered for
|
|
367
|
+
# the debounce, and carry any removal waiting with them.
|
|
368
|
+
self._lazy_at = None
|
|
369
|
+
self._debounce_at = now + RECONNECT_DEBOUNCE_MS / 1000
|
|
370
|
+
case _ScopeLetGo(resource):
|
|
371
|
+
if self._scoped.pop(resource, None) is not None and self._debounce_at is None and self._lazy_at is None:
|
|
372
|
+
# A removal only narrows what is delivered, so it waits: a
|
|
373
|
+
# client that brushes past scopes would otherwise reopen
|
|
374
|
+
# its stream at each.
|
|
375
|
+
self._lazy_at = now + self._timing.lazy_remove_ms / 1000
|
|
376
|
+
case _TokenChanged():
|
|
377
|
+
self._token_changed(tasks, now)
|
|
378
|
+
case _Renewed(keep_previous):
|
|
379
|
+
# Whatever it answered, it has fed the token what it could: a
|
|
380
|
+
# connect now sends the token there is, or finds none it may
|
|
381
|
+
# send and waits for another.
|
|
382
|
+
self._schedule_retry(0, keep_previous, now)
|
|
383
|
+
|
|
384
|
+
async def _watch_token(self) -> None:
|
|
385
|
+
async for _ in self._token:
|
|
386
|
+
self._reports.put_nowait(_TokenChanged())
|
|
387
|
+
|
|
388
|
+
def _current_token(self) -> str | None:
|
|
389
|
+
return self._token.value or None
|
|
390
|
+
|
|
391
|
+
def _transition(self, to: ConnectionState, now: float) -> None:
|
|
392
|
+
current = self._state.value
|
|
393
|
+
if to == current or to not in CONNECTION_STATE_MAY_BECOME[current]:
|
|
394
|
+
return
|
|
395
|
+
# A stream that stays down is degraded once it has been reconnecting
|
|
396
|
+
# for the threshold: the timer runs while the state is `reconnecting`.
|
|
397
|
+
self._degraded_at = now + DEGRADED_THRESHOLD_MS / 1000 if to == "reconnecting" else None
|
|
398
|
+
self._state.set(to)
|
|
399
|
+
|
|
400
|
+
def _schedule_retry(self, after_ms: float, keep_previous: bool, now: float) -> None:
|
|
401
|
+
if self._running:
|
|
402
|
+
self._retry_at = (now + after_ms / 1000, keep_previous)
|
|
403
|
+
|
|
404
|
+
def _backoff_ms(self) -> float:
|
|
405
|
+
"""Equal-jitter exponential backoff: a wait in [cap/2, cap]."""
|
|
406
|
+
cap = backoff_cap_ms(self._timing.reconnect_ms, self._failed_connects)
|
|
407
|
+
self._failed_connects += 1
|
|
408
|
+
return equal_jitter(cap)
|
|
409
|
+
|
|
410
|
+
def _abort(self, conn: int) -> None:
|
|
411
|
+
connection = self._connections.pop(conn, None)
|
|
412
|
+
if connection is not None:
|
|
413
|
+
connection.task.cancel()
|
|
414
|
+
self._superseded.discard(conn)
|
|
415
|
+
|
|
416
|
+
# ── connecting ───────────────────────────────────────────────────────
|
|
417
|
+
|
|
418
|
+
def _connect(self, tasks: asyncio.TaskGroup, keep_previous: bool, now: float) -> None:
|
|
419
|
+
# A connect with no token cannot succeed, and neither can one that
|
|
420
|
+
# sends again the token the gateway just refused. Neither is
|
|
421
|
+
# attempted: the client waits, with no request, for a different one.
|
|
422
|
+
token = self._current_token()
|
|
423
|
+
if token is None or token == self._refused_token:
|
|
424
|
+
if self._running:
|
|
425
|
+
# With a stream still live the state stays `open`: only the
|
|
426
|
+
# change of subscription is waiting for a credential.
|
|
427
|
+
if self._live is None:
|
|
428
|
+
self._transition("unauthenticated", now)
|
|
429
|
+
self._awaiting_credential = True
|
|
430
|
+
self._schedule_retry(self._timing.reconnect_ms, keep_previous, now)
|
|
431
|
+
return
|
|
432
|
+
self._refused_token = None
|
|
433
|
+
self._awaiting_credential = False
|
|
434
|
+
|
|
435
|
+
previous = tuple(self._connections)
|
|
436
|
+
if not keep_previous:
|
|
437
|
+
# An initial connect, or the recovery of a drop: nothing live is worth keeping.
|
|
438
|
+
for conn in previous:
|
|
439
|
+
self._abort(conn)
|
|
440
|
+
self._live = None
|
|
441
|
+
self._lingering.clear()
|
|
442
|
+
# Opening beside a live stream is a handoff, and the state stays `open`. With none, this is `connecting`.
|
|
443
|
+
if self._live is None:
|
|
444
|
+
self._transition("connecting", now)
|
|
445
|
+
|
|
446
|
+
awaited = self._router.awaited()
|
|
447
|
+
body = BusSubscribeRequest(
|
|
448
|
+
client_id=self._client_id,
|
|
449
|
+
global_=list(self._global),
|
|
450
|
+
scoped=[
|
|
451
|
+
BusSubscribeRequestScopedItem(scope=scope, channels=list(channels), last_event_id=self._watermarks.get(scope))
|
|
452
|
+
for scope, channels in sorted(self._scoped.items())
|
|
453
|
+
],
|
|
454
|
+
pending_replies=awaited or None,
|
|
455
|
+
)
|
|
456
|
+
conn = self._next_conn
|
|
457
|
+
self._next_conn += 1
|
|
458
|
+
self._connections[conn] = _Connection(
|
|
459
|
+
task=tasks.create_task(self._connection(conn, token, body)),
|
|
460
|
+
token=token,
|
|
461
|
+
keep_previous=keep_previous,
|
|
462
|
+
previous=previous if keep_previous else (),
|
|
463
|
+
)
|
|
464
|
+
self._connecting = conn
|
|
465
|
+
|
|
466
|
+
def _reconnect(self, tasks: asyncio.TaskGroup, now: float) -> None:
|
|
467
|
+
"""A changed subscription: hand it to a new stream. The live one is not touched, and the state does not move."""
|
|
468
|
+
if not self._running:
|
|
469
|
+
return
|
|
470
|
+
if self._connecting is not None:
|
|
471
|
+
self._reconnect_owed = True
|
|
472
|
+
return
|
|
473
|
+
self._retry_at = None
|
|
474
|
+
self._connect(tasks, True, now)
|
|
475
|
+
|
|
476
|
+
def _settle_connect(self, tasks: asyncio.TaskGroup, conn: int, opened: bool, now: float) -> None:
|
|
477
|
+
"""The connect `conn` made is answered.
|
|
478
|
+
|
|
479
|
+
Opened, it serves the changes asked for while it was in flight with
|
|
480
|
+
one follow-up. Failed, it leaves them to the retry, which reads the
|
|
481
|
+
same subscription.
|
|
482
|
+
"""
|
|
483
|
+
if self._connecting != conn:
|
|
484
|
+
return
|
|
485
|
+
self._connecting = None
|
|
486
|
+
owed, self._reconnect_owed = self._reconnect_owed, False
|
|
487
|
+
if opened and owed and self._running:
|
|
488
|
+
self._reconnect(tasks, now)
|
|
489
|
+
|
|
490
|
+
def _opened(self, tasks: asyncio.TaskGroup, conn: int, now: float) -> None:
|
|
491
|
+
connection = self._connections.get(conn)
|
|
492
|
+
if connection is None:
|
|
493
|
+
return
|
|
494
|
+
if connection.keep_previous:
|
|
495
|
+
# The new stream is established, so the ones it replaces are
|
|
496
|
+
# retired by drain: read for `linger_ms` more, then closed. Closing
|
|
497
|
+
# them now would discard what they have received and not yet
|
|
498
|
+
# delivered.
|
|
499
|
+
previous = tuple(retired for retired in connection.previous if retired in self._connections)
|
|
500
|
+
self._superseded.update(previous)
|
|
501
|
+
self._lingering.append((now + self._timing.linger_ms / 1000, previous))
|
|
502
|
+
self._live = conn
|
|
503
|
+
self._transition("open", now)
|
|
504
|
+
self._failed_connects = 0
|
|
505
|
+
self._refresher_asked = False
|
|
506
|
+
self._settle_connect(tasks, conn, True, now)
|
|
507
|
+
|
|
508
|
+
def _framed(self, conn: int, event_id: str | None, data: str) -> None:
|
|
509
|
+
if conn not in self._connections:
|
|
510
|
+
return
|
|
511
|
+
if event_id is not None and not self._seen.remember(event_id):
|
|
512
|
+
return
|
|
513
|
+
try:
|
|
514
|
+
carried = _Carried.model_validate_json(data)
|
|
515
|
+
except ValidationError as error:
|
|
516
|
+
# What is not a frame is not delivered, and is said so.
|
|
517
|
+
_LOG.warning("the stream carried what is not a frame (id %s): %s", event_id, error)
|
|
518
|
+
return
|
|
519
|
+
payload = carried.payload
|
|
520
|
+
sent_under: TraceContext | None = None
|
|
521
|
+
if _TRACE_FIELD in payload:
|
|
522
|
+
sent_under = _sent_under(payload[_TRACE_FIELD])
|
|
523
|
+
payload = {name: value for name, value in payload.items() if name != _TRACE_FIELD}
|
|
524
|
+
frame = Frame(
|
|
525
|
+
channel=carried.channel,
|
|
526
|
+
payload=payload,
|
|
527
|
+
correlation_id=carried.correlation_id,
|
|
528
|
+
scope=carried.scope,
|
|
529
|
+
trace=telemetry.received(carried.channel, carried.scope, sent_under),
|
|
530
|
+
)
|
|
531
|
+
self._router.route(frame)
|
|
532
|
+
# A scope's position is the last recorded event delivered on it: only
|
|
533
|
+
# a recorded event's id (`p-…`) is one, and it always comes on its scope.
|
|
534
|
+
if event_id is not None and frame.scope is not None and event_id.startswith("p-"):
|
|
535
|
+
self._watermarks[frame.scope] = event_id
|
|
536
|
+
self._hub.deliver(frame)
|
|
537
|
+
# Kept only once the event has been given to those it changes: a place is never ahead of what was taken in.
|
|
538
|
+
if self._bookmarks is not None and event_id is not None and frame.scope is not None and event_id.startswith("p-"):
|
|
539
|
+
self._bookmarks.save(frame.scope, event_id)
|
|
540
|
+
|
|
541
|
+
def _refused(self, tasks: asyncio.TaskGroup, conn: int, error: TransportError, now: float) -> None:
|
|
542
|
+
connection = self._connections.pop(conn, None)
|
|
543
|
+
if connection is None:
|
|
544
|
+
return
|
|
545
|
+
self._settle_connect(tasks, conn, False, now)
|
|
546
|
+
# A refused connect is a request the gateway refused, reported as one.
|
|
547
|
+
# A connect that got no answer is not: it is a state.
|
|
548
|
+
self._failures.deliver(error)
|
|
549
|
+
superseded = conn in self._superseded
|
|
550
|
+
self._superseded.discard(conn)
|
|
551
|
+
if error.status == 401 and self._running and not superseded:
|
|
552
|
+
# Sending this token again gets the same answer, so it is not sent
|
|
553
|
+
# again: the client waits for a different one, and asks its
|
|
554
|
+
# refresher for one once per outage.
|
|
555
|
+
self._refused_token = connection.token
|
|
556
|
+
self._failed_connects = 0
|
|
557
|
+
if self._live is None:
|
|
558
|
+
self._transition("unauthenticated", now)
|
|
559
|
+
if self._refresher is not None and not self._refresher_asked:
|
|
560
|
+
self._refresher_asked = True
|
|
561
|
+
tasks.create_task(self._ask_refresher(self._refresher, connection.keep_previous))
|
|
562
|
+
else:
|
|
563
|
+
self._awaiting_credential = True
|
|
564
|
+
self._schedule_retry(self._timing.reconnect_ms, connection.keep_previous, now)
|
|
565
|
+
return
|
|
566
|
+
self._dropped_or_failed(conn, superseded, error.retry_after_ms or 0, now)
|
|
567
|
+
|
|
568
|
+
def _ended(self, tasks: asyncio.TaskGroup, conn: int, now: float) -> None:
|
|
569
|
+
if self._connections.pop(conn, None) is None:
|
|
570
|
+
return
|
|
571
|
+
self._settle_connect(tasks, conn, False, now)
|
|
572
|
+
superseded = conn in self._superseded
|
|
573
|
+
self._superseded.discard(conn)
|
|
574
|
+
self._dropped_or_failed(conn, superseded, 0, now)
|
|
575
|
+
|
|
576
|
+
def _dropped_or_failed(self, conn: int, superseded: bool, stated_wait_ms: int, now: float) -> None:
|
|
577
|
+
"""A stream ended, or a connect failed. A superseded connection ending is expected and restarts nothing."""
|
|
578
|
+
if not self._running or superseded:
|
|
579
|
+
return
|
|
580
|
+
if self._live is not None and self._live != conn:
|
|
581
|
+
# A handoff that could not open. The stream it was to replace is
|
|
582
|
+
# still live, so nothing has dropped: it is tried again on the
|
|
583
|
+
# backoff, and the state does not move.
|
|
584
|
+
self._schedule_retry(max(self._backoff_ms(), stated_wait_ms), True, now)
|
|
585
|
+
return
|
|
586
|
+
# A drop.
|
|
587
|
+
self._live = None
|
|
588
|
+
self._transition("reconnecting", now)
|
|
589
|
+
if self._connecting is not None:
|
|
590
|
+
# A handoff's connect is in flight: it is the recovery now, and comes back through here if it fails.
|
|
591
|
+
self._transition("connecting", now)
|
|
592
|
+
return
|
|
593
|
+
self._schedule_retry(max(self._backoff_ms(), stated_wait_ms), False, now)
|
|
594
|
+
|
|
595
|
+
async def _ask_refresher(self, refresher: TokenRefresher, keep_previous: bool) -> None:
|
|
596
|
+
with suppress(SemiontError):
|
|
597
|
+
await refresher()
|
|
598
|
+
self._reports.put_nowait(_Renewed(keep_previous))
|
|
599
|
+
|
|
600
|
+
def _token_changed(self, tasks: asyncio.TaskGroup, now: float) -> None:
|
|
601
|
+
if not self._running:
|
|
602
|
+
# The stream starts when there is a token to open it with.
|
|
603
|
+
if self._state.value == "initial" and self._current_token() is not None:
|
|
604
|
+
self._running = True
|
|
605
|
+
self._connect(tasks, False, now)
|
|
606
|
+
return
|
|
607
|
+
# A client waiting for a credential tries at once.
|
|
608
|
+
if self._awaiting_credential and self._retry_at is not None:
|
|
609
|
+
self._retry_at = (now, self._retry_at[1])
|
|
610
|
+
|
|
611
|
+
# ── time ─────────────────────────────────────────────────────────────
|
|
612
|
+
|
|
613
|
+
def _next_deadline(self) -> float | None:
|
|
614
|
+
deadlines = [
|
|
615
|
+
deadline
|
|
616
|
+
for deadline in (
|
|
617
|
+
None if self._retry_at is None else self._retry_at[0],
|
|
618
|
+
self._debounce_at,
|
|
619
|
+
self._lazy_at,
|
|
620
|
+
self._degraded_at,
|
|
621
|
+
min((at for at, _ in self._lingering), default=None),
|
|
622
|
+
)
|
|
623
|
+
if deadline is not None
|
|
624
|
+
]
|
|
625
|
+
return min(deadlines, default=None)
|
|
626
|
+
|
|
627
|
+
def _timers(self, tasks: asyncio.TaskGroup, now: float) -> None:
|
|
628
|
+
if self._degraded_at is not None and self._degraded_at <= now:
|
|
629
|
+
self._degraded_at = None
|
|
630
|
+
if self._state.value == "reconnecting":
|
|
631
|
+
self._state.set("degraded")
|
|
632
|
+
over = [conns for at, conns in self._lingering if at <= now]
|
|
633
|
+
self._lingering = [(at, conns) for at, conns in self._lingering if at > now]
|
|
634
|
+
for conns in over:
|
|
635
|
+
for conn in conns:
|
|
636
|
+
self._abort(conn)
|
|
637
|
+
debounced = self._debounce_at is not None and self._debounce_at <= now
|
|
638
|
+
lazy = self._lazy_at is not None and self._lazy_at <= now
|
|
639
|
+
if debounced or lazy:
|
|
640
|
+
self._debounce_at = None
|
|
641
|
+
self._lazy_at = None
|
|
642
|
+
self._reconnect(tasks, now)
|
|
643
|
+
if self._retry_at is not None and self._retry_at[0] <= now:
|
|
644
|
+
keep_previous = self._retry_at[1]
|
|
645
|
+
self._retry_at = None
|
|
646
|
+
if self._running:
|
|
647
|
+
self._connect(tasks, keep_previous, now)
|
|
648
|
+
|
|
649
|
+
# ── one connection ───────────────────────────────────────────────────
|
|
650
|
+
|
|
651
|
+
async def _connection(self, conn: int, token: str, body: BusSubscribeRequest) -> None:
|
|
652
|
+
"""One connection: the subscribe request, and every event its stream carries, reported in order."""
|
|
653
|
+
request = self._http.stream(
|
|
654
|
+
"POST",
|
|
655
|
+
f"{self._base_url}/bus/subscribe",
|
|
656
|
+
content=body.model_dump_json(exclude_none=True),
|
|
657
|
+
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json", "Accept": "text/event-stream"},
|
|
658
|
+
)
|
|
659
|
+
try:
|
|
660
|
+
async with request as response:
|
|
661
|
+
if not response.is_success:
|
|
662
|
+
self._reports.put_nowait(_Refused(conn, await _refusal(response)))
|
|
663
|
+
return
|
|
664
|
+
self._reports.put_nowait(_Opened(conn))
|
|
665
|
+
parser = SseParser()
|
|
666
|
+
async for read in response.aiter_bytes():
|
|
667
|
+
for event in parser.feed(read):
|
|
668
|
+
if event.event == "bus-event" and event.data != "":
|
|
669
|
+
self._reports.put_nowait(_Framed(conn, event.id, event.data))
|
|
670
|
+
except httpx.HTTPError:
|
|
671
|
+
# No answer, or a stream that broke: a state, reported as the stream's end.
|
|
672
|
+
pass
|
|
673
|
+
self._reports.put_nowait(_Ended(conn))
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
async def _refusal(response: httpx.Response) -> TransportError:
|
|
677
|
+
"""The gateway's refusal of a connect: its status, in its own words when its body states them."""
|
|
678
|
+
status = response.status_code
|
|
679
|
+
try:
|
|
680
|
+
said = f": {ErrorResponse.model_validate_json(await response.aread()).error}"
|
|
681
|
+
except (ValidationError, httpx.HTTPError):
|
|
682
|
+
said = ""
|
|
683
|
+
return TransportError.of_status(f"SSE connect failed: {status}{said}", status, retry_after_ms(response.headers.get("retry-after")))
|