pycyphal2 2.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pycyphal2/_node.py ADDED
@@ -0,0 +1,1472 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ from collections import OrderedDict
5
+ import logging
6
+ import math
7
+ import os
8
+ import random
9
+ import time
10
+ from dataclasses import dataclass, field
11
+ from enum import Enum, auto
12
+ from typing import TYPE_CHECKING, Any, Callable
13
+
14
+ from ._hash import rapidhash
15
+ from ._header import (
16
+ HEADER_SIZE,
17
+ GossipHeader,
18
+ LAGE_MAX,
19
+ MsgAckHeader,
20
+ MsgBeHeader,
21
+ MsgNackHeader,
22
+ MsgRelHeader,
23
+ RspAckHeader,
24
+ RspBeHeader,
25
+ RspNackHeader,
26
+ RspRelHeader,
27
+ ScoutHeader,
28
+ deserialize_header,
29
+ )
30
+ from ._transport import SubjectWriter, Transport, TransportArrival
31
+ from ._api import Topic, Node, Publisher, Subscriber, Breadcrumb, Closable, Instant, Priority, SendError
32
+ from ._api import SUBJECT_ID_PINNED_MAX
33
+
34
+ if TYPE_CHECKING:
35
+ from ._publisher import ResponseStreamImpl
36
+ from ._subscriber import RespondTracker
37
+
38
+ _logger = logging.getLogger(__name__)
39
+
40
+ # =====================================================================================================================
41
+ # Constants
42
+ # =====================================================================================================================
43
+
44
+ TOPIC_NAME_MAX = 200
45
+ EVICTIONS_PINNED_MIN = 0xFFFFE000
46
+ GOSSIP_PERIOD = 5.0
47
+ GOSSIP_URGENT_DELAY_MAX = 0.01
48
+ GOSSIP_BROADCAST_RATIO = 10
49
+ GOSSIP_PERIOD_DITHER_RATIO = 8
50
+ ACK_BASELINE_DEFAULT_TIMEOUT = 0.016
51
+ ACK_TX_TIMEOUT = 1.0
52
+ SESSION_LIFETIME = 60.0
53
+ IMPLICIT_TOPIC_TIMEOUT = 600.0
54
+ REORDERING_CAPACITY = 16
55
+ ASSOC_SLACK_LIMIT = 2
56
+ DEDUP_HISTORY = 512
57
+ ACK_SEQNO_MAX_LAG = 100000
58
+ U64_MASK = (1 << 64) - 1
59
+
60
+
61
+ class GossipScope(Enum):
62
+ UNICAST = auto()
63
+ BROADCAST = auto()
64
+ SHARDED = auto()
65
+ INLINE = auto()
66
+
67
+
68
+ # =====================================================================================================================
69
+ # Name Resolution
70
+ # =====================================================================================================================
71
+
72
+
73
+ def _name_normalize(name: str) -> str:
74
+ """Collapse separators, strip leading/trailing separators."""
75
+ parts: list[str] = []
76
+ for seg in name.split("/"):
77
+ if seg:
78
+ parts.append(seg)
79
+ return "/".join(parts)
80
+
81
+
82
+ def _name_consume_pin_suffix(name: str) -> tuple[str, int | None]:
83
+ """Extract pin suffix like 'foo#123' -> ('foo', 123). Returns (name, None) if no valid pin."""
84
+ hash_pos = -1
85
+ for i in range(len(name) - 1, -1, -1):
86
+ ch = name[i]
87
+ if ch == "#":
88
+ hash_pos = i
89
+ break
90
+ if not ch.isdigit():
91
+ return (name, None)
92
+ if hash_pos < 0:
93
+ return (name, None)
94
+ digits = name[hash_pos + 1 :]
95
+ if len(digits) == 0:
96
+ return (name, None)
97
+ if len(digits) > 1 and digits[0] == "0":
98
+ return (name, None) # leading zeros not allowed
99
+ pin = int(digits)
100
+ if pin > SUBJECT_ID_PINNED_MAX:
101
+ return (name, None)
102
+ return (name[:hash_pos], pin)
103
+
104
+
105
+ def _name_join(left: str, right: str) -> str:
106
+ """Join two name parts with separator, normalizing the result."""
107
+ left = _name_normalize(left)
108
+ right = _name_normalize(right)
109
+ if left and right:
110
+ return left + "/" + right
111
+ return left or right
112
+
113
+
114
+ def _name_is_homeful(name: str) -> bool:
115
+ return name == "~" or name.startswith("~/")
116
+
117
+
118
+ def resolve_name(
119
+ name: str, home: str, namespace: str, remaps: dict[str, str] | None = None
120
+ ) -> tuple[str, int | None, bool]:
121
+ """
122
+ Resolve a topic name to (resolved_name, pin_or_None, is_verbatim).
123
+ Raises ValueError on invalid names.
124
+ """
125
+ # REFERENCE PARITY: Python-only ergonomic deviation -- outer whitespace is trimmed before validation.
126
+ # The reference resolver rejects such names because spaces are invalid topic-name characters.
127
+ name = name.strip()
128
+ if not name:
129
+ raise ValueError("Empty name")
130
+
131
+ # Strip pin suffix first.
132
+ name, pin = _name_consume_pin_suffix(name)
133
+
134
+ # Apply remapping: lookup on normalized pin-free name; matched rule replaces both name and pin.
135
+ if remaps:
136
+ lookup = _name_normalize(name)
137
+ if lookup in remaps:
138
+ name = remaps[lookup]
139
+ name, pin = _name_consume_pin_suffix(name)
140
+
141
+ # Classify and construct.
142
+ if name.startswith("/"):
143
+ resolved = _name_normalize(name)
144
+ elif _name_is_homeful(name):
145
+ tail = name[1:].lstrip("/") if len(name) > 1 else ""
146
+ resolved = _name_join(home, tail)
147
+ else:
148
+ if _name_is_homeful(namespace):
149
+ ns_tail = namespace[1:].lstrip("/") if len(namespace) > 1 else ""
150
+ expanded_ns = _name_join(home, ns_tail)
151
+ else:
152
+ expanded_ns = namespace
153
+ resolved = _name_join(expanded_ns, name)
154
+
155
+ if not resolved:
156
+ raise ValueError("Name resolves to empty string")
157
+ if len(resolved) > TOPIC_NAME_MAX:
158
+ raise ValueError(f"Resolved name exceeds {TOPIC_NAME_MAX} characters")
159
+ # Validate characters: ASCII 33-126 and '/' only.
160
+ for ch in resolved:
161
+ o = ord(ch)
162
+ if o < 33 or o > 126:
163
+ raise ValueError(f"Invalid character in name: {ch!r}")
164
+
165
+ verbatim = "*" not in resolved and ">" not in resolved
166
+ if pin is not None and not verbatim:
167
+ raise ValueError("Pattern names cannot be pinned")
168
+ return resolved, pin, verbatim
169
+
170
+
171
+ # =====================================================================================================================
172
+ # Pattern Matching
173
+ # =====================================================================================================================
174
+
175
+
176
+ def match_pattern(pattern: str, name: str) -> list[tuple[str, int]] | None:
177
+ """
178
+ Match a pattern against a topic name.
179
+ Returns substitutions list on match, None on no match.
180
+ Empty list for verbatim match (pattern == name).
181
+
182
+ REFERENCE PARITY: Intentional deviation from the current C reference -- only a terminal '>' acts as an
183
+ any-segment wildcard. Non-terminal '>' is treated literally until the reference behavior converges.
184
+ """
185
+ if pattern == name:
186
+ return []
187
+ p_parts = pattern.split("/")
188
+ n_parts = name.split("/")
189
+ subs: list[tuple[str, int]] = []
190
+ for i, pp in enumerate(p_parts):
191
+ if pp == ">" and i == (len(p_parts) - 1):
192
+ subs.append(("/".join(n_parts[i:]), i))
193
+ return subs
194
+ if i >= len(n_parts):
195
+ return None
196
+ if pp == "*":
197
+ subs.append((n_parts[i], i))
198
+ elif pp != n_parts[i]:
199
+ return None
200
+ if len(p_parts) != len(n_parts):
201
+ return None
202
+ return subs
203
+
204
+
205
+ # =====================================================================================================================
206
+ # Subject-ID Computation
207
+ # =====================================================================================================================
208
+
209
+
210
+ def compute_subject_id(topic_hash: int, evictions: int, modulus: int) -> int:
211
+ """Compute the subject-ID for a topic given its hash, evictions, and subject-ID modulus."""
212
+ if evictions >= EVICTIONS_PINNED_MIN:
213
+ return 0xFFFFFFFF - evictions
214
+ return SUBJECT_ID_PINNED_MAX + 1 + ((topic_hash + (evictions * evictions)) % modulus)
215
+
216
+
217
+ # =====================================================================================================================
218
+ # Internal Data Structures
219
+ # =====================================================================================================================
220
+
221
+
222
+ @dataclass
223
+ class Association:
224
+ """Tracks a known remote subscriber for reliable delivery ACK tracking."""
225
+
226
+ remote_id: int
227
+ last_seen: float
228
+ slack: int = 0
229
+ seqno_witness: int = 0
230
+ pending_count: int = 0
231
+
232
+
233
+ @dataclass
234
+ class DedupState:
235
+ """Per-remote deduplication state for reliable messages."""
236
+
237
+ tag_frontier: int = 0
238
+ bitmap: int = 0
239
+ last_active: float = 0.0
240
+
241
+ def check(self, tag: int) -> bool:
242
+ rev = (self.tag_frontier - tag) & U64_MASK
243
+ return rev < DEDUP_HISTORY and bool((self.bitmap >> rev) & 1)
244
+
245
+ def check_and_record(self, tag: int, now: float) -> bool:
246
+ """Returns True if this is a new (non-duplicate) tag."""
247
+ if (now - self.last_active) > SESSION_LIFETIME:
248
+ self.tag_frontier = tag
249
+ self.bitmap = 0
250
+ self.last_active = now
251
+ fwd = (tag - self.tag_frontier) & U64_MASK
252
+ rev = (self.tag_frontier - tag) & U64_MASK
253
+ if rev < DEDUP_HISTORY:
254
+ mask = 1 << rev
255
+ if self.bitmap & mask:
256
+ return False
257
+ self.bitmap |= mask
258
+ return True
259
+ if fwd < DEDUP_HISTORY:
260
+ self.bitmap = (self.bitmap << fwd) & ((1 << DEDUP_HISTORY) - 1)
261
+ else:
262
+ self.bitmap = 0
263
+ self.tag_frontier = tag
264
+ self.bitmap |= 1
265
+ return True
266
+
267
+
268
+ @dataclass
269
+ class SubscriberRoot:
270
+ """Groups subscribers sharing the same subscription name/pattern."""
271
+
272
+ name: str
273
+ is_pattern: bool
274
+ subscribers: list[Any] = field(default_factory=list) # list[SubscriberImpl]
275
+ needs_scouting: bool = False
276
+ scout_task: asyncio.Task[None] | None = None
277
+
278
+
279
+ @dataclass
280
+ class Coupling:
281
+ """Links a topic to a subscriber root with pattern substitutions."""
282
+
283
+ root: SubscriberRoot
284
+ substitutions: list[tuple[str, int]]
285
+
286
+
287
+ @dataclass
288
+ class SharedSubjectListener:
289
+ """One transport listener shared by all topics bound to the same subject-ID."""
290
+
291
+ handle: Closable
292
+ owners: set[Topic] = field(default_factory=set)
293
+
294
+
295
+ @dataclass
296
+ class SharedSubjectWriter:
297
+ """One transport writer shared by all topics bound to the same subject-ID."""
298
+
299
+ handle: SubjectWriter
300
+ owners: set[Topic] = field(default_factory=set)
301
+
302
+
303
+ @dataclass(frozen=True)
304
+ class _TopicFlyweight(Topic):
305
+ """Short-lived topic view for unknown gossip."""
306
+
307
+ _topic_hash: int
308
+ _name: str
309
+
310
+ @property
311
+ def hash(self) -> int:
312
+ return self._topic_hash
313
+
314
+ @property
315
+ def name(self) -> str:
316
+ return self._name
317
+
318
+ def match(self, pattern: str) -> list[tuple[str, int]] | None:
319
+ return match_pattern(pattern, self._name)
320
+
321
+
322
+ @dataclass
323
+ class _MonitorHandle(Closable):
324
+ _node: NodeImpl | None
325
+ _callback_id: int
326
+
327
+ def close(self) -> None:
328
+ node = self._node
329
+ if node is None:
330
+ return
331
+ node.monitor_unregister(self._callback_id)
332
+ self._node = None
333
+
334
+
335
+ @dataclass
336
+ class PublishTracker:
337
+ """Tracks a pending reliable publication awaiting ACKs."""
338
+
339
+ tag: int
340
+ deadline_ns: int
341
+ ack_event: asyncio.Event
342
+ acknowledged: bool = False
343
+ data: bytes | None = None
344
+ ack_timeout: float = ACK_BASELINE_DEFAULT_TIMEOUT
345
+ compromised: bool = False
346
+ remaining: set[int] = field(default_factory=set)
347
+ associations: list[Association] = field(default_factory=list)
348
+
349
+ def on_ack(self, remote_id: int, positive: bool) -> None:
350
+ self.remaining.discard(remote_id)
351
+ self.acknowledged = self.acknowledged or positive
352
+ if not self.remaining and self.acknowledged:
353
+ self.ack_event.set()
354
+
355
+
356
+ # =====================================================================================================================
357
+ # Topic
358
+ # =====================================================================================================================
359
+
360
+
361
+ class TopicImpl(Topic):
362
+
363
+ def __init__(self, node: NodeImpl, name: str, evictions: int, now: float) -> None:
364
+ self._node = node
365
+ self._name = name
366
+ self._topic_hash = rapidhash(name)
367
+ self.evictions = evictions
368
+ self.ts_origin = now
369
+ self.ts_animated = now
370
+ self._pub_tag_baseline = int.from_bytes(os.urandom(8), "little")
371
+ self._pub_seqno = 0
372
+ self.pub_count = 0
373
+ self.pub_writer: SubjectWriter | None = None
374
+ self.sub_listener: Closable | None = None
375
+ self.couplings: list[Coupling] = []
376
+ self.is_implicit = True
377
+ self.associations: dict[int, Association] = {}
378
+ self.dedup: dict[int, DedupState] = {}
379
+ self.publish_futures: dict[int, PublishTracker] = {}
380
+ self.request_futures: dict[int, ResponseStreamImpl] = {} # tag -> ResponseStreamImpl
381
+ self.gossip_task: asyncio.Task[None] | None = None
382
+ self.gossip_deadline: float | None = None
383
+ self.gossip_task_is_periodic = False
384
+ self.gossip_counter = 0
385
+
386
+ # -- Topic ABC --
387
+ @property
388
+ def hash(self) -> int:
389
+ return self._topic_hash
390
+
391
+ @property
392
+ def name(self) -> str:
393
+ return self._name
394
+
395
+ def match(self, pattern: str) -> list[tuple[str, int]] | None:
396
+ return match_pattern(pattern, self._name)
397
+
398
+ # -- Internal --
399
+ @property
400
+ def subject_id(self) -> int:
401
+ return compute_subject_id(self._topic_hash, self.evictions, self._node.transport.subject_id_modulus)
402
+
403
+ def lage(self, now: float) -> int:
404
+ return log_age(self.ts_origin, now)
405
+
406
+ def merge_lage(self, now: float, remote_lage: int) -> None:
407
+ """Shift ts_origin backward if the remote claims an older origin."""
408
+ self.ts_origin = min(self.ts_origin, now - lage_to_seconds(remote_lage))
409
+
410
+ def animate(self, ts: float) -> None:
411
+ self.ts_animated = ts
412
+ if self.is_implicit:
413
+ self._node.touch_implicit_topic(self)
414
+
415
+ def next_tag(self) -> int:
416
+ tag = (self._pub_tag_baseline + self._pub_seqno) & ((1 << 64) - 1)
417
+ self._pub_seqno += 1
418
+ return tag
419
+
420
+ @property
421
+ def pub_seqno(self) -> int:
422
+ return self._pub_seqno
423
+
424
+ def tag_seqno(self, tag: int) -> int:
425
+ return (tag - self._pub_tag_baseline) & U64_MASK
426
+
427
+ def ensure_writer(self) -> SubjectWriter:
428
+ if self.pub_writer is None:
429
+ sid = self.subject_id
430
+ self.pub_writer = self._node.acquire_subject_writer(self, sid)
431
+ _logger.info("Writer acquired for '%s' sid=%d", self._name, sid)
432
+ return self.pub_writer
433
+
434
+ def ensure_listener(self) -> None:
435
+ if self.sub_listener is None and self.couplings:
436
+ sid = self.subject_id
437
+ self.sub_listener = self._node.acquire_subject_listener(self, sid)
438
+ _logger.info("Listener acquired for '%s' sid=%d", self._name, sid)
439
+
440
+ def sync_listener(self) -> None:
441
+ if self.couplings:
442
+ self.ensure_listener()
443
+ elif self.sub_listener is not None:
444
+ self._node.release_subject_listener(self, self.subject_id)
445
+ self.sub_listener = None
446
+ _logger.info("Listener released for '%s'", self._name)
447
+
448
+ def release_transport_handles(self) -> None:
449
+ sid = self.subject_id
450
+ if self.pub_writer is not None:
451
+ self._node.release_subject_writer(self, sid)
452
+ self.pub_writer = None
453
+ if self.sub_listener is not None:
454
+ self._node.release_subject_listener(self, sid)
455
+ self.sub_listener = None
456
+
457
+ def compute_is_implicit(self) -> bool:
458
+ has_verbatim_sub = any(not c.root.is_pattern for c in self.couplings)
459
+ return self.pub_count == 0 and not has_verbatim_sub
460
+
461
+ def sync_implicit(self) -> None:
462
+ """Sync implicitness and transport state with the reference state machine."""
463
+ self._node.sync_topic_lifecycle(self)
464
+
465
+
466
+ def log_age(origin: float, now: float) -> int:
467
+ diff = int(now - origin)
468
+ if diff <= 0:
469
+ return -1
470
+ return int(math.log2(diff))
471
+
472
+
473
+ def lage_to_seconds(lage: int) -> float:
474
+ if lage < 0:
475
+ return 0.0
476
+ return float(1 << min(LAGE_MAX, lage))
477
+
478
+
479
+ def left_wins(l_lage: int, l_hash: int, r_lage: int, r_hash: int) -> bool:
480
+ return l_lage > r_lage if l_lage != r_lage else l_hash < r_hash
481
+
482
+
483
+ # =====================================================================================================================
484
+ # Node
485
+ # =====================================================================================================================
486
+
487
+
488
+ class NodeImpl(Node):
489
+ def __init__(self, transport: Transport, *, home: str, namespace: str) -> None:
490
+ self.transport = transport
491
+ self._home = home
492
+ self._namespace = namespace
493
+ self._remaps: dict[str, str] = {}
494
+ self._closed = False
495
+ self.loop = asyncio.get_running_loop()
496
+ self._now_mono = time.monotonic()
497
+ self._monitor_callbacks: dict[int, Callable[[Topic], None]] = {}
498
+ self._next_monitor_callback_id = 0
499
+
500
+ # Topic indexes.
501
+ self.topics_by_name: dict[str, TopicImpl] = {}
502
+ self.topics_by_hash: dict[int, TopicImpl] = {}
503
+ self.topics_by_subject_id: dict[int, TopicImpl] = {} # non-pinned only
504
+
505
+ # Subscriber roots.
506
+ self.sub_roots_verbatim: dict[str, SubscriberRoot] = {}
507
+ self.sub_roots_pattern: dict[str, SubscriberRoot] = {}
508
+
509
+ # Respond futures for reliable responses.
510
+ self.respond_futures: dict[tuple[int, ...], RespondTracker] = {}
511
+
512
+ # Compute broadcast and gossip shard subject IDs.
513
+ modulus = transport.subject_id_modulus
514
+ sid_max = SUBJECT_ID_PINNED_MAX + modulus
515
+ self.broadcast_subject_id = (1 << (int(math.log2(sid_max)) + 1)) - 1
516
+ self.gossip_shard_count = self.broadcast_subject_id - (sid_max + 1)
517
+ assert self.gossip_shard_count > 0
518
+
519
+ # Set up broadcast writer and listener.
520
+ self.broadcast_writer = transport.subject_advertise(self.broadcast_subject_id)
521
+
522
+ def broadcast_handler(arrival: TransportArrival) -> None:
523
+ self.on_subject_arrival(self.broadcast_subject_id, arrival)
524
+
525
+ self.broadcast_listener = transport.subject_listen(self.broadcast_subject_id, broadcast_handler)
526
+
527
+ # Gossip shard state: lazily created per shard.
528
+ self.gossip_shard_writers: dict[int, SubjectWriter] = {}
529
+ self.gossip_shard_listeners: dict[int, Closable] = {}
530
+ self.shared_subject_writers: dict[int, SharedSubjectWriter] = {}
531
+ self.shared_subject_listeners: dict[int, SharedSubjectListener] = {}
532
+
533
+ # Register unicast handler.
534
+ transport.unicast_listen(self.on_unicast_arrival)
535
+
536
+ # Implicit topic GC task, driven by the earliest implicit-topic expiry.
537
+ self._implicit_topics: OrderedDict[TopicImpl, None] = OrderedDict()
538
+ self._implicit_gc_wakeup = asyncio.Event()
539
+ self._gc_task = self.loop.create_task(self.implicit_gc_loop())
540
+
541
+ _logger.info(
542
+ "Node init home='%s' ns='%s' broadcast_sid=%d shards=%d",
543
+ home,
544
+ namespace,
545
+ self.broadcast_subject_id,
546
+ self.gossip_shard_count,
547
+ )
548
+
549
+ # -- Node ABC --
550
+ @property
551
+ def home(self) -> str:
552
+ return self._home
553
+
554
+ @property
555
+ def namespace(self) -> str:
556
+ return self._namespace
557
+
558
+ def remap(self, spec: str | dict[str, str]) -> None:
559
+ if isinstance(spec, str):
560
+ spec = dict(x.split("=", 1) for x in spec.split() if "=" in x)
561
+ assert isinstance(spec, dict)
562
+ for from_name, to_name in spec.items():
563
+ if key := _name_normalize(from_name):
564
+ self._remaps[key] = to_name
565
+
566
+ def advertise(self, name: str) -> Publisher:
567
+ from ._publisher import PublisherImpl
568
+
569
+ resolved, pin, verbatim = resolve_name(name, self._home, self._namespace, self._remaps)
570
+ if not verbatim:
571
+ raise ValueError("Cannot advertise on a pattern name")
572
+ topic = self.topic_ensure(resolved, pin)
573
+ topic.pub_count += 1
574
+ topic.sync_implicit()
575
+ topic.ensure_writer()
576
+ _logger.info("Advertise '%s' -> '%s' sid=%d", name, resolved, topic.subject_id)
577
+ return PublisherImpl(self, topic)
578
+
579
+ def subscribe(self, name: str, *, reordering_window: float | None = None) -> Subscriber:
580
+ from ._subscriber import SubscriberImpl
581
+
582
+ resolved, pin, verbatim = resolve_name(name, self._home, self._namespace, self._remaps)
583
+ if pin is not None and not verbatim:
584
+ raise ValueError("Pattern names cannot be pinned")
585
+
586
+ # Ensure subscriber root.
587
+ if verbatim:
588
+ root = self.sub_roots_verbatim.get(resolved)
589
+ if root is None:
590
+ root = SubscriberRoot(name=resolved, is_pattern=False)
591
+ self.sub_roots_verbatim[resolved] = root
592
+ else:
593
+ root = self.sub_roots_pattern.get(resolved)
594
+ if root is None:
595
+ root = SubscriberRoot(name=resolved, is_pattern=True, needs_scouting=True)
596
+ self.sub_roots_pattern[resolved] = root
597
+
598
+ subscriber = SubscriberImpl(self, root, resolved, verbatim, reordering_window)
599
+ root.subscribers.append(subscriber)
600
+
601
+ if verbatim:
602
+ # Ensure topic exists and couple.
603
+ topic = self.topic_ensure(resolved, pin)
604
+ self.couple_topic_root(topic, root)
605
+ topic.sync_implicit()
606
+ else:
607
+ # Pattern subscriber: couple with all existing matching topics and scout once per root.
608
+ for topic in list(self.topics_by_name.values()):
609
+ self.couple_topic_root(topic, root)
610
+ topic.sync_implicit()
611
+ self._ensure_root_scouting(root)
612
+
613
+ _logger.info("Subscribe '%s' -> '%s' verbatim=%s", name, resolved, verbatim)
614
+ return subscriber
615
+
616
+ def monitor(self, callback: Callable[[Topic], None]) -> Closable:
617
+ callback_id = self._next_monitor_callback_id
618
+ self._next_monitor_callback_id += 1
619
+ self._monitor_callbacks[callback_id] = callback
620
+ return _MonitorHandle(self, callback_id)
621
+
622
+ def monitor_unregister(self, callback_id: int) -> None:
623
+ self._monitor_callbacks.pop(callback_id, None)
624
+
625
+ def _notify_monitors(self, topic: Topic) -> None:
626
+ for callback in list(self._monitor_callbacks.values()):
627
+ try:
628
+ callback(topic)
629
+ except Exception:
630
+ _logger.exception("monitor() callback failed for %s", topic)
631
+
632
+ async def scout(self, pattern: str) -> None:
633
+ resolved, pin, _ = resolve_name(pattern, self._home, self._namespace, self._remaps)
634
+ if pin is not None:
635
+ raise ValueError("Cannot scout a pinned name/pattern")
636
+ try:
637
+ await self._transmit_scout(resolved)
638
+ except SendError:
639
+ raise
640
+ except Exception as ex:
641
+ raise SendError(f"Scout send failed for '{resolved}'") from ex
642
+
643
+ # -- Topic Management --
644
+
645
+ def topic_ensure(self, name: str, pin: int | None) -> TopicImpl:
646
+ """Get or create a topic by resolved name."""
647
+ topic = self.topics_by_name.get(name)
648
+ if topic is not None:
649
+ return topic
650
+ now = time.monotonic()
651
+ evictions = 0
652
+ if pin is not None:
653
+ evictions = 0xFFFFFFFF - pin
654
+ topic = TopicImpl(self, name, evictions, now)
655
+ self.topics_by_name[name] = topic
656
+ self.topics_by_hash[topic.hash] = topic
657
+ self.ensure_gossip_shard(self.gossip_shard_subject_id(topic.hash))
658
+ self.touch_implicit_topic(topic)
659
+ self.topic_allocate(topic, evictions, now)
660
+ # Couple with existing pattern subscriber roots.
661
+ for root in self.sub_roots_pattern.values():
662
+ self.couple_topic_root(topic, root)
663
+ topic.sync_listener()
664
+ self.notify_implicit_gc()
665
+ _logger.info("Topic created '%s' hash=%016x sid=%d", name, topic.hash, topic.subject_id)
666
+ return topic
667
+
668
+ def topic_allocate(self, topic: TopicImpl, new_evictions: int, now: float) -> None:
669
+ """Iterative subject-ID allocation with collision resolution. Mirrors topic_allocate() in cy.c."""
670
+ # Work queue: list of (topic, new_evictions) pairs to process.
671
+ work: list[tuple[TopicImpl, int]] = [(topic, new_evictions)]
672
+ while work:
673
+ t, ev = work.pop(0)
674
+ # Remove from subject-ID index first.
675
+ old_sid = t.subject_id
676
+ if old_sid in self.topics_by_subject_id and self.topics_by_subject_id[old_sid] is t:
677
+ del self.topics_by_subject_id[old_sid]
678
+
679
+ if ev >= EVICTIONS_PINNED_MIN:
680
+ # Pinned topic: no collision detection, shared subject-IDs are fine.
681
+ t.release_transport_handles()
682
+ t.evictions = ev
683
+ t.sync_listener()
684
+ self.schedule_gossip_urgent(t)
685
+ continue
686
+
687
+ modulus = self.transport.subject_id_modulus
688
+ new_sid = compute_subject_id(t.hash, ev, modulus)
689
+ collider = self.topics_by_subject_id.get(new_sid)
690
+
691
+ if collider is not None and collider is t:
692
+ collider = None # same topic, no real collision
693
+
694
+ if collider is None:
695
+ # No collision, install.
696
+ t.release_transport_handles()
697
+ t.evictions = ev
698
+ self.topics_by_subject_id[new_sid] = t
699
+ t.sync_listener()
700
+ self.schedule_gossip_urgent(t)
701
+ elif left_wins(t.lage(now), t.hash, collider.lage(now), collider.hash):
702
+ # Our topic wins: take the slot, evict the collider.
703
+ t.release_transport_handles()
704
+ t.evictions = ev
705
+ del self.topics_by_subject_id[new_sid]
706
+ self.topics_by_subject_id[new_sid] = t
707
+ if collider.pub_writer is not None:
708
+ t.pub_writer = self.acquire_subject_writer(t, new_sid)
709
+ t.sync_listener()
710
+ self.schedule_gossip_urgent(t)
711
+ # Schedule collider for reallocation.
712
+ collider.release_transport_handles()
713
+ work.append((collider, collider.evictions + 1))
714
+ else:
715
+ # Our topic loses: increment evictions and retry.
716
+ work.append((t, ev + 1))
717
+
718
+ def sync_topic_lifecycle(self, topic: TopicImpl) -> None:
719
+ implicit = topic.compute_is_implicit()
720
+ if implicit != topic.is_implicit:
721
+ topic.is_implicit = implicit
722
+ if implicit:
723
+ self.touch_implicit_topic(topic)
724
+ self._cancel_gossip(topic)
725
+ else:
726
+ self.discard_implicit_topic(topic)
727
+ self.schedule_gossip_urgent(topic)
728
+ elif (not implicit) and (topic.gossip_task is None):
729
+ self.schedule_gossip(topic)
730
+ topic.sync_listener()
731
+ self.notify_implicit_gc()
732
+
733
+ def touch_implicit_topic(self, topic: TopicImpl) -> None:
734
+ self._implicit_topics[topic] = None
735
+ self._implicit_topics.move_to_end(topic, last=False)
736
+ self.notify_implicit_gc()
737
+
738
+ def discard_implicit_topic(self, topic: TopicImpl) -> None:
739
+ if topic in self._implicit_topics:
740
+ del self._implicit_topics[topic]
741
+ self.notify_implicit_gc()
742
+
743
+ def decouple_topic_root(
744
+ self, topic: TopicImpl, root: SubscriberRoot, *, silenced: bool = True, sync_lifecycle: bool = True
745
+ ) -> None:
746
+ from ._subscriber import SubscriberImpl
747
+
748
+ topic.couplings = [c for c in topic.couplings if c.root is not root]
749
+ for sub in root.subscribers:
750
+ if isinstance(sub, SubscriberImpl):
751
+ sub.forget_topic_reordering(topic.hash, silenced=silenced)
752
+ if sync_lifecycle:
753
+ self.sync_topic_lifecycle(topic)
754
+
755
+ @staticmethod
756
+ def forget_association(topic: TopicImpl, assoc: Association) -> None:
757
+ current = topic.associations.get(assoc.remote_id)
758
+ if current is assoc:
759
+ del topic.associations[assoc.remote_id]
760
+
761
+ @staticmethod
762
+ def publish_tracker_release(topic: TopicImpl, tracker: PublishTracker) -> None:
763
+ seqno = topic.tag_seqno(tracker.tag)
764
+ for assoc in tracker.associations:
765
+ if assoc.remote_id in tracker.remaining and seqno >= assoc.seqno_witness and not tracker.compromised:
766
+ assoc.slack += 1
767
+ if assoc.pending_count > 0:
768
+ assoc.pending_count -= 1
769
+ if assoc.slack >= ASSOC_SLACK_LIMIT and assoc.pending_count == 0:
770
+ NodeImpl.forget_association(topic, assoc)
771
+ tracker.associations.clear()
772
+ tracker.remaining.clear()
773
+
774
+ @staticmethod
775
+ def prepare_publish_tracker(topic: TopicImpl, tag: int, deadline_ns: int, data: bytes) -> PublishTracker:
776
+ tracker = PublishTracker(
777
+ tag=tag,
778
+ deadline_ns=deadline_ns,
779
+ ack_event=asyncio.Event(),
780
+ data=data,
781
+ )
782
+ tracker.ack_timeout = ACK_BASELINE_DEFAULT_TIMEOUT
783
+ for assoc in sorted(topic.associations.values(), key=lambda x: x.remote_id):
784
+ if assoc.slack < ASSOC_SLACK_LIMIT:
785
+ tracker.associations.append(assoc)
786
+ tracker.remaining.add(assoc.remote_id)
787
+ assoc.pending_count += 1
788
+ return tracker
789
+
790
+ @staticmethod
791
+ def couple_topic_root(topic: TopicImpl, root: SubscriberRoot) -> None:
792
+ """Create a coupling between a topic and a subscriber root if not already coupled."""
793
+ for c in topic.couplings:
794
+ if c.root is root:
795
+ return # already coupled
796
+ subs = match_pattern(root.name, topic.name) if root.is_pattern else ([] if root.name == topic.name else None)
797
+ if subs is not None:
798
+ topic.couplings.append(Coupling(root=root, substitutions=subs))
799
+ _logger.debug("Coupled '%s' <-> root '%s'", topic.name, root.name)
800
+
801
+ # -- Gossip --
802
+
803
+ def gossip_shard_subject_id(self, topic_hash: int) -> int:
804
+ modulus = self.transport.subject_id_modulus
805
+ sid_max = SUBJECT_ID_PINNED_MAX + modulus
806
+ shard_index = topic_hash % self.gossip_shard_count
807
+ return sid_max + 1 + shard_index
808
+
809
+ def ensure_gossip_shard(self, shard_sid: int) -> SubjectWriter:
810
+ writer = self.gossip_shard_writers.get(shard_sid)
811
+ if writer is None:
812
+ writer = self.transport.subject_advertise(shard_sid)
813
+ self.gossip_shard_writers[shard_sid] = writer
814
+
815
+ def handler(arrival: TransportArrival) -> None:
816
+ self.on_subject_arrival(shard_sid, arrival)
817
+
818
+ self.gossip_shard_listeners[shard_sid] = self.transport.subject_listen(shard_sid, handler)
819
+ _logger.debug("Gossip shard writer/listener for sid=%d", shard_sid)
820
+ return writer
821
+
822
+ def acquire_subject_writer(self, topic: TopicImpl, subject_id: int) -> SubjectWriter:
823
+ entry = self.shared_subject_writers.get(subject_id)
824
+ if entry is None:
825
+ entry = SharedSubjectWriter(handle=self.transport.subject_advertise(subject_id))
826
+ self.shared_subject_writers[subject_id] = entry
827
+ _logger.debug("Shared subject writer created sid=%d", subject_id)
828
+ entry.owners.add(topic)
829
+ return entry.handle
830
+
831
+ def release_subject_writer(self, topic: TopicImpl, subject_id: int) -> None:
832
+ entry = self.shared_subject_writers.get(subject_id)
833
+ if entry is None:
834
+ return
835
+ entry.owners.discard(topic)
836
+ if not entry.owners:
837
+ entry.handle.close()
838
+ del self.shared_subject_writers[subject_id]
839
+ _logger.debug("Shared subject writer released sid=%d", subject_id)
840
+
841
+ def acquire_subject_listener(self, topic: TopicImpl, subject_id: int) -> Closable:
842
+ entry = self.shared_subject_listeners.get(subject_id)
843
+ if entry is None:
844
+
845
+ def handler(arrival: TransportArrival) -> None:
846
+ self.on_subject_arrival(subject_id, arrival)
847
+
848
+ entry = SharedSubjectListener(handle=self.transport.subject_listen(subject_id, handler))
849
+ self.shared_subject_listeners[subject_id] = entry
850
+ _logger.debug("Shared subject listener created sid=%d", subject_id)
851
+ entry.owners.add(topic)
852
+ return entry.handle
853
+
854
+ def release_subject_listener(self, topic: TopicImpl, subject_id: int) -> None:
855
+ entry = self.shared_subject_listeners.get(subject_id)
856
+ if entry is None:
857
+ return
858
+ entry.owners.discard(topic)
859
+ if not entry.owners:
860
+ entry.handle.close()
861
+ del self.shared_subject_listeners[subject_id]
862
+ _logger.debug("Shared subject listener released sid=%d", subject_id)
863
+
864
+ def schedule_gossip(self, topic: TopicImpl) -> None:
865
+ """Start periodic gossip for an explicit topic."""
866
+ if topic.gossip_task is not None:
867
+ return # already scheduled
868
+ self._reschedule_gossip_periodic(topic, suppressed=False)
869
+
870
+ @staticmethod
871
+ def _cancel_gossip(topic: TopicImpl) -> None:
872
+ if topic.gossip_task is not None:
873
+ topic.gossip_task.cancel()
874
+ topic.gossip_task = None
875
+ topic.gossip_deadline = None
876
+
877
+ def _schedule_gossip_task(self, topic: TopicImpl, deadline: float, *, periodic: bool) -> None:
878
+ self._cancel_gossip(topic)
879
+ topic.gossip_task_is_periodic = periodic
880
+ topic.gossip_deadline = deadline
881
+ topic.gossip_task = self.loop.create_task(self._gossip_wait(topic, deadline))
882
+
883
+ def _reschedule_gossip_periodic(self, topic: TopicImpl, *, suppressed: bool) -> None:
884
+ if topic.is_implicit:
885
+ self._cancel_gossip(topic)
886
+ return
887
+ dither = GOSSIP_PERIOD / GOSSIP_PERIOD_DITHER_RATIO
888
+ if suppressed:
889
+ delay_min = GOSSIP_PERIOD + dither
890
+ delay_max = GOSSIP_PERIOD * 3
891
+ else:
892
+ delay_min = GOSSIP_PERIOD - dither
893
+ delay_max = GOSSIP_PERIOD + dither
894
+ if topic.gossip_counter < GOSSIP_BROADCAST_RATIO:
895
+ delay_min /= 16
896
+ delay = random.uniform(max(0.0, delay_min), max(delay_min, delay_max))
897
+ self._schedule_gossip_task(topic, time.monotonic() + delay, periodic=True)
898
+
899
+ def schedule_gossip_urgent(self, topic: TopicImpl) -> None:
900
+ """Schedule an urgent gossip, preserving an earlier pending deadline when possible."""
901
+ at = time.monotonic() + (random.random() * GOSSIP_URGENT_DELAY_MAX)
902
+ if (topic.gossip_task is None) or (topic.gossip_deadline is None) or (at < topic.gossip_deadline):
903
+ self._schedule_gossip_task(topic, at, periodic=False)
904
+ else:
905
+ topic.gossip_task_is_periodic = False
906
+
907
+ async def _gossip_wait(self, topic: TopicImpl, deadline: float) -> None:
908
+ try:
909
+ await asyncio.sleep(max(0.0, deadline - time.monotonic()))
910
+ except asyncio.CancelledError:
911
+ return
912
+ if topic.gossip_task is not asyncio.current_task():
913
+ return
914
+ topic.gossip_task = None
915
+ topic.gossip_deadline = None
916
+ if self._closed:
917
+ return
918
+ if topic.gossip_task_is_periodic:
919
+ await self._gossip_event_periodic(topic)
920
+ else:
921
+ await self._gossip_event_urgent(topic)
922
+
923
+ async def _gossip_event_urgent(self, topic: TopicImpl) -> None:
924
+ self._reschedule_gossip_periodic(topic, suppressed=False)
925
+ topic.gossip_counter = 0
926
+ await self.send_gossip(topic, broadcast=True)
927
+
928
+ async def _gossip_event_periodic(self, topic: TopicImpl) -> None:
929
+ self._reschedule_gossip_periodic(topic, suppressed=False)
930
+ broadcast = (topic.gossip_counter < GOSSIP_BROADCAST_RATIO) or (
931
+ (topic.gossip_counter % GOSSIP_BROADCAST_RATIO) == 0
932
+ )
933
+ topic.gossip_counter += 1
934
+ await self.send_gossip(topic, broadcast=broadcast)
935
+
936
+ async def send_gossip(self, topic: TopicImpl, *, broadcast: bool = False) -> None:
937
+ now = time.monotonic()
938
+ lage = topic.lage(now)
939
+ name_bytes = topic.name.encode("utf-8")
940
+ hdr = GossipHeader(
941
+ topic_log_age=lage,
942
+ topic_hash=topic.hash,
943
+ topic_evictions=topic.evictions,
944
+ name_len=len(name_bytes),
945
+ )
946
+ payload = hdr.serialize() + name_bytes
947
+ deadline = Instant.now() + 1.0
948
+ try:
949
+ if broadcast:
950
+ await self.broadcast_writer(deadline, Priority.NOMINAL, payload)
951
+ else:
952
+ shard_sid = self.gossip_shard_subject_id(topic.hash)
953
+ writer = self.ensure_gossip_shard(shard_sid)
954
+ await writer(deadline, Priority.NOMINAL, payload)
955
+ _logger.debug("Gossip sent '%s' broadcast=%s", topic.name, broadcast)
956
+ except (SendError, OSError) as e:
957
+ _logger.warning("Gossip send failed for '%s': %s", topic.name, e)
958
+
959
+ async def send_gossip_unicast(
960
+ self,
961
+ topic: TopicImpl,
962
+ remote_id: int,
963
+ priority: Priority = Priority.NOMINAL,
964
+ ) -> None:
965
+ now = time.monotonic()
966
+ lage = topic.lage(now)
967
+ name_bytes = topic.name.encode("utf-8")
968
+ hdr = GossipHeader(
969
+ topic_log_age=lage,
970
+ topic_hash=topic.hash,
971
+ topic_evictions=topic.evictions,
972
+ name_len=len(name_bytes),
973
+ )
974
+ payload = hdr.serialize() + name_bytes
975
+ deadline = Instant.now() + 1.0
976
+ try:
977
+ await self.transport.unicast(deadline, priority, remote_id, payload)
978
+ except (SendError, OSError) as e:
979
+ _logger.warning("Gossip unicast send failed for '%s': %s", topic.name, e)
980
+
981
+ # -- Scout --
982
+
983
+ async def _transmit_scout(self, pattern: str) -> None:
984
+ pattern_bytes = pattern.encode("utf-8")
985
+ hdr = ScoutHeader(pattern_len=len(pattern_bytes))
986
+ payload = hdr.serialize() + pattern_bytes
987
+ deadline = Instant.now() + 1.0
988
+ await self.broadcast_writer(deadline, Priority.NOMINAL, payload)
989
+ _logger.debug("Scout sent for pattern '%s'", pattern)
990
+
991
+ async def _send_scout_once(self, pattern: str) -> bool:
992
+ try:
993
+ await self._transmit_scout(pattern)
994
+ except Exception as e:
995
+ _logger.warning("Scout send failed for '%s': %s", pattern, e)
996
+ return False
997
+ return True
998
+
999
+ def _ensure_root_scouting(self, root: SubscriberRoot) -> None:
1000
+ if (not root.is_pattern) or (not root.needs_scouting) or (root.scout_task is not None):
1001
+ return
1002
+
1003
+ async def do_send() -> None:
1004
+ try:
1005
+ root.needs_scouting = not await self._send_scout_once(root.name)
1006
+ finally:
1007
+ root.scout_task = None
1008
+
1009
+ root.scout_task = self.loop.create_task(do_send())
1010
+
1011
+ def send_scout(self, pattern: str) -> None:
1012
+ """Send a scout message to discover topics matching a pattern."""
1013
+
1014
+ async def do_send() -> None:
1015
+ await self._send_scout_once(pattern)
1016
+
1017
+ self.loop.create_task(do_send())
1018
+
1019
+ # -- Message Dispatch --
1020
+
1021
+ def on_subject_arrival(self, subject_id: int, arrival: TransportArrival) -> None:
1022
+ """Handle an arrival on a subject (multicast)."""
1023
+ self.dispatch_arrival(arrival, subject_id=subject_id, unicast=False)
1024
+
1025
+ def on_unicast_arrival(self, arrival: TransportArrival) -> None:
1026
+ """Handle an arrival via unicast."""
1027
+ self.dispatch_arrival(arrival, subject_id=None, unicast=True)
1028
+
1029
+ def dispatch_arrival(self, arrival: TransportArrival, *, subject_id: int | None, unicast: bool) -> None:
1030
+ msg = arrival.message
1031
+ if len(msg) < HEADER_SIZE:
1032
+ _logger.debug("Drop short msg len=%d", len(msg))
1033
+ return
1034
+ hdr = deserialize_header(msg[:HEADER_SIZE])
1035
+ if hdr is None:
1036
+ _logger.debug("Drop bad header")
1037
+ return
1038
+ payload = msg[HEADER_SIZE:]
1039
+
1040
+ if isinstance(hdr, (MsgBeHeader, MsgRelHeader)):
1041
+ self.on_msg(arrival, hdr, payload, subject_id=subject_id, unicast=unicast)
1042
+ elif isinstance(hdr, (MsgAckHeader, MsgNackHeader)):
1043
+ if unicast:
1044
+ self.on_msg_ack(arrival, hdr)
1045
+ elif isinstance(hdr, (RspBeHeader, RspRelHeader)):
1046
+ if unicast:
1047
+ self.on_rsp(arrival, hdr, payload)
1048
+ elif isinstance(hdr, (RspAckHeader, RspNackHeader)):
1049
+ if unicast:
1050
+ self.on_rsp_ack(arrival, hdr)
1051
+ elif isinstance(hdr, GossipHeader):
1052
+ if hdr.name_len > TOPIC_NAME_MAX or len(payload) < hdr.name_len:
1053
+ return
1054
+ scope = (
1055
+ GossipScope.UNICAST
1056
+ if unicast
1057
+ else GossipScope.BROADCAST if subject_id == self.broadcast_subject_id else GossipScope.SHARDED
1058
+ )
1059
+ self.on_gossip(arrival.timestamp.s, hdr, payload, scope)
1060
+ elif isinstance(hdr, ScoutHeader):
1061
+ self.on_scout(arrival, hdr, payload)
1062
+
1063
+ def on_msg(
1064
+ self,
1065
+ arrival: TransportArrival,
1066
+ hdr: MsgBeHeader | MsgRelHeader,
1067
+ payload: bytes,
1068
+ *,
1069
+ subject_id: int | None,
1070
+ unicast: bool,
1071
+ ) -> None:
1072
+ if (
1073
+ (not unicast)
1074
+ and (subject_id is not None)
1075
+ and (subject_id <= (SUBJECT_ID_PINNED_MAX + self.transport.subject_id_modulus))
1076
+ and (
1077
+ compute_subject_id(hdr.topic_hash, hdr.topic_evictions, self.transport.subject_id_modulus) != subject_id
1078
+ )
1079
+ ):
1080
+ _logger.debug("MSG drop subject mismatch sid=%d hash=%016x", subject_id, hdr.topic_hash)
1081
+ return
1082
+ topic = self.topics_by_hash.get(hdr.topic_hash)
1083
+ reliable = isinstance(hdr, MsgRelHeader)
1084
+ accepted = False
1085
+ if topic is not None:
1086
+ self.on_gossip_known(topic, hdr.topic_evictions, hdr.topic_log_age, arrival.timestamp.s, GossipScope.INLINE)
1087
+ accepted = self.accept_message(topic, arrival, hdr.tag, payload, reliable)
1088
+ else:
1089
+ self.on_gossip_unknown(hdr.topic_hash, hdr.topic_evictions, hdr.topic_log_age, arrival.timestamp.s)
1090
+ _logger.debug("MSG drop unknown hash=%016x", hdr.topic_hash)
1091
+
1092
+ has_subscribers = (topic is not None) and bool(topic.couplings)
1093
+ if reliable and (accepted or (unicast and not has_subscribers)):
1094
+ self.send_msg_ack(arrival.remote_id, hdr.topic_hash, hdr.tag, arrival.timestamp, arrival.priority, accepted)
1095
+
1096
+ def accept_message(
1097
+ self,
1098
+ topic: TopicImpl,
1099
+ arrival: TransportArrival,
1100
+ tag: int,
1101
+ payload: bytes,
1102
+ reliable: bool,
1103
+ ) -> bool:
1104
+ topic.animate(arrival.timestamp.s)
1105
+ if not topic.couplings:
1106
+ if reliable:
1107
+ dedup = topic.dedup.get(arrival.remote_id)
1108
+ if dedup is not None and (arrival.timestamp.s - dedup.last_active) > SESSION_LIFETIME:
1109
+ del topic.dedup[arrival.remote_id]
1110
+ dedup = None
1111
+ return dedup.check(tag) if dedup is not None else False
1112
+ return False
1113
+
1114
+ if reliable:
1115
+ dedup = topic.dedup.get(arrival.remote_id)
1116
+ if dedup is not None and (arrival.timestamp.s - dedup.last_active) > SESSION_LIFETIME:
1117
+ del topic.dedup[arrival.remote_id]
1118
+ dedup = None
1119
+ if dedup is None:
1120
+ dedup = DedupState(tag_frontier=tag)
1121
+ topic.dedup[arrival.remote_id] = dedup
1122
+ if not dedup.check_and_record(tag, arrival.timestamp.s):
1123
+ _logger.debug("MSG dedup drop hash=%016x tag=%d", topic.hash, tag)
1124
+ return True
1125
+
1126
+ from ._subscriber import BreadcrumbImpl
1127
+
1128
+ breadcrumb = BreadcrumbImpl(
1129
+ node=self,
1130
+ remote_id=arrival.remote_id,
1131
+ topic=topic,
1132
+ message_tag=tag,
1133
+ initial_priority=arrival.priority,
1134
+ )
1135
+ return self.deliver_to_subscribers(topic, arrival, breadcrumb, payload, tag)
1136
+
1137
+ @staticmethod
1138
+ def deliver_to_subscribers(
1139
+ topic: TopicImpl,
1140
+ arrival: TransportArrival,
1141
+ breadcrumb: Breadcrumb,
1142
+ payload: bytes,
1143
+ tag: int,
1144
+ ) -> bool:
1145
+ from ._api import Arrival
1146
+ from ._subscriber import SubscriberImpl
1147
+
1148
+ arr = Arrival(
1149
+ timestamp=arrival.timestamp,
1150
+ breadcrumb=breadcrumb,
1151
+ message=payload,
1152
+ )
1153
+ accepted = False
1154
+ for coupling in topic.couplings:
1155
+ for sub in coupling.root.subscribers:
1156
+ if isinstance(sub, SubscriberImpl) and not sub.closed:
1157
+ accepted = sub.deliver(arr, tag, arrival.remote_id) or accepted
1158
+ return accepted
1159
+
1160
+ def send_msg_ack(
1161
+ self,
1162
+ remote_id: int,
1163
+ topic_hash: int,
1164
+ tag: int,
1165
+ ts: Instant,
1166
+ priority: Priority,
1167
+ positive: bool,
1168
+ ) -> None:
1169
+ hdr: MsgAckHeader | MsgNackHeader
1170
+ hdr = (
1171
+ MsgAckHeader(topic_hash=topic_hash, tag=tag) if positive else MsgNackHeader(topic_hash=topic_hash, tag=tag)
1172
+ )
1173
+ payload = hdr.serialize()
1174
+ deadline = ts + ACK_TX_TIMEOUT
1175
+
1176
+ async def do_send() -> None:
1177
+ try:
1178
+ await self.transport.unicast(deadline, priority, remote_id, payload)
1179
+ except (SendError, OSError) as e:
1180
+ _logger.debug("ACK send failed: %s", e)
1181
+
1182
+ self.loop.create_task(do_send())
1183
+
1184
+ def on_msg_ack(self, arrival: TransportArrival, hdr: MsgAckHeader | MsgNackHeader) -> None:
1185
+ topic = self.topics_by_hash.get(hdr.topic_hash)
1186
+ if topic is None:
1187
+ return
1188
+ seqno = topic.tag_seqno(hdr.tag)
1189
+ if seqno >= topic.pub_seqno or (topic.pub_seqno - seqno) > ACK_SEQNO_MAX_LAG:
1190
+ return
1191
+ positive = isinstance(hdr, MsgAckHeader)
1192
+ remote_id = arrival.remote_id
1193
+
1194
+ assoc = topic.associations.get(remote_id)
1195
+ if assoc is None:
1196
+ if not positive:
1197
+ return
1198
+ assoc = Association(remote_id=remote_id, last_seen=arrival.timestamp.s)
1199
+ topic.associations[remote_id] = assoc
1200
+ assoc.last_seen = arrival.timestamp.s
1201
+ if seqno >= assoc.seqno_witness:
1202
+ assoc.slack = 0 if positive else ASSOC_SLACK_LIMIT
1203
+ assoc.seqno_witness = seqno
1204
+ if (not positive) and assoc.pending_count == 0:
1205
+ assoc.slack = 0
1206
+ self.forget_association(topic, assoc)
1207
+ return
1208
+
1209
+ tracker = topic.publish_futures.get(hdr.tag)
1210
+ if tracker is not None:
1211
+ tracker.on_ack(remote_id, positive)
1212
+
1213
+ def on_rsp(self, arrival: TransportArrival, hdr: RspBeHeader | RspRelHeader, payload: bytes) -> None:
1214
+ """Handle a response message (for RPC)."""
1215
+ ack = False
1216
+ topic = self.topics_by_hash.get(hdr.topic_hash)
1217
+ if topic is not None:
1218
+ stream = topic.request_futures.get(hdr.message_tag)
1219
+ if stream is not None:
1220
+ ack = stream.on_response(arrival, hdr, payload)
1221
+ if not ack and not isinstance(hdr, RspBeHeader):
1222
+ _logger.debug("RSP drop no matching request tag=%d", hdr.message_tag)
1223
+ elif topic is None or hdr.message_tag not in topic.request_futures:
1224
+ _logger.debug("RSP drop no matching request tag=%d", hdr.message_tag)
1225
+ if isinstance(hdr, RspRelHeader):
1226
+ self.send_rsp_ack(
1227
+ arrival.remote_id,
1228
+ hdr.message_tag,
1229
+ hdr.seqno,
1230
+ hdr.tag,
1231
+ hdr.topic_hash,
1232
+ arrival.timestamp,
1233
+ arrival.priority,
1234
+ ack,
1235
+ )
1236
+
1237
+ def on_rsp_ack(self, arrival: TransportArrival, hdr: RspAckHeader | RspNackHeader) -> None:
1238
+ """Handle a response ACK/NACK."""
1239
+ key = (arrival.remote_id, hdr.message_tag, hdr.topic_hash, hdr.seqno, hdr.tag)
1240
+ future = self.respond_futures.get(key)
1241
+ if future is not None:
1242
+ positive = isinstance(hdr, RspAckHeader)
1243
+ future.on_ack(positive)
1244
+
1245
+ def send_rsp_ack(
1246
+ self,
1247
+ remote_id: int,
1248
+ message_tag: int,
1249
+ seqno: int,
1250
+ tag: int,
1251
+ topic_hash: int,
1252
+ ts: Instant,
1253
+ priority: Priority,
1254
+ positive: bool,
1255
+ ) -> None:
1256
+ hdr: RspAckHeader | RspNackHeader
1257
+ if positive:
1258
+ hdr = RspAckHeader(tag=tag, seqno=seqno, topic_hash=topic_hash, message_tag=message_tag)
1259
+ else:
1260
+ hdr = RspNackHeader(tag=tag, seqno=seqno, topic_hash=topic_hash, message_tag=message_tag)
1261
+ payload = hdr.serialize()
1262
+ deadline = ts + ACK_TX_TIMEOUT
1263
+
1264
+ async def do_send() -> None:
1265
+ try:
1266
+ await self.transport.unicast(deadline, priority, remote_id, payload)
1267
+ except (SendError, OSError) as e:
1268
+ _logger.debug("RSP ACK send failed: %s", e)
1269
+
1270
+ self.loop.create_task(do_send())
1271
+
1272
+ def on_gossip(
1273
+ self,
1274
+ ts: float,
1275
+ hdr: GossipHeader,
1276
+ payload: bytes,
1277
+ scope: GossipScope,
1278
+ ) -> None:
1279
+ name = ""
1280
+ if hdr.name_len > 0:
1281
+ name = payload[: hdr.name_len].decode("utf-8", errors="replace")
1282
+
1283
+ topic = self.topics_by_hash.get(hdr.topic_hash)
1284
+
1285
+ # If unknown topic with a name, check for pattern subscriber matches.
1286
+ if topic is None and name:
1287
+ if scope in {GossipScope.UNICAST, GossipScope.BROADCAST}:
1288
+ topic = self.topic_subscribe_if_matching(
1289
+ name, hdr.topic_hash, hdr.topic_evictions, hdr.topic_log_age, ts
1290
+ )
1291
+ if topic is not None:
1292
+ self.on_gossip_known(topic, hdr.topic_evictions, hdr.topic_log_age, ts, scope)
1293
+ self._notify_monitors(topic)
1294
+ else:
1295
+ self.on_gossip_unknown(hdr.topic_hash, hdr.topic_evictions, hdr.topic_log_age, ts)
1296
+ self._notify_monitors(_TopicFlyweight(hdr.topic_hash, name))
1297
+
1298
+ def on_gossip_known(
1299
+ self,
1300
+ topic: TopicImpl,
1301
+ evictions: int,
1302
+ lage: int,
1303
+ now: float,
1304
+ scope: GossipScope,
1305
+ ) -> None:
1306
+ topic.animate(now)
1307
+ my_lage = topic.lage(now)
1308
+ if topic.evictions != evictions:
1309
+ win = my_lage > lage or (my_lage == lage and topic.evictions > evictions)
1310
+ topic.merge_lage(now, lage)
1311
+ if win:
1312
+ self.schedule_gossip_urgent(topic)
1313
+ else:
1314
+ self.topic_allocate(topic, evictions, now)
1315
+ if topic.evictions == evictions:
1316
+ self._reschedule_gossip_periodic(topic, suppressed=True)
1317
+ else:
1318
+ topic.merge_lage(now, lage)
1319
+ suppress = (
1320
+ (scope in {GossipScope.BROADCAST, GossipScope.SHARDED})
1321
+ and (topic.lage(now) == lage)
1322
+ and (topic.gossip_task_is_periodic or scope == GossipScope.BROADCAST)
1323
+ )
1324
+ if suppress:
1325
+ self._reschedule_gossip_periodic(topic, suppressed=True)
1326
+ topic.sync_listener()
1327
+
1328
+ def on_gossip_unknown(self, topic_hash: int, evictions: int, lage: int, now: float) -> None:
1329
+ modulus = self.transport.subject_id_modulus
1330
+ remote_sid = compute_subject_id(topic_hash, evictions, modulus)
1331
+ mine = self.topics_by_subject_id.get(remote_sid)
1332
+ if mine is None:
1333
+ return
1334
+ win = left_wins(mine.lage(now), mine.hash, lage, topic_hash)
1335
+ if win:
1336
+ self.schedule_gossip_urgent(mine)
1337
+ else:
1338
+ self.topic_allocate(mine, mine.evictions + 1, now)
1339
+
1340
+ def topic_subscribe_if_matching(
1341
+ self,
1342
+ name: str,
1343
+ topic_hash: int,
1344
+ evictions: int,
1345
+ lage: int,
1346
+ now: float,
1347
+ ) -> TopicImpl | None:
1348
+ """Create an implicit topic if any pattern subscriber matches the name."""
1349
+ # Validate that the hash matches the name to prevent corrupt gossip from creating inconsistencies.
1350
+ if rapidhash(name) != topic_hash:
1351
+ _logger.debug("Gossip hash mismatch for '%s': got %016x, expected %016x", name, topic_hash, rapidhash(name))
1352
+ return None
1353
+ matches = [root for pattern, root in self.sub_roots_pattern.items() if match_pattern(pattern, name) is not None]
1354
+ if matches:
1355
+ topic = TopicImpl(self, name, evictions, now)
1356
+ topic.ts_origin = now - lage_to_seconds(lage)
1357
+ self.topics_by_name[name] = topic
1358
+ self.topics_by_hash[topic_hash] = topic
1359
+ self.ensure_gossip_shard(self.gossip_shard_subject_id(topic.hash))
1360
+ self.touch_implicit_topic(topic)
1361
+ self.topic_allocate(topic, evictions, now)
1362
+ for root in matches:
1363
+ self.couple_topic_root(topic, root)
1364
+ topic.sync_listener()
1365
+ self.notify_implicit_gc()
1366
+ _logger.info("Implicit topic '%s' created from gossip", name)
1367
+ return topic
1368
+ return None
1369
+
1370
+ def on_scout(self, arrival: TransportArrival, hdr: ScoutHeader, payload: bytes) -> None:
1371
+ if hdr.pattern_len == 0 or hdr.pattern_len > TOPIC_NAME_MAX or len(payload) < hdr.pattern_len:
1372
+ return
1373
+ pattern = payload[: hdr.pattern_len].decode("utf-8", errors="replace")
1374
+ _logger.debug("Scout received pattern='%s' from %016x", pattern, arrival.remote_id)
1375
+ for topic in list(self.topics_by_name.values()):
1376
+ subs = match_pattern(pattern, topic.name)
1377
+ if subs is not None:
1378
+ self.loop.create_task(self.send_gossip_unicast(topic, arrival.remote_id, arrival.priority))
1379
+
1380
+ # -- Implicit Topic GC --
1381
+
1382
+ def notify_implicit_gc(self) -> None:
1383
+ if not self._closed:
1384
+ self._implicit_gc_wakeup.set()
1385
+
1386
+ def _next_implicit_gc_delay(self, now: float | None = None) -> float | None:
1387
+ now = time.monotonic() if now is None else now
1388
+ if not self._implicit_topics:
1389
+ return None
1390
+ oldest = next(reversed(self._implicit_topics))
1391
+ return max(0.0, (oldest.ts_animated + IMPLICIT_TOPIC_TIMEOUT) - now)
1392
+
1393
+ def _retire_one_expired_implicit_topic(self, now: float) -> bool:
1394
+ if not self._implicit_topics:
1395
+ return False
1396
+ oldest = next(reversed(self._implicit_topics))
1397
+ if (oldest.ts_animated + IMPLICIT_TOPIC_TIMEOUT) >= now:
1398
+ return False
1399
+ self.destroy_topic(oldest.name)
1400
+ _logger.info("GC removed implicit topic '%s'", oldest.name)
1401
+ return True
1402
+
1403
+ async def implicit_gc_loop(self) -> None:
1404
+ try:
1405
+ while not self._closed:
1406
+ self._implicit_gc_wakeup.clear()
1407
+ delay = self._next_implicit_gc_delay()
1408
+ if delay is None:
1409
+ await self._implicit_gc_wakeup.wait()
1410
+ continue
1411
+ if delay > 0:
1412
+ try:
1413
+ await asyncio.wait_for(self._implicit_gc_wakeup.wait(), timeout=delay)
1414
+ continue
1415
+ except asyncio.TimeoutError:
1416
+ pass
1417
+ self._retire_one_expired_implicit_topic(time.monotonic())
1418
+ except asyncio.CancelledError:
1419
+ pass
1420
+
1421
+ def destroy_topic(self, name: str) -> None:
1422
+ topic = self.topics_by_name.get(name)
1423
+ if topic is None:
1424
+ return
1425
+ if topic.gossip_task is not None:
1426
+ self._cancel_gossip(topic)
1427
+ self.discard_implicit_topic(topic)
1428
+ topic.release_transport_handles()
1429
+ while topic.couplings:
1430
+ self.decouple_topic_root(topic, topic.couplings[0].root, sync_lifecycle=False)
1431
+ self.topics_by_name.pop(name, None)
1432
+ self.topics_by_hash.pop(topic.hash, None)
1433
+ sid = topic.subject_id
1434
+ if self.topics_by_subject_id.get(sid) is topic:
1435
+ del self.topics_by_subject_id[sid]
1436
+ topic.associations.clear()
1437
+ topic.dedup.clear()
1438
+ topic.publish_futures.clear()
1439
+ self.notify_implicit_gc()
1440
+ _logger.info("Topic destroyed '%s'", name)
1441
+
1442
+ # -- Cleanup --
1443
+
1444
+ def close(self) -> None:
1445
+ if self._closed:
1446
+ return
1447
+ self._closed = True
1448
+ _logger.info("Node closing home='%s'", self._home)
1449
+ self._gc_task.cancel()
1450
+ for root in list(self.sub_roots_pattern.values()):
1451
+ if root.scout_task is not None:
1452
+ root.scout_task.cancel()
1453
+ root.scout_task = None
1454
+ for topic in list(self.topics_by_name.values()):
1455
+ if topic.gossip_task is not None:
1456
+ self._cancel_gossip(topic)
1457
+ topic.release_transport_handles()
1458
+ self.broadcast_writer.close()
1459
+ self.broadcast_listener.close()
1460
+ for shared_writer in list(self.shared_subject_writers.values()):
1461
+ shared_writer.handle.close()
1462
+ self.shared_subject_writers.clear()
1463
+ for shared_listener in list(self.shared_subject_listeners.values()):
1464
+ shared_listener.handle.close()
1465
+ self.shared_subject_listeners.clear()
1466
+ for w in self.gossip_shard_writers.values():
1467
+ w.close()
1468
+ for gossip_listener in self.gossip_shard_listeners.values():
1469
+ gossip_listener.close()
1470
+ self._monitor_callbacks.clear()
1471
+ self._implicit_topics.clear()
1472
+ self.transport.close()