pycyphal2 2.0.0.dev0__py3-none-any.whl

pycyphal2/_subscriber.py

@@ -0,0 +1,430 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+from dataclasses import dataclass, field
+
+from ._api import DeliveryError, Instant, LivenessError, NackError, Priority, SendError
+from ._api import Subscriber, Breadcrumb, Topic, Arrival
+from ._header import SEQNO48_MASK, RspBeHeader, RspRelHeader
+from ._node import (
+    ACK_BASELINE_DEFAULT_TIMEOUT,
+    REORDERING_CAPACITY,
+    SESSION_LIFETIME,
+    NodeImpl,
+    SubscriberRoot,
+    TopicImpl,
+    match_pattern,
+)
+
+_logger = logging.getLogger(__name__)
+REORDERING_WINDOW_MAX = SESSION_LIFETIME / 2
+
+
+# =====================================================================================================================
+# Reordering
+# =====================================================================================================================
+
+
+@dataclass
+class InternedMsg:
+    arrival: Arrival
+    tag: int
+    remote_id: int
+    lin_tag: int
+
+
+@dataclass
+class ReorderingState:
+    """Per (remote_id, topic_hash) reordering state for ordered subscriptions."""
+
+    tag_baseline: int = 0
+    last_ejected_lin_tag: int = 0
+    last_active_at: float = 0.0
+    interned: dict[int, InternedMsg] = field(default_factory=dict)  # lin_tag -> msg
+    timeout_handle: asyncio.TimerHandle | None = None
+
+
+class SubscriberImpl(Subscriber):
+    def __init__(
+        self,
+        node: NodeImpl,
+        root: SubscriberRoot,
+        pattern: str,
+        verbatim: bool,
+        reordering_window: float | None,
+    ) -> None:
+        self._node = node
+        self._root = root
+        self._pattern = pattern
+        self._verbatim = verbatim
+        self._timeout = float("inf")
+        self._reordering_window = self._normalize_reordering_window(reordering_window)
+        self.queue: asyncio.Queue[Arrival | BaseException] = asyncio.Queue()
+        self._reordering: dict[tuple[int, int], ReorderingState] = {}  # (remote_id, topic_hash)
+        self.closed = False
+
+    @staticmethod
+    def _normalize_reordering_window(reordering_window: float | None) -> float | None:
+        if reordering_window is None:
+            return None
+        out = float(reordering_window)
+        if (out < 0.0) or (not math.isfinite(out)):
+            raise ValueError("Reordering window must be a finite non-negative duration")
+        if out > REORDERING_WINDOW_MAX:
+            raise ValueError(f"Reordering window is too large")
+        return out
+
+    @property
+    def pattern(self) -> str:
+        return self._pattern
+
+    @property
+    def verbatim(self) -> bool:
+        return self._verbatim
+
+    @property
+    def timeout(self) -> float:
+        return self._timeout
+
+    @timeout.setter
+    def timeout(self, duration: float) -> None:
+        self._timeout = duration
+
+    def substitutions(self, topic: Topic) -> list[tuple[str, int]] | None:
+        return match_pattern(self._pattern, topic.name)
+
+    def __aiter__(self) -> SubscriberImpl:
+        return self
+
+    async def __anext__(self) -> Arrival:
+        if self.closed:
+            raise StopAsyncIteration
+        timeout = self._timeout if self._timeout != float("inf") else None
+        try:
+            item = await asyncio.wait_for(self.queue.get(), timeout=timeout)
+        except asyncio.TimeoutError:
+            raise LivenessError("No message received within timeout")
+        if isinstance(item, StopAsyncIteration):
+            raise item
+        if isinstance(item, BaseException):
+            raise item
+        return item
+
+    def deliver(self, arrival: Arrival, tag: int, remote_id: int) -> bool:
+        """Called by the node to deliver a message to this subscriber."""
+        if self.closed:
+            return False
+        if self._reordering_window is None:
+            self.queue.put_nowait(arrival)
+            return True
+        # Reordering enabled.
+        self._drop_stale_reordering(arrival.timestamp.s)
+        topic_hash = arrival.breadcrumb.topic.hash
+        key = (remote_id, topic_hash)
+        state = self._reordering.get(key)
+        if state is None:
+            state = ReorderingState(
+                tag_baseline=tag - (REORDERING_CAPACITY // 2),
+                last_ejected_lin_tag=0,
+                last_active_at=arrival.timestamp.s,
+            )
+            self._reordering[key] = state
+        state.last_active_at = arrival.timestamp.s
+        lin_tag = (tag - state.tag_baseline) & ((1 << 64) - 1)
+
+        # Detect wraparound / very late messages.
+        if lin_tag > ((1 << 63) - 1):
+            _logger.debug("Reorder drop late tag=%d lin=%d", tag, lin_tag)
+            return False
+        if lin_tag <= state.last_ejected_lin_tag:
+            _logger.debug("Reorder drop dup/late tag=%d lin=%d last=%d", tag, lin_tag, state.last_ejected_lin_tag)
+            return False
+
+        while state.interned and lin_tag > (state.last_ejected_lin_tag + REORDERING_CAPACITY):
+            self._scan_reordering(state, force_first=True)
+
+        expected = state.last_ejected_lin_tag + 1
+        if lin_tag == expected:
+            # In-order: eject immediately and scan for consecutive.
+            self.queue.put_nowait(arrival)
+            state.last_ejected_lin_tag = lin_tag
+            self._scan_reordering(state, force_first=False)
+            return True
+
+        if lin_tag > (state.last_ejected_lin_tag + REORDERING_CAPACITY):
+            state.tag_baseline = tag - (REORDERING_CAPACITY // 2)
+            state.last_ejected_lin_tag = 0
+            lin_tag = (tag - state.tag_baseline) & ((1 << 64) - 1)
+            _logger.debug("Reorder resequence tag=%d lin=%d", tag, lin_tag)
+
+        # Out-of-order but within capacity: intern.
+        if lin_tag in state.interned:
+            return True
+        state.interned[lin_tag] = InternedMsg(arrival=arrival, tag=tag, remote_id=remote_id, lin_tag=lin_tag)
+        self._rearm_reorder_timeout(state)
+        return True
+
+    def _scan_reordering(self, state: ReorderingState, force_first: bool) -> None:
+        while True:
+            if not state.interned:
+                if state.timeout_handle is not None:
+                    state.timeout_handle.cancel()
+                    state.timeout_handle = None
+                break
+
+            lin_tag = min(state.interned)
+            if force_first or ((state.last_ejected_lin_tag + 1) == lin_tag):
+                force_first = False
+                interned = state.interned.pop(lin_tag)
+                self.queue.put_nowait(interned.arrival)
+                state.last_ejected_lin_tag = lin_tag
+                continue
+
+            self._rearm_reorder_timeout(state)
+            break
+
+    def _force_eject_all(self, state: ReorderingState, *, silenced: bool = False) -> None:
+        """Force-eject all interned messages in tag order."""
+        while state.interned:
+            lin_tag = min(state.interned)
+            interned = state.interned.pop(lin_tag)
+            state.last_ejected_lin_tag = lin_tag
+            if not silenced:
+                self.queue.put_nowait(interned.arrival)
+        if state.timeout_handle is not None:
+            state.timeout_handle.cancel()
+            state.timeout_handle = None
+
+    def _rearm_reorder_timeout(self, state: ReorderingState) -> None:
+        """Arm or rearm the reordering timeout against the current head-of-line slot."""
+        if self._reordering_window is None:
+            return
+        if not state.interned:
+            if state.timeout_handle is not None:
+                state.timeout_handle.cancel()
+                state.timeout_handle = None
+            return
+
+        lin_tag = min(state.interned)
+        delay = max(0.0, (state.interned[lin_tag].arrival.timestamp.s + self._reordering_window) - Instant.now().s)
+
+        loop = self._node.loop
+        if state.timeout_handle is not None:
+            state.timeout_handle.cancel()
+
+        def on_timeout() -> None:
+            state.timeout_handle = None
+            self._scan_reordering(state, force_first=True)
+
+        state.timeout_handle = loop.call_later(delay, on_timeout)
+
+    def _arm_reorder_timeout(self, state: ReorderingState) -> None:
+        self._rearm_reorder_timeout(state)
+
+    def _drop_stale_reordering(self, now: float) -> None:
+        stale = [key for key, state in self._reordering.items() if (state.last_active_at + SESSION_LIFETIME) < now]
+        for key in stale:
+            state = self._reordering.pop(key)
+            self._force_eject_all(state)
+
+    def forget_topic_reordering(self, topic_hash: int, *, silenced: bool = True) -> None:
+        keys = [key for key in self._reordering if key[1] == topic_hash]
+        for key in keys:
+            state = self._reordering.pop(key)
+            self._force_eject_all(state, silenced=silenced)
+
+    def close(self) -> None:
+        if self.closed:
+            return
+        self.closed = True
+        for state in self._reordering.values():
+            self._force_eject_all(state)
+        self._reordering.clear()
+        if self in self._root.subscribers:
+            self._root.subscribers.remove(self)
+        if not self._root.subscribers:
+            if self._root.scout_task is not None:
+                self._root.scout_task.cancel()
+                self._root.scout_task = None
+            if self._root.is_pattern:
+                self._node.sub_roots_pattern.pop(self._root.name, None)
+            else:
+                self._node.sub_roots_verbatim.pop(self._root.name, None)
+            for topic in list(self._node.topics_by_name.values()):
+                self._node.decouple_topic_root(topic, self._root)
+        self.queue.put_nowait(StopAsyncIteration())
+        _logger.info("Subscriber closed for '%s'", self._pattern)
+
+
+# =====================================================================================================================
+# Breadcrumb
+# =====================================================================================================================
+
+
+class BreadcrumbImpl(Breadcrumb):
+    def __init__(
+        self,
+        node: NodeImpl,
+        remote_id: int,
+        topic: TopicImpl,
+        message_tag: int,
+        initial_priority: Priority,
+    ) -> None:
+        self._node = node
+        self._remote_id = remote_id
+        self._topic = topic
+        self._message_tag = message_tag
+        self._priority = initial_priority
+        self._seqno = 0
+
+    @property
+    def remote_id(self) -> int:
+        return self._remote_id
+
+    @property
+    def topic(self) -> Topic:
+        return self._topic
+
+    @property
+    def tag(self) -> int:
+        return self._message_tag
+
+    async def __call__(
+        self,
+        deadline: Instant,
+        message: memoryview | bytes,
+        *,
+        reliable: bool = False,
+    ) -> None:
+        seqno = self._seqno & SEQNO48_MASK
+        self._seqno += 1
+
+        hdr: RspBeHeader | RspRelHeader
+        if not reliable:
+            hdr = RspBeHeader(
+                tag=0xFF,
+                seqno=seqno,
+                topic_hash=self._topic.hash,
+                message_tag=self._message_tag,
+            )
+        else:
+            rsp_tag = self._allocate_response_tag(seqno)
+            hdr = RspRelHeader(
+                tag=rsp_tag,
+                seqno=seqno,
+                topic_hash=self._topic.hash,
+                message_tag=self._message_tag,
+            )
+
+        data = hdr.serialize() + bytes(message)
+        if not reliable:
+            await self._node.transport.unicast(deadline, self._priority, self._remote_id, data)
+            _logger.debug("Response BE sent seqno=%d to %016x", seqno, self._remote_id)
+            return
+
+        # Reliable response with retransmission.
+        tracker = RespondTracker(
+            remote_id=self._remote_id,
+            message_tag=self._message_tag,
+            topic_hash=self._topic.hash,
+            seqno=seqno,
+            tag=hdr.tag,
+        )
+        key = tracker.key
+        self._node.respond_futures[key] = tracker
+
+        ack_timeout = ACK_BASELINE_DEFAULT_TIMEOUT * (1 << int(self._priority))
+        try:
+            initial_window = _ack_window(deadline.ns, ack_timeout)
+            if initial_window is None:
+                raise DeliveryError("Reliable response not acknowledged before deadline")
+
+            ack_deadline_ns, last_attempt = initial_window
+            tracker.ack_event.clear()
+            try:
+                await self._node.transport.unicast(Instant(ns=ack_deadline_ns), self._priority, self._remote_id, data)
+            except SendError:
+                raise
+            except OSError as ex:
+                raise SendError("Reliable response initial send failed") from ex
+
+            while True:
+                if tracker.done:
+                    if tracker.nacked:
+                        raise NackError("Response NACK'd by remote")
+                    return
+
+                wait_until_ns = deadline.ns if last_attempt else ack_deadline_ns
+                wait_time = max(0.0, (wait_until_ns - Instant.now().ns) * 1e-9)
+                try:
+                    await asyncio.wait_for(tracker.ack_event.wait(), timeout=wait_time)
+                except asyncio.TimeoutError:
+                    pass
+
+                if tracker.done:
+                    if tracker.nacked:
+                        raise NackError("Response NACK'd by remote")
+                    return
+
+                if last_attempt:
+                    break
+                ack_timeout *= 2
+                next_window = _ack_window(deadline.ns, ack_timeout)
+                if next_window is None:
+                    break
+                ack_deadline_ns, last_attempt = next_window
+                tracker.ack_event.clear()
+                try:
+                    await self._node.transport.unicast(
+                        Instant(ns=ack_deadline_ns), self._priority, self._remote_id, data
+                    )
+                except (SendError, OSError):
+                    pass
+
+            if not tracker.done:
+                raise DeliveryError("Reliable response not acknowledged before deadline")
+        finally:
+            self._node.respond_futures.pop(key, None)
+
+    def _allocate_response_tag(self, seqno: int) -> int:
+        for tag in range(256):
+            key = (self._remote_id, self._message_tag, self._topic.hash, seqno, tag)
+            if key not in self._node.respond_futures:
+                return tag
+        raise DeliveryError("Reliable response tag space exhausted")
+
+
+class RespondTracker:
+    """Tracks a pending reliable response awaiting ACK."""
+
+    def __init__(self, remote_id: int, message_tag: int, topic_hash: int, seqno: int, tag: int) -> None:
+        self.remote_id = remote_id
+        self.message_tag = message_tag
+        self.topic_hash = topic_hash
+        self.seqno = seqno
+        self.tag = tag
+        self.key = (remote_id, message_tag, topic_hash, seqno, tag)
+        self.ack_event = asyncio.Event()
+        self.done = False
+        self.nacked = False
+
+    def on_ack(self, positive: bool) -> None:
+        self.done = True
+        self.nacked = not positive
+        self.ack_event.set()
+
+
+def _ack_is_last_attempt(current_ack_deadline_ns: int, current_ack_timeout: float, total_deadline_ns: int) -> bool:
+    next_ack_timeout_ns = round(current_ack_timeout * 2 * 1e9)
+    remaining_budget_ns = total_deadline_ns - current_ack_deadline_ns
+    return remaining_budget_ns < next_ack_timeout_ns
+
+
+def _ack_window(deadline_ns: int, ack_timeout: float) -> tuple[int, bool] | None:
+    now_ns = Instant.now().ns
+    if now_ns >= deadline_ns:
+        return None
+    ack_deadline_ns = min(deadline_ns, now_ns + round(ack_timeout * 1e9))
+    return ack_deadline_ns, _ack_is_last_attempt(ack_deadline_ns, ack_timeout, deadline_ns)

pycyphal2/_transport.py

@@ -0,0 +1,92 @@
+"""
+The bottom-layer API that connects the session layer to the underlying transport layer.
+Normally, applications don't care about this unless a custom transport is needed (very uncommon),
+so it is moved into a separate module.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from ._api import Closable, Instant, Priority
+
+SUBJECT_ID_MODULUS_16bit = 57203  # Suitable for all Cyphal transports
+SUBJECT_ID_MODULUS_23bit = 8378431  # Incompatible with Cyphal/CAN
+SUBJECT_ID_MODULUS_32bit = 4294954663  # Incompatible with Cyphal/CAN and Cyphal/UDPv4
+
+
+class SubjectWriter(Closable):
+    @abstractmethod
+    async def __call__(self, deadline: Instant, priority: Priority, message: bytes | memoryview) -> None:
+        raise NotImplementedError
+
+
+@dataclass(frozen=True)
+class TransportArrival:
+    """
+    Arrival of a transfer from the underlying transport.
+    The session layer (this library) will parse the header and process the message.
+    """
+
+    timestamp: Instant
+    priority: Priority
+    remote_id: int
+    message: bytes
+
+
+class Transport(Closable):
+    """
+    Serves the same purpose as cy_platform_t in Cy, with several Pythonic deviations documented below.
+    """
+
+    @property
+    @abstractmethod
+    def subject_id_modulus(self) -> int:
+        """
+        Constant, cannot be changed while the transport is in used because that would invalidate subject allocations.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def subject_listen(self, subject_id: int, handler: Callable[[TransportArrival], None]) -> Closable:
+        """
+        Subscribe to a subject to receive messages from it until the returned closable handle is closed.
+        The session layer may request at most one listener per subject at any given time, similar to the reference impl.
+        Duplicate requests for the same subject should raise ValueError.
+
+        REFERENCE PARITY: Unlike the reference implementation, our listeners do not have the extent setting --
+        the extent mostly matters for high-reliability/real-time applications; this Python implementation
+        assumes infinite extent.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def subject_advertise(self, subject_id: int) -> SubjectWriter:
+        """
+        Begin sending messages on a subject.
+        The session layer may request at most one writer per subject at any given time, similar to the reference impl.
+        Duplicate requests for the same subject should raise ValueError.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def unicast_listen(self, handler: Callable[[TransportArrival], None]) -> None:
+        """
+        The session layer will invoke this once to configure the handler that will process incoming unicast messages.
+        Normally it will happen very early in initialization so no messages are lost; if, however, it somehow comes
+        to pass that messages arrive while the handler is still not set, they may be silently dropped.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def unicast(self, deadline: Instant, priority: Priority, remote_id: int, message: bytes | memoryview) -> None:
+        """
+        Send a unicast message to the specified remote node.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def __repr__(self) -> str:
+        raise NotImplementedError

pycyphal2/can/__init__.py

@@ -0,0 +1,43 @@
+"""
+Cyphal/CAN transport — real-time reliable pub/sub over Classic CAN and CAN FD.
+Supports various backends such as SocketCAN and Python-CAN.
+
+```python
+from pycyphal2.can import CANTransport
+# Import the backend you need.
+# Beware: optional dependencies may be needed, check pyproject.toml.
+from pycyphal2.can.socketcan import SocketCANInterface
+
+transport = CANTransport.new(SocketCANInterface("can0"))
+```
+
+Python-CAN is useful when the application runs not on GNU/Linux or already uses `python-can` or needs
+`one of its *many* hardware backends <https://python-can.readthedocs.io/en/stable/interfaces.html>`_
+-- GS-USB, SLCAN, PCAN, etc:
+
+```python
+import can
+from pycyphal2.can import CANTransport
+from pycyphal2.can.pythoncan import PythonCANInterface
+
+bus = can.ThreadSafeBus(interface="socketcan", channel="can0")
+transport = CANTransport.new(PythonCANInterface(bus))
+```
+
+Pass the transport to `pycyphal2.Node.new()` to start a node.
+
+For the available dependencies see the submodules such as `socketcan` et al.
+"""
+
+from __future__ import annotations
+
+from ._interface import Filter as Filter
+from ._interface import Frame as Frame
+from ._interface import Interface as Interface
+from ._interface import TimestampedFrame as TimestampedFrame
+from ._transport import CANTransport as CANTransport
+
+# Backend submodules importable via pycyphal2.can.pythoncan / pycyphal2.can.socketcan;
+# they are not eagerly imported here because they pull in optional dependencies.
+
+__all__ = ["CANTransport", "Frame", "TimestampedFrame", "Filter", "Interface"]

pycyphal2/can/_interface.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Iterable
+import itertools
+
+from .. import Closable, Instant
+
+_CAN_EXT_ID_MASK = (1 << 29) - 1
+
+
+@dataclass(frozen=True)
+class Frame:
+    """29-bit extended data frame."""
+
+    id: int
+    data: bytes
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.id, int) or not (0 <= self.id <= _CAN_EXT_ID_MASK):
+            raise ValueError(f"Invalid CAN identifier: {self.id!r}")
+        data = bytes(self.data)
+        if len(data) > 64:
+            raise ValueError(f"Invalid CAN data length: {len(data)}")
+        object.__setattr__(self, "data", data)
+
+
+@dataclass(frozen=True)
+class TimestampedFrame(Frame):
+    timestamp: Instant
+
+
+@dataclass(frozen=True)
+class Filter:
+    """29-bit extended identifier acceptance filter."""
+
+    id: int
+    mask: int
+
+    def __post_init__(self) -> None:
+        if not (0 <= self.id <= _CAN_EXT_ID_MASK):
+            raise ValueError(f"Invalid CAN identifier: {self.id!r}")
+        if not (0 <= self.mask <= _CAN_EXT_ID_MASK):
+            raise ValueError(f"Invalid CAN mask: {self.mask!r}")
+
+    @property
+    def rank(self) -> int:
+        return self.mask.bit_count()
+
+    def merge(self, other: Filter) -> Filter:
+        mask = self.mask & other.mask & ~(self.id ^ other.id)
+        return Filter(id=self.id & mask, mask=mask)
+
+    @staticmethod
+    def promiscuous() -> Filter:
+        return Filter(id=0, mask=0)
+
+    @staticmethod
+    def coalesce(filters: Iterable[Filter], count: int) -> list[Filter]:
+        if count < 1:
+            raise ValueError("The target number of filters must be positive")
+        filters = list(filters)
+        assert isinstance(filters, list)
+        # REFERENCE PARITY: Do not flag this as a divergence; this implementation is correct.
+        while len(filters) > count:
+            options = itertools.starmap(
+                lambda ia, ib: (ia[0], ib[0], ia[1].merge(ib[1])), itertools.permutations(enumerate(filters), 2)
+            )
+            index_replace, index_remove, merged = max(options, key=lambda x: int(x[2].rank))
+            filters[index_replace] = merged
+            del filters[index_remove]  # Invalidates indexes
+        assert all(map(lambda x: isinstance(x, Filter), filters))
+        return filters
+
+
+class Interface(Closable, ABC):
+    """
+    A local CAN controller interface.
+    Only extended-ID data frames are supported; everything else is silently dropped.
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def fd(self) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def filter(self, filters: Iterable[Filter]) -> None:
+        """
+        Request the hardware acceptance filter configuration.
+        Implementations with a smaller hardware capacity shall coalesce the list locally.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def enqueue(self, id: int, data: Iterable[memoryview], deadline: Instant) -> None:
+        """
+        Schedule one or more frames for transmission. All frames share the same extended identifier.
+        The frame order within the iterable shall be preserved. Implementations may prioritize queued
+        frames by CAN identifier to approximate bus arbitration, but the relative order of frames
+        belonging to one transfer shall remain unchanged.
+        """
+        # REFERENCE PARITY: TX queue ownership intentionally belongs to the interface rather than the transport.
+        # This differs from libcanard's internal queue placement but it is not a parity drift because it does not
+        # affect the wire-visible behavior by itself.
+        raise NotImplementedError
+
+    @abstractmethod
+    def purge(self) -> None:
+        """
+        Drop all queued but not yet transmitted frames.
+        Used when the local node-ID changes and queued continuations become invalid.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def receive(self) -> TimestampedFrame:
+        """
+        Suspend until the next frame is received.
+        Raises an exception if the interface is closed or has failed.
+        """
+        raise NotImplementedError
+
+    def __repr__(self) -> str:
+        raise NotImplementedError