pyxcp 0.25.4__cp313-cp313-win_amd64.whl

pyxcp/transport/eth.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+import selectors
+import socket
+import struct
+import threading
+from collections import deque
+from typing import Optional
+
+from pyxcp.transport.base import (
+    BaseTransport,
+    ChecksumType,
+    XcpFramingConfig,
+    XcpTransportLayerType,
+)
+from pyxcp.utils import short_sleep
+
+
+DEFAULT_XCP_PORT = 5555
+RECV_SIZE = 8196
+
+
+def socket_to_str(sock: socket.socket) -> str:
+    peer = sock.getpeername()
+    local = sock.getsockname()
+    AF = {
+        socket.AF_INET: "AF_INET",
+        socket.AF_INET6: "AF_INET6",
+    }
+    TYPE = {
+        socket.SOCK_DGRAM: "SOCK_DGRAM",
+        socket.SOCK_STREAM: "SOCK_STREAM",
+    }
+    family = AF.get(sock.family, "OTHER")
+    typ = TYPE.get(sock.type, "UNKNOWN")
+    res = f"XCPonEth - Connected to: {peer[0]}:{peer[1]}  local address: {local[0]}:{local[1]} [{family}][{typ}]"
+    return res
+
+
+class Eth(BaseTransport):
+    """"""
+
+    MAX_DATAGRAM_SIZE = 512
+    HEADER = struct.Struct("<HH")
+
+    def __init__(self, config=None, policy=None, transport_layer_interface: Optional[socket.socket] = None) -> None:
+        self.load_config(config)
+        framing_config = XcpFramingConfig(
+            transport_layer_type=XcpTransportLayerType.ETH,
+            header_len=2,
+            header_ctr=2,
+            header_fill=0,
+            tail_fill=False,
+            tail_cs=ChecksumType.NO_CHECKSUM,
+        )
+        super().__init__(config, framing_config, policy, transport_layer_interface)
+        self.host: str = self.config.host
+        self.port: int = self.config.port
+        self.protocol: int = self.config.protocol
+        self.ipv6: bool = self.config.ipv6
+        self.use_tcp_no_delay: bool = self.config.tcp_nodelay
+        address_to_bind: str = self.config.bind_to_address
+        bind_to_port: int = self.config.bind_to_port
+        self._local_address = (address_to_bind, bind_to_port) if address_to_bind else None
+        if self.ipv6 and not socket.has_ipv6:
+            msg = "XCPonEth - IPv6 not supported by your platform."
+            self.logger.critical(msg)
+            raise RuntimeError(msg)
+        else:
+            address_family = socket.AF_INET6 if self.ipv6 else socket.AF_INET
+        proto = socket.SOCK_STREAM if self.protocol == "TCP" else socket.SOCK_DGRAM
+        if self.host.lower() == "localhost":
+            self.host = "::1" if self.ipv6 else "localhost"
+
+        try:
+            addrinfo = socket.getaddrinfo(self.host, self.port, address_family, proto)
+            (
+                self.address_family,
+                self.socktype,
+                self.proto,
+                self.canonname,
+                self.sockaddr,
+            ) = addrinfo[0]
+        except BaseException as ex:  # noqa: B036
+            msg = f"XCPonEth - Failed to resolve address {self.host}:{self.port} ({self.protocol}, ipv6={self.ipv6}): {ex.__class__.__name__}: {ex}"
+            self.logger.critical(msg, extra={"transport": "eth", "host": self.host, "port": self.port, "protocol": self.protocol})
+            raise Exception(msg) from ex
+        self.status: int = 0
+        self.sock = socket.socket(self.address_family, self.socktype, self.proto)
+        self.selector = selectors.DefaultSelector()
+        self.selector.register(self.sock, selectors.EVENT_READ)
+        self.use_tcp = self.protocol == "TCP"
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        if self.use_tcp and self.use_tcp_no_delay:
+            self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        if hasattr(socket, "SO_REUSEPORT"):
+            self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        self.sock.settimeout(0.5)
+        if self._local_address:
+            try:
+                self.sock.bind(self._local_address)
+            except BaseException as ex:  # noqa: B036
+                msg = f"XCPonEth - Failed to bind socket to given address {self._local_address}: {ex.__class__.__name__}: {ex}"
+                self.logger.critical(
+                    msg, extra={"transport": "eth", "host": self.host, "port": self.port, "protocol": self.protocol}
+                )
+                raise Exception(msg) from ex
+        self._packet_listener = threading.Thread(
+            target=self._packet_listen,
+            args=(),
+            kwargs={},
+            daemon=True,
+        )
+        self._packets = deque()
+
+    def connect(self) -> None:
+        if self.status == 0:
+            self.sock.connect(self.sockaddr)
+            self.logger.info(socket_to_str(self.sock))
+            self.start_listener()
+            self.status = 1  # connected
+
+    def start_listener(self) -> None:
+        super().start_listener()
+        if self._packet_listener.is_alive():
+            self._packet_listener.join(timeout=2.0)
+        self._packet_listener = threading.Thread(target=self._packet_listen, daemon=True)
+        self._packet_listener.start()
+
+    def close(self) -> None:
+        """Close the transport-layer connection and event-loop."""
+        self.finish_listener()
+        try:
+            if self.listener.is_alive():
+                self.listener.join(timeout=2.0)
+        except Exception:
+            pass
+        try:
+            if self._packet_listener.is_alive():
+                self._packet_listener.join(timeout=2.0)
+        except Exception:
+            pass
+        self.close_connection()
+
+    def _packet_listen(self) -> None:
+        use_tcp: bool = self.use_tcp
+        EVENT_READ = selectors.EVENT_READ
+        close_event_set = self.closeEvent.is_set
+        socket_fileno = self.sock.fileno
+        select = self.selector.select
+        _packets = self._packets
+        if use_tcp:
+            sock_recv = self.sock.recv
+        else:
+            sock_recv = self.sock.recvfrom
+        while True:
+            try:
+                if close_event_set() or socket_fileno() == -1:
+                    return
+                sel = select(0.02)
+                for _, events in sel:
+                    if events & EVENT_READ:
+                        recv_timestamp = self.timestamp.value
+                        if use_tcp:
+                            response = sock_recv(RECV_SIZE)
+                            if not response:
+                                self.sock.close()
+                                self.status = 0
+                                break
+                            else:
+                                _packets.append((response, recv_timestamp))
+                        else:
+                            response, _ = sock_recv(Eth.MAX_DATAGRAM_SIZE)
+                            if not response:
+                                self.sock.close()
+                                self.status = 0
+                                break
+                            else:
+                                _packets.append((response, recv_timestamp))
+            except BaseException:  # noqa: B036
+                self.status = 0  # disconnected
+                break
+
+    def listen(self) -> None:
+        process_response = self.process_response
+        popleft = self._packets.popleft
+        close_event_set = self.closeEvent.is_set
+        socket_fileno = self.sock.fileno
+        _packets = self._packets
+        length: Optional[int] = None
+        counter: int = 0
+        data: bytearray = bytearray(b"")
+        while True:
+            if close_event_set() or socket_fileno() == -1:
+                return
+            count: int = len(_packets)
+            if not count:
+                short_sleep()
+                continue
+            for _ in range(count):
+                bts, timestamp = popleft()
+                data += bts
+                current_size: int = len(data)
+                current_position: int = 0
+                while True:
+                    if length is None:
+                        if current_size >= self.framing.header_size:
+                            length, counter = self.framing.unpack_header(bytes(data), initial_offset=current_position)
+                            current_position += self.framing.header_size
+                            current_size -= self.framing.header_size
+                        else:
+                            data = data[current_position:]
+                            break
+                    else:
+                        if current_size >= length:
+                            response = data[current_position : current_position + length]
+                            try:
+                                process_response(response, length, counter, timestamp)
+                            except BaseException as ex:  # Guard listener against unhandled exceptions (e.g., disk full in policy)
+                                try:
+                                    self.logger.critical(
+                                        f"Listener error in process_response: {ex.__class__.__name__}: {ex}. Stopping listener.",
+                                        extra={"event": "listener_error"},
+                                    )
+                                except Exception:
+                                    pass
+                                try:
+                                    # Signal all loops to stop
+                                    if hasattr(self, "closeEvent"):
+                                        self.closeEvent.set()
+                                except Exception:
+                                    pass
+                                return
+                            current_size -= length
+                            current_position += length
+                            length = None
+                        else:
+                            data = data[current_position:]
+                            break
+
+    def send(self, frame) -> None:
+        self.pre_send_timestamp = self.timestamp.value
+        self.sock.send(frame)
+        self.post_send_timestamp = self.timestamp.value
+
+    def close_connection(self) -> None:
+        if not self.invalidSocket:
+            # Seems to be problematic /w IPv6
+            # if self.status == 1:
+            #     self.sock.shutdown(socket.SHUT_RDWR)
+            self.sock.close()
+
+    @property
+    def invalidSocket(self) -> bool:
+        return not hasattr(self, "sock") or self.sock.fileno() == -1

pyxcp/transport/hdf5_policy.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+from pathlib import Path
+from typing import List, Dict
+
+import h5py
+import numpy as np
+from pyxcp.daq_stim import DaqOnlinePolicy, DaqList
+
+
+BATCH_SIZE = 4096
+
+MAP_TO_NP = {
+    "U8": np.uint8,
+    "I8": np.int8,
+    "U16": np.uint16,
+    "I16": np.int16,
+    "U32": np.uint32,
+    "I32": np.int32,
+    "U64": np.uint64,
+    "I64": np.int64,
+    "F32": np.float32,
+    "F64": np.float64,
+    "F16": np.float16,
+    "BF16": np.float16,
+}
+
+
+class BufferedDataset:
+    def __init__(self, dataset: h5py.Dataset):
+        self.dataset = dataset
+        self.buffer: List[int | float] = []
+
+    def add_sample(self, sample: int | float):
+        self.buffer.append(sample)
+        if len(self.buffer) >= BATCH_SIZE:
+            self.flush()
+
+    def flush(self):
+        batch = np.array(self.buffer)
+        self.dataset.resize((self.dataset.shape[0] + len(batch),))
+        self.dataset[-len(batch) :] = batch
+        self.buffer.clear()
+        self.dataset.flush()
+
+    def __len__(self):
+        return len(self.buffer)
+
+
+class DatasetGroup:
+    def __init__(
+        self,
+        ts0_ds: BufferedDataset,
+        ts1_ds: BufferedDataset,
+        datasets: List[BufferedDataset],
+    ):
+        self.ts0_ds = ts0_ds
+        self.ts1_ds = ts1_ds
+        self.datasets = datasets
+
+    def feed(self, ts0: int, ts1: int, *datasets):
+        self.ts0_ds.add_sample(ts0)
+        self.ts1_ds.add_sample(ts1)
+        for dataset, value in zip(self.datasets, datasets):
+            dataset.add_sample(value)
+
+    def finalize(self):
+        for dataset in self.datasets:
+            dataset.flush()
+        self.ts0_ds.flush()
+        self.ts1_ds.flush()
+
+
+def create_timestamp_column(hdf_file: h5py.File, group_name: str, num: int) -> h5py.Dataset:
+    return hdf_file.create_dataset(
+        f"/{group_name}/timestamp{num}",
+        shape=(0,),
+        maxshape=(None,),
+        dtype=np.uint64,
+        chunks=True,
+    )
+
+
+class Hdf5OnlinePolicy(DaqOnlinePolicy):
+    def __init__(self, file_name: str | Path, daq_lists: List[DaqList]):
+        super().__init__(daq_lists=daq_lists)
+        path = Path(file_name)
+        if path.suffix != ".h5":
+            path = path.with_suffix(".h5")
+        self.hdf = h5py.File(path, "w", libver="latest")
+
+    def initialize(self):
+        self.log.debug("Hdf5OnlinePolicy::Initialize()")
+        self.datasets: Dict[int, DatasetGroup] = {}
+        for num, daq_list in enumerate(self.daq_lists):
+            if daq_list.stim:
+                continue
+            self.hdf.create_group(daq_list.name)
+            ts0 = BufferedDataset(create_timestamp_column(self.hdf, daq_list.name, 0))
+            ts1 = BufferedDataset(create_timestamp_column(self.hdf, daq_list.name, 1))
+            meas_map = {m.name: m for m in self.daq_lists[num].measurements}
+            dsets = []
+            for name, _ in daq_list.headers:
+                meas = meas_map[name]
+                dataset = self.hdf.create_dataset(
+                    f"/{daq_list.name}/{meas.name}",
+                    shape=(0,),
+                    maxshape=(None,),
+                    dtype=MAP_TO_NP[meas.data_type],
+                    chunks=(1024,),
+                )
+                dsets.append(BufferedDataset(dataset))
+            self.datasets[num] = DatasetGroup(ts0_ds=ts0, ts1_ds=ts1, datasets=dsets)
+        self.hdf.flush()
+
+    def finalize(self):
+        self.log.debug("Hdf5OnlinePolicy::finalize()")
+        if hasattr(self, "datasets"):
+            for group in self.datasets.values():
+                group.finalize()
+        if hasattr(self, "hdf"):
+            self.hdf.close()
+
+    def on_daq_list(self, daq_list: int, timestamp0: int, timestamp1: int, payload: list):
+        group = self.datasets.get(daq_list)
+        if group is None:
+            self.log.warning(f"Received data for unknown DAQ list {daq_list}")
+            return
+        group.feed(timestamp0, timestamp1, *payload)

pyxcp/transport/sxi.py

@@ -0,0 +1,209 @@
+import threading
+from collections import deque
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import serial
+
+from pyxcp.transport.transport_ext import (
+    SxiFrLBCN,
+    SxiFrLBC8,
+    SxiFrLBC16,
+    SxiFrLCBCN,
+    SxiFrLCBC8,
+    SxiFrLCBC16,
+    SxiFrLFBCN,
+    SxiFrLFBC8,
+    SxiFrLFBC16,
+    SxiFrLWCN,
+    SxiFrLWC8,
+    SxiFrLWC16,
+    SxiFrLCWCN,
+    SxiFrLCWC8,
+    SxiFrLCWC16,
+    SxiFrLFWCN,
+    SxiFrLFWC8,
+    SxiFrLFWC16,
+)
+
+from pyxcp.transport.base import (
+    BaseTransport,
+    ChecksumType,
+    XcpFramingConfig,
+    XcpTransportLayerType,
+    parse_header_format,
+)
+
+
+@dataclass
+class HeaderValues:
+    length: int = 0
+    counter: int = 0
+    filler: int = 0
+
+
+RECV_SIZE = 16384
+
+
+def get_receiver_class(header_format: str, checksum_format: str) -> Any:
+    COLUMN = {"NO_CHECKSUM": 0, "CHECKSUM_BYTE": 1, "CHECKSUM_WORD": 2}
+    FORMATS = {
+        "HEADER_LEN_BYTE": (SxiFrLBCN, SxiFrLBC8, SxiFrLBC16),
+        "HEADER_LEN_CTR_BYTE": (SxiFrLCBCN, SxiFrLCBC8, SxiFrLCBC16),
+        "HEADER_LEN_FILL_BYTE": (SxiFrLFBCN, SxiFrLFBC8, SxiFrLFBC16),
+        "HEADER_LEN_WORD": (SxiFrLWCN, SxiFrLWC8, SxiFrLWC16),
+        "HEADER_LEN_CTR_WORD": (SxiFrLCWCN, SxiFrLCWC8, SxiFrLCWC16),
+        "HEADER_LEN_FILL_WORD": (SxiFrLFWCN, SxiFrLFWC8, SxiFrLFWC16),
+    }
+    return FORMATS[header_format][COLUMN[checksum_format]]
+
+
+class SxI(BaseTransport):
+    """"""
+
+    def __init__(self, config=None, policy=None, transport_layer_interface: Optional[serial.Serial] = None) -> None:
+        self.load_config(config)
+        self.port_name = self.config.port
+        self.baudrate = self.config.bitrate
+        self.bytesize = self.config.bytesize
+        self.parity = self.config.parity
+        self.stopbits = self.config.stopbits
+        self.mode = self.config.mode
+        header_len, header_ctr, header_fill = parse_header_format(self.config.header_format)
+        tail_cs_map = {
+            "NO_CHECKSUM": ChecksumType.NO_CHECKSUM,
+            "CHECKSUM_BYTE": ChecksumType.BYTE_CHECKSUM,
+            "CHECKSUM_WORD": ChecksumType.WORD_CHECKSUM,
+        }
+        # self._listener_running = threading.Event()
+        tail_cs = tail_cs_map[self.config.tail_format]
+        ReceiverKlass = get_receiver_class(self.config.header_format, self.config.tail_format)
+        self.receiver = ReceiverKlass(self.frame_dispatcher)
+        framing_config = XcpFramingConfig(
+            transport_layer_type=XcpTransportLayerType.SXI,
+            header_len=header_len,
+            header_ctr=header_ctr,
+            header_fill=header_fill,
+            tail_fill=False,
+            tail_cs=tail_cs,
+        )
+        super().__init__(config, framing_config, policy, transport_layer_interface)
+        self.tail_format = self.config.tail_format
+        self.esc_sync = self.config.esc_sync
+        self.esc_esc = self.config.esc_esc
+        self.comm_port: serial.Serial
+
+        if self.has_user_supplied_interface and transport_layer_interface:
+            self.comm_port = transport_layer_interface
+        else:
+            self.logger.info(f"XCPonSxI - trying to open serial comm_port {self.port_name!r}.")
+            try:
+                self.comm_port = serial.Serial(
+                    port=self.port_name,
+                    baudrate=self.baudrate,
+                    bytesize=self.bytesize,
+                    parity=self.parity,
+                    stopbits=self.stopbits,
+                    timeout=0.1,  # self.timeout,
+                    write_timeout=self.timeout,
+                )
+            except serial.SerialException as e:
+                self.logger.critical(f"XCPonSxI - {e}")
+                raise
+        self._condition = threading.Condition()
+        self._frames = deque()
+        # self._frame_listener = threading.Thread(
+        #    target=self._frame_listen,
+        #    args=(),
+        #    kwargs={},
+        # )
+
+    def __del__(self) -> None:
+        self.close_connection()
+
+    def connect(self) -> None:
+        self.logger.info(
+            f"XCPonSxI - serial comm_port {self.comm_port.portstr!r} openend "
+            f"[{self.baudrate}/{self.bytesize}-{self.parity}-{self.stopbits}] "
+            f"mode: {self.config.mode}"
+        )
+        self.logger.info(f"Framing: {self.config.header_format} {self.config.tail_format}")
+        self.start_listener()
+
+    def output(self, enable) -> None:
+        if enable:
+            self.comm_port.rts = False
+            self.comm_port.dtr = False
+        else:
+            self.comm_port.rts = True
+            self.comm_port.dtr = True
+
+    def flush(self) -> None:
+        self.comm_port.flush()
+
+    def start_listener(self) -> None:
+        super().start_listener()
+        if hasattr(self, "_frame_listener") and self._frame_listener.is_alive():
+            self._frame_listener.join(timeout=2.0)
+        self._frame_listener = threading.Thread(target=self._frame_listen, daemon=True)
+        self._frame_listener.start()
+        # self._listener_running.wait(2.0)
+
+    def close(self) -> None:
+        """Close the transport-layer connection and event-loop."""
+        self.finish_listener()
+        self.closeEvent.set()
+        try:
+            if self.listener.is_alive():
+                self.listener.join(timeout=2.0)
+        except Exception:
+            pass
+        try:
+            if hasattr(self, "_frame_listener") and self._frame_listener.is_alive():
+                self._frame_listener.join(timeout=2.0)
+        except Exception:
+            pass
+        self.close_connection()
+
+    def listen(self) -> None:
+        while True:
+            if self.closeEvent.is_set():
+                return
+            frame_to_process = None
+            with self._condition:
+                while not self._frames:
+                    res = self._condition.wait(1.0)
+                    if not res:
+                        break
+                if self._frames:
+                    frame_to_process = self._frames.popleft()
+
+            if frame_to_process:
+                frame, length, counter, timestamp = frame_to_process
+                self.process_response(frame, length, counter, timestamp)
+
+    def _frame_listen(self) -> None:
+        # self._listener_running.set()
+        while True:
+            if self.closeEvent.is_set():
+                return
+            data = self.comm_port.read(1)
+            if data:
+                self.receiver.feed_bytes(data)
+                data = self.comm_port.read(self.comm_port.in_waiting)
+                if data:
+                    self.receiver.feed_bytes(data)
+
+    def frame_dispatcher(self, data: bytes, length: int, counter: int) -> None:
+        with self._condition:
+            self._frames.append((bytes(data), length, counter, self.timestamp.value))
+            self._condition.notify()
+
+    def send(self, frame: bytes) -> None:
+        self.pre_send_timestamp = self.timestamp.value
+        self.comm_port.write(frame)
+        self.post_send_timestamp = self.timestamp.value
+
+    def close_connection(self) -> None:
+        if hasattr(self, "comm_port") and self.comm_port.is_open and not self.has_user_supplied_interface:
+            self.comm_port.close()