pyxcp 0.23.3__cp312-cp312-win_arm64.whl → 0.25.6__cp312-cp312-win_arm64.whl

pyxcp/transport/eth.py

@@ -6,7 +6,12 @@ import threading
 from collections import deque
 from typing import Optional
 
-from pyxcp.transport.base import BaseTransport
+from pyxcp.transport.base import (
+    BaseTransport,
+    ChecksumType,
+    XcpFramingConfig,
+    XcpTransportLayerType,
+)
 from pyxcp.utils import short_sleep
 
 
@@ -36,11 +41,18 @@ class Eth(BaseTransport):
 
     MAX_DATAGRAM_SIZE = 512
     HEADER = struct.Struct("<HH")
-    HEADER_SIZE = HEADER.size
 
     def __init__(self, config=None, policy=None, transport_layer_interface: Optional[socket.socket] = None) -> None:
-        super().__init__(config, policy, transport_layer_interface)
         self.load_config(config)
+        framing_config = XcpFramingConfig(
+            transport_layer_type=XcpTransportLayerType.ETH,
+            header_len=2,
+            header_ctr=2,
+            header_fill=0,
+            tail_fill=False,
+            tail_cs=ChecksumType.NO_CHECKSUM,
+        )
+        super().__init__(config, framing_config, policy, transport_layer_interface)
         self.host: str = self.config.host
         self.port: int = self.config.port
         self.protocol: int = self.config.protocol
@@ -69,8 +81,8 @@ class Eth(BaseTransport):
                 self.sockaddr,
             ) = addrinfo[0]
         except BaseException as ex:  # noqa: B036
-            msg = f"XCPonEth - Failed to resolve address {self.host}:{self.port}"
-            self.logger.critical(msg)
+            msg = f"XCPonEth - Failed to resolve address {self.host}:{self.port} ({self.protocol}, ipv6={self.ipv6}): {ex.__class__.__name__}: {ex}"
+            self.logger.critical(msg, extra={"transport": "eth", "host": self.host, "port": self.port, "protocol": self.protocol})
             raise Exception(msg) from ex
         self.status: int = 0
         self.sock = socket.socket(self.address_family, self.socktype, self.proto)
@@ -87,13 +99,16 @@ class Eth(BaseTransport):
             try:
                 self.sock.bind(self._local_address)
             except BaseException as ex:  # noqa: B036
-                msg = f"XCPonEth - Failed to bind socket to given address {self._local_address}"
-                self.logger.critical(msg)
+                msg = f"XCPonEth - Failed to bind socket to given address {self._local_address}: {ex.__class__.__name__}: {ex}"
+                self.logger.critical(
+                    msg, extra={"transport": "eth", "host": self.host, "port": self.port, "protocol": self.protocol}
+                )
                 raise Exception(msg) from ex
         self._packet_listener = threading.Thread(
             target=self._packet_listen,
             args=(),
             kwargs={},
+            daemon=True,
         )
         self._packets = deque()
 
@@ -107,17 +122,23 @@ class Eth(BaseTransport):
     def start_listener(self) -> None:
         super().start_listener()
         if self._packet_listener.is_alive():
-            self._packet_listener.join()
-        self._packet_listener = threading.Thread(target=self._packet_listen)
+            self._packet_listener.join(timeout=2.0)
+        self._packet_listener = threading.Thread(target=self._packet_listen, daemon=True)
         self._packet_listener.start()
 
     def close(self) -> None:
         """Close the transport-layer connection and event-loop."""
         self.finish_listener()
-        if self.listener.is_alive():
-            self.listener.join()
-        if self._packet_listener.is_alive():
-            self._packet_listener.join()
+        try:
+            if self.listener.is_alive():
+                self.listener.join(timeout=2.0)
+        except Exception:
+            pass
+        try:
+            if self._packet_listener.is_alive():
+                self._packet_listener.join(timeout=2.0)
+        except Exception:
+            pass
         self.close_connection()
 
     def _packet_listen(self) -> None:
@@ -160,8 +181,6 @@ class Eth(BaseTransport):
                 break
 
     def listen(self) -> None:
-        HEADER_UNPACK_FROM = self.HEADER.unpack_from
-        HEADER_SIZE = self.HEADER_SIZE
         process_response = self.process_response
         popleft = self._packets.popleft
         close_event_set = self.closeEvent.is_set
@@ -184,17 +203,33 @@ class Eth(BaseTransport):
                 current_position: int = 0
                 while True:
                     if length is None:
-                        if current_size >= HEADER_SIZE:
-                            length, counter = HEADER_UNPACK_FROM(data, current_position)
-                            current_position += HEADER_SIZE
-                            current_size -= HEADER_SIZE
+                        if current_size >= self.framing.header_size:
+                            length, counter = self.framing.unpack_header(bytes(data), initial_offset=current_position)
+                            current_position += self.framing.header_size
+                            current_size -= self.framing.header_size
                         else:
                             data = data[current_position:]
                             break
                     else:
                         if current_size >= length:
                             response = data[current_position : current_position + length]
-                            process_response(response, length, counter, timestamp)
+                            try:
+                                process_response(response, length, counter, timestamp)
+                            except BaseException as ex:  # Guard listener against unhandled exceptions (e.g., disk full in policy)
+                                try:
+                                    self.logger.critical(
+                                        f"Listener error in process_response: {ex.__class__.__name__}: {ex}. Stopping listener.",
+                                        extra={"event": "listener_error"},
+                                    )
+                                except Exception:
+                                    pass
+                                try:
+                                    # Signal all loops to stop
+                                    if hasattr(self, "closeEvent"):
+                                        self.closeEvent.set()
+                                except Exception:
+                                    pass
+                                return
                             current_size -= length
                             current_position += length
                             length = None

pyxcp/transport/hdf5_policy.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+
+import datetime
+from pathlib import Path
+from typing import List, Dict
+
+import h5py
+import numpy as np
+from pyxcp.daq_stim import DaqOnlinePolicy, DaqList
+from pyxcp import __version__ as pyxcp_version
+
+BATCH_SIZE = 4096
+
+MAP_TO_NP = {
+    "U8": np.uint8,
+    "I8": np.int8,
+    "U16": np.uint16,
+    "I16": np.int16,
+    "U32": np.uint32,
+    "I32": np.int32,
+    "U64": np.uint64,
+    "I64": np.int64,
+    "F32": np.float32,
+    "F64": np.float64,
+    "F16": np.float16,
+    "BF16": np.float16,
+}
+
+MAP_TO_ASAM_HO = {
+    "U8": "A_UINT8",
+    "I8": "A_INT8",
+    "U16": "A_UINT16",
+    "I16": "A_INT16",
+    "U32": "A_UINT32",
+    "I32": "A_INT32",
+    "U64": "A_UINT64",
+    "I64": "A_INT64",
+    "F32": "A_FLOAT32",
+    "F64": "A_FLOAT64",
+    "F16": "A_FLOAT16",
+    "BF16": "A_FLOAT16",
+}
+
+
+class BufferedDataset:
+    def __init__(self, dataset: h5py.Dataset):
+        self.dataset = dataset
+        self.buffer: List[int | float] = []
+
+    def add_sample(self, sample: int | float):
+        self.buffer.append(sample)
+        if len(self.buffer) >= BATCH_SIZE:
+            self.flush()
+
+    def flush(self):
+        batch = np.array(self.buffer)
+        self.dataset.resize((self.dataset.shape[0] + len(batch),))
+        self.dataset[-len(batch) :] = batch
+        self.buffer.clear()
+        self.dataset.flush()
+
+    def __len__(self):
+        return len(self.buffer)
+
+
+class DatasetGroup:
+    def __init__(
+        self,
+        ts0_ds: BufferedDataset,
+        ts1_ds: BufferedDataset,
+        datasets: List[BufferedDataset],
+    ):
+        self.ts0_ds = ts0_ds
+        self.ts1_ds = ts1_ds
+        self.datasets = datasets
+
+    def feed(self, ts0: int, ts1: int, *datasets):
+        self.ts0_ds.add_sample(ts0)
+        self.ts1_ds.add_sample(ts1)
+        for dataset, value in zip(self.datasets, datasets):
+            dataset.add_sample(value)
+
+    def finalize(self):
+        for dataset in self.datasets:
+            dataset.flush()
+        self.ts0_ds.flush()
+        self.ts1_ds.flush()
+
+
+def create_timestamp_column(hdf_file: h5py.File, group_name: str, num: int) -> h5py.Dataset:
+    result = hdf_file.create_dataset(
+        f"/{group_name}/timestamp{num}",
+        shape=(0,),
+        maxshape=(None,),
+        dtype=np.uint64,
+        chunks=True,
+    )
+    result.attrs["asam_data_type"] = "A_UINT64"
+    result.attrs["resolution"] = ("1 nanosecond",)
+    return result
+
+
+class Hdf5OnlinePolicy(DaqOnlinePolicy):
+    def __init__(self, file_name: str | Path, daq_lists: List[DaqList], **metadata):
+        super().__init__(daq_lists=daq_lists)
+        path = Path(file_name)
+        if path.suffix != ".h5":
+            path = path.with_suffix(".h5")
+        self.hdf = h5py.File(path, "w", libver="latest")
+        self.metadata = self.set_metadata(**metadata)
+
+    def set_metadata(self, **metadata):
+        basic = {
+            "tool_name": "pyXCP",
+            "tool_version": f"{pyxcp_version}",
+            "created": f"{datetime.datetime.now().astimezone().isoformat()}",
+        }
+        for k, v in (basic | metadata).items():
+            self.hdf.attrs[k] = v
+
+    def initialize(self):
+        self.log.debug("Hdf5OnlinePolicy::Initialize()")
+        self.datasets: Dict[int, DatasetGroup] = {}
+        for num, daq_list in enumerate(self.daq_lists):
+            if daq_list.stim:
+                continue
+            grp = self.hdf.create_group(daq_list.name)
+            grp.attrs["event_num"] = daq_list.event_num
+            grp.attrs["enable_timestamps"] = daq_list.enable_timestamps
+            grp.attrs["prescaler"] = daq_list.prescaler
+            grp.attrs["priority"] = daq_list.priority
+            grp.attrs["direction"] = "STIM" if daq_list.stim else "DAQ"
+            ts0 = BufferedDataset(create_timestamp_column(self.hdf, daq_list.name, 0))
+            ts1 = BufferedDataset(create_timestamp_column(self.hdf, daq_list.name, 1))
+            meas_map = {m.name: m for m in self.daq_lists[num].measurements}
+            dsets = []
+            for name, _ in daq_list.headers:
+                meas = meas_map[name]
+                dataset = self.hdf.create_dataset(
+                    f"/{daq_list.name}/{meas.name}/raw",
+                    shape=(0,),
+                    maxshape=(None,),
+                    dtype=MAP_TO_NP[meas.data_type],
+                    chunks=(1024,),
+                )
+                sub_group = dataset.parent
+                sub_group.attrs["asam_data_type"] = MAP_TO_ASAM_HO.get(meas.data_type, "n/a")
+                dataset.attrs["ecu_address"] = meas.address
+                dataset.attrs["ecu_address_extension"] = meas.ext
+                dsets.append(BufferedDataset(dataset))
+            self.datasets[num] = DatasetGroup(ts0_ds=ts0, ts1_ds=ts1, datasets=dsets)
+        self.hdf.flush()
+
+    def finalize(self):
+        self.log.debug("Hdf5OnlinePolicy::finalize()")
+        if hasattr(self, "datasets"):
+            for group in self.datasets.values():
+                group.finalize()
+        if hasattr(self, "hdf"):
+            self.hdf.close()
+
+    def on_daq_list(self, daq_list: int, timestamp0: int, timestamp1: int, payload: list):
+        group = self.datasets.get(daq_list)
+        if group is None:
+            self.log.warning(f"Received data for unknown DAQ list {daq_list}")
+            return
+        group.feed(timestamp0, timestamp1, *payload)

pyxcp/transport/sxi.py

@@ -1,12 +1,38 @@
-import struct
+import threading
 from collections import deque
 from dataclasses import dataclass
-from typing import Optional
+from typing import Any, Optional
 
 import serial
 
-import pyxcp.types as types
-from pyxcp.transport.base import BaseTransport
+from pyxcp.transport.transport_ext import (
+    SxiFrLBCN,
+    SxiFrLBC8,
+    SxiFrLBC16,
+    SxiFrLCBCN,
+    SxiFrLCBC8,
+    SxiFrLCBC16,
+    SxiFrLFBCN,
+    SxiFrLFBC8,
+    SxiFrLFBC16,
+    SxiFrLWCN,
+    SxiFrLWC8,
+    SxiFrLWC16,
+    SxiFrLCWCN,
+    SxiFrLCWC8,
+    SxiFrLCWC16,
+    SxiFrLFWCN,
+    SxiFrLFWC8,
+    SxiFrLFWC16,
+)
+
+from pyxcp.transport.base import (
+    BaseTransport,
+    ChecksumType,
+    XcpFramingConfig,
+    XcpTransportLayerType,
+    parse_header_format,
+)
 
 
 @dataclass
@@ -19,11 +45,23 @@ class HeaderValues:
 RECV_SIZE = 16384
 
 
+def get_receiver_class(header_format: str, checksum_format: str) -> Any:
+    COLUMN = {"NO_CHECKSUM": 0, "CHECKSUM_BYTE": 1, "CHECKSUM_WORD": 2}
+    FORMATS = {
+        "HEADER_LEN_BYTE": (SxiFrLBCN, SxiFrLBC8, SxiFrLBC16),
+        "HEADER_LEN_CTR_BYTE": (SxiFrLCBCN, SxiFrLCBC8, SxiFrLCBC16),
+        "HEADER_LEN_FILL_BYTE": (SxiFrLFBCN, SxiFrLFBC8, SxiFrLFBC16),
+        "HEADER_LEN_WORD": (SxiFrLWCN, SxiFrLWC8, SxiFrLWC16),
+        "HEADER_LEN_CTR_WORD": (SxiFrLCWCN, SxiFrLCWC8, SxiFrLCWC16),
+        "HEADER_LEN_FILL_WORD": (SxiFrLFWCN, SxiFrLFWC8, SxiFrLFWC16),
+    }
+    return FORMATS[header_format][COLUMN[checksum_format]]
+
+
 class SxI(BaseTransport):
     """"""
 
     def __init__(self, config=None, policy=None, transport_layer_interface: Optional[serial.Serial] = None) -> None:
-        super().__init__(config, policy, transport_layer_interface)
         self.load_config(config)
         self.port_name = self.config.port
         self.baudrate = self.config.bitrate
@@ -31,12 +69,28 @@ class SxI(BaseTransport):
         self.parity = self.config.parity
         self.stopbits = self.config.stopbits
         self.mode = self.config.mode
-        self.header_format = self.config.header_format
+        header_len, header_ctr, header_fill = parse_header_format(self.config.header_format)
+        tail_cs_map = {
+            "NO_CHECKSUM": ChecksumType.NO_CHECKSUM,
+            "CHECKSUM_BYTE": ChecksumType.BYTE_CHECKSUM,
+            "CHECKSUM_WORD": ChecksumType.WORD_CHECKSUM,
+        }
+        # self._listener_running = threading.Event()
+        tail_cs = tail_cs_map[self.config.tail_format]
+        ReceiverKlass = get_receiver_class(self.config.header_format, self.config.tail_format)
+        self.receiver = ReceiverKlass(self.frame_dispatcher)
+        framing_config = XcpFramingConfig(
+            transport_layer_type=XcpTransportLayerType.SXI,
+            header_len=header_len,
+            header_ctr=header_ctr,
+            header_fill=header_fill,
+            tail_fill=False,
+            tail_cs=tail_cs,
+        )
+        super().__init__(config, framing_config, policy, transport_layer_interface)
         self.tail_format = self.config.tail_format
-        self.framing = self.config.framing
         self.esc_sync = self.config.esc_sync
         self.esc_esc = self.config.esc_esc
-        self.make_header()
         self.comm_port: serial.Serial
 
         if self.has_user_supplied_interface and transport_layer_interface:
@@ -50,47 +104,30 @@ class SxI(BaseTransport):
                     bytesize=self.bytesize,
                     parity=self.parity,
                     stopbits=self.stopbits,
-                    timeout=self.timeout,
+                    timeout=0.1,  # self.timeout,
                     write_timeout=self.timeout,
                 )
             except serial.SerialException as e:
                 self.logger.critical(f"XCPonSxI - {e}")
                 raise
-        self._packets = deque()
+        self._condition = threading.Condition()
+        self._frames = deque()
+        # self._frame_listener = threading.Thread(
+        #    target=self._frame_listen,
+        #    args=(),
+        #    kwargs={},
+        # )
 
     def __del__(self) -> None:
         self.close_connection()
 
-    def make_header(self) -> None:
-        def unpack_len(args):
-            (length,) = args
-            return HeaderValues(length=length)
-
-        def unpack_len_counter(args):
-            length, counter = args
-            return HeaderValues(length=length, counter=counter)
-
-        def unpack_len_filler(args):
-            length, filler = args
-            return HeaderValues(length=length, filler=filler)
-
-        HEADER_FORMATS = {
-            "HEADER_LEN_BYTE": ("B", unpack_len),
-            "HEADER_LEN_CTR_BYTE": ("BB", unpack_len_counter),
-            "HEADER_LEN_FILL_BYTE": ("BB", unpack_len_filler),
-            "HEADER_LEN_WORD": ("H", unpack_len),
-            "HEADER_LEN_CTR_WORD": ("HH", unpack_len_counter),
-            "HEADER_LEN_FILL_WORD": ("HH", unpack_len_filler),
-        }
-        fmt, unpacker = HEADER_FORMATS[self.header_format]
-        self.HEADER = struct.Struct(f"<{fmt}")
-        self.HEADER_SIZE = self.HEADER.size
-        self.unpacker = unpacker
-
     def connect(self) -> None:
         self.logger.info(
-            f"XCPonSxI - serial comm_port {self.comm_port.portstr!r} openend [{self.baudrate}/{self.bytesize}-{self.parity}-{self.stopbits}]"
+            f"XCPonSxI - serial comm_port {self.comm_port.portstr!r} openend "
+            f"[{self.baudrate}/{self.bytesize}-{self.parity}-{self.stopbits}] "
+            f"mode: {self.config.mode}"
         )
+        self.logger.info(f"Framing: {self.config.header_format} {self.config.tail_format}")
         self.start_listener()
 
     def output(self, enable) -> None:
@@ -106,26 +143,63 @@ class SxI(BaseTransport):
 
     def start_listener(self) -> None:
         super().start_listener()
+        if hasattr(self, "_frame_listener") and self._frame_listener.is_alive():
+            self._frame_listener.join(timeout=2.0)
+        self._frame_listener = threading.Thread(target=self._frame_listen, daemon=True)
+        self._frame_listener.start()
+        # self._listener_running.wait(2.0)
+
+    def close(self) -> None:
+        """Close the transport-layer connection and event-loop."""
+        self.finish_listener()
+        self.closeEvent.set()
+        try:
+            if self.listener.is_alive():
+                self.listener.join(timeout=2.0)
+        except Exception:
+            pass
+        try:
+            if hasattr(self, "_frame_listener") and self._frame_listener.is_alive():
+                self._frame_listener.join(timeout=2.0)
+        except Exception:
+            pass
+        self.close_connection()
 
     def listen(self) -> None:
         while True:
             if self.closeEvent.is_set():
                 return
-            if not self.comm_port.in_waiting:
-                continue
-
-            recv_timestamp = self.timestamp.value
-            header_values = self.unpacker(self.HEADER.unpack(self.comm_port.read(self.HEADER_SIZE)))
-            length, counter, _ = header_values.length, header_values.counter, header_values.filler
-
-            response = self.comm_port.read(length)
-            self.timing.stop()
-
-            if len(response) != length:
-                raise types.FrameSizeError("Size mismatch.")
-            self.process_response(response, length, counter, recv_timestamp)
-
-    def send(self, frame) -> None:
+            frame_to_process = None
+            with self._condition:
+                while not self._frames:
+                    res = self._condition.wait(1.0)
+                    if not res:
+                        break
+                if self._frames:
+                    frame_to_process = self._frames.popleft()
+
+            if frame_to_process:
+                frame, length, counter, timestamp = frame_to_process
+                self.process_response(frame, length, counter, timestamp)
+
+    def _frame_listen(self) -> None:
+        # self._listener_running.set()
+        while True:
+            if self.closeEvent.is_set():
+                return
+            data = self.comm_port.read(1)
+            if data:
+                self.receiver.feed_bytes(data)
+                data = self.comm_port.read(self.comm_port.in_waiting)
+                if data:
+                    self.receiver.feed_bytes(data)
+
+    def frame_dispatcher(self, data: bytes, length: int, counter: int) -> None:
+        with self._condition:
+            self._frames.append((bytes(data), length, counter, self.timestamp.value))
+            self._condition.notify()
+
+    def send(self, frame: bytes) -> None:
         self.pre_send_timestamp = self.timestamp.value
         self.comm_port.write(frame)
         self.post_send_timestamp = self.timestamp.value