pyxcp 0.22.24__cp310-cp310-win_amd64.whl → 0.22.26__cp310-cp310-win_amd64.whl

pyxcp/__init__.py

@@ -17,4 +17,4 @@ tb_install(show_locals=True, max_frames=3)  # Install custom exception handler.
 
 # if you update this manually, do not forget to update
 # .bumpversion.cfg and pyproject.toml.
-__version__ = "0.22.24"
+__version__ = "0.22.26"

pyxcp/config/__init__.py

@@ -822,9 +822,11 @@ if there is no response to a command.""",
 class General(Configurable):
     """ """
 
-    # loglevel = Unicode("INFO", help="Set the log level by value or name.").tag(config=True)
     disable_error_handling = Bool(False, help="Disable XCP error-handler for performance reasons.").tag(config=True)
     disconnect_response_optional = Bool(False, help="Ignore missing response on DISCONNECT request.").tag(config=True)
+    connect_retries = Integer(help="Number of CONNECT retries (None for infinite retries).", allow_none=True, default_value=3).tag(
+        config=True
+    )
     seed_n_key_dll = Unicode("", allow_none=False, help="Dynamic library used for slave resource unlocking.").tag(config=True)
     seed_n_key_dll_same_bit_width = Bool(False, help="").tag(config=True)
     seed_n_key_function = Callable(

pyxcp/daq_stim/__init__.py

@@ -38,7 +38,7 @@ class DaqProcessor:
     def setup(self, start_datetime: Optional[CurrentDatetime] = None, write_multiple: bool = True):
         if not self.xcp_master.slaveProperties.supportsDaq:
             raise RuntimeError("DAQ functionality is not supported.")
-        self.daq_info = self.xcp_master.getDaqInfo()
+        self.daq_info = self.xcp_master.getDaqInfo(include_event_lists=False)
         if start_datetime is None:
             start_datetime = CurrentDatetime(time_ns())
         self.start_datetime = start_datetime

pyxcp/daq_stim/stim_wrapper.cpp

@@ -15,7 +15,9 @@ using namespace py::literals;
 
 PYBIND11_MODULE(stim, m) {
     py::class_<DaqEventInfo>(m, "DaqEventInfo")
-        .def(py::init<const std::string&, std::int8_t, std::size_t, std::size_t, std::size_t, std::string_view, bool, bool, bool>()
+        .def(py::init<const std::string&, std::int8_t, std::size_t, std::size_t, std::size_t, std::string_view, bool, bool, bool>(),
+        "name"_a, "type_code"_a, "cycle"_a, "max_daq_lists"_a, "priority"_a, "consistency"_a, "daq_supported"_a,
+        "stim_supported"_a, "packed_supported"_a
         );
 
     py::class_<Stim>(m, "Stim")

pyxcp/examples/run_daq.py

@@ -137,8 +137,6 @@ with ap.run(policy=daq_parser) as x:
 
     x.cond_unlock("DAQ")  # DAQ resource is locked in many cases.
 
-    DAQ_LISTS[1].event_num = 0
-
     print("setup DAQ lists.")
     daq_parser.setup()  # Execute setup procedures.
     print("start DAQ lists.")

pyxcp/master/errorhandler.py

@@ -356,6 +356,7 @@ class Executor(SingletonBase):
         self.arguments = arguments
         handler = Handler(inst, func, arguments)
         self.handlerStack.push(handler)
+        connect_retries = inst.config.connect_retries
         try:
             while True:
                 try:
@@ -366,9 +367,14 @@ class Executor(SingletonBase):
                     self.error_code = e.get_error_code()
                     handler.error_code = self.error_code
                 except XcpTimeoutError:
-                    # self.logger.error(f"XcpTimeoutError [{str(e)}]")
+                    is_connect = func.__name__ == "connect"
+                    self.logger.warning(f"XcpTimeoutError -- Service: {func.__name__!r}")
                     self.error_code = XcpError.ERR_TIMEOUT
                     handler.error_code = self.error_code
+                    if is_connect and connect_retries is not None:
+                        if connect_retries == 0:
+                            raise XcpTimeoutError("Maximum CONNECT retries reached.")
+                        connect_retries -= 1
                 except TimeoutError:
                     raise
                 except can.CanError:

pyxcp/master/master.py

@@ -1670,7 +1670,7 @@ class Master:
         self.logger.debug(f"Our checksum          : 0x{cc:08X}")
         return cs.checksum == cc
 
-    def getDaqInfo(self):
+    def getDaqInfo(self, include_event_lists=True):
         """Get DAQ information: processor, resolution, events."""
         result = {}
         dpi = self.getDaqProcessorInfo()
@@ -1711,45 +1711,46 @@ class Master:
         result["resolution"] = resolutionInfo
         channels = []
         daq_events = []
-        for ecn in range(dpi.maxEventChannel):
-            eci = self.getDaqEventInfo(ecn)
-            cycle = eci["eventChannelTimeCycle"]
-            maxDaqList = eci["maxDaqList"]
-            priority = eci["eventChannelPriority"]
-            time_unit = eci["eventChannelTimeUnit"]
-            consistency = eci["daqEventProperties"]["consistency"]
-            daq_supported = eci["daqEventProperties"]["daq"]
-            stim_supported = eci["daqEventProperties"]["stim"]
-            packed_supported = eci["daqEventProperties"]["packed"]
-            name = self.fetch(eci.eventChannelNameLength)
-            if name:
-                name = decode_bytes(name)
-            channel = {
-                "name": name,
-                "priority": eci["eventChannelPriority"],
-                "unit": eci["eventChannelTimeUnit"],
-                "cycle": eci["eventChannelTimeCycle"],
-                "maxDaqList": eci["maxDaqList"],
-                "properties": {
-                    "consistency": consistency,
-                    "daq": daq_supported,
-                    "stim": stim_supported,
-                    "packed": packed_supported,
-                },
-            }
-            daq_event_info = DaqEventInfo(
-                name,
-                types.EVENT_CHANNEL_TIME_UNIT_TO_EXP[time_unit],
-                cycle,
-                maxDaqList,
-                priority,
-                consistency,
-                daq_supported,
-                stim_supported,
-                packed_supported,
-            )
-            daq_events.append(daq_event_info)
-            channels.append(channel)
+        if include_event_lists:
+            for ecn in range(dpi.maxEventChannel):
+                eci = self.getDaqEventInfo(ecn)
+                cycle = eci["eventChannelTimeCycle"]
+                maxDaqList = eci["maxDaqList"]
+                priority = eci["eventChannelPriority"]
+                time_unit = eci["eventChannelTimeUnit"]
+                consistency = eci["daqEventProperties"]["consistency"]
+                daq_supported = eci["daqEventProperties"]["daq"]
+                stim_supported = eci["daqEventProperties"]["stim"]
+                packed_supported = eci["daqEventProperties"]["packed"]
+                name = self.fetch(eci.eventChannelNameLength)
+                if name:
+                    name = decode_bytes(name)
+                channel = {
+                    "name": name,
+                    "priority": eci["eventChannelPriority"],
+                    "unit": eci["eventChannelTimeUnit"],
+                    "cycle": eci["eventChannelTimeCycle"],
+                    "maxDaqList": eci["maxDaqList"],
+                    "properties": {
+                        "consistency": consistency,
+                        "daq": daq_supported,
+                        "stim": stim_supported,
+                        "packed": packed_supported,
+                    },
+                }
+                daq_event_info = DaqEventInfo(
+                    name,
+                    types.EVENT_CHANNEL_TIME_UNIT_TO_EXP[time_unit],
+                    cycle,
+                    maxDaqList,
+                    priority,
+                    consistency,
+                    daq_supported,
+                    stim_supported,
+                    packed_supported,
+                )
+                daq_events.append(daq_event_info)
+                channels.append(channel)
         result["channels"] = channels
         self.stim.setDaqEventInfo(daq_events)
         return result

pyxcp/recorder/__init__.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """XCP Frame Recording Facility.
 """
+
 from dataclasses import dataclass
 from typing import Union
 
@@ -16,15 +17,13 @@ else:
 
 from pyxcp.recorder.rekorder import DaqOnlinePolicy  # noqa: F401
 from pyxcp.recorder.rekorder import (
-    DaqRecorderPolicy,
-    Deserializer,
-    MeasurementParameters,
-    ValueHolder,
-    XcpLogFileDecoder,
-    _PyXcpLogFileReader,
-    _PyXcpLogFileWriter,
-    data_types,
+    DaqRecorderPolicy,      # noqa: F401
+    Deserializer,           # noqa: F401
+    MeasurementParameters,  # noqa: F401
+    ValueHolder,            # noqa: F401
 )
+from pyxcp.recorder.rekorder import XcpLogFileDecoder as _XcpLogFileDecoder
+from pyxcp.recorder.rekorder import _PyXcpLogFileReader, _PyXcpLogFileWriter, data_types
 
 
 DATA_TYPES = data_types()

pyxcp/recorder/converter/__init__.py

@@ -1,8 +1,60 @@
+"""Convert pyXCPs .xmraw files to common data formats.
+"""
+
+import csv
 import logging
+import os
+import sqlite3
 from array import array
 from dataclasses import dataclass, field
+from mmap import PAGESIZE
+from pathlib import Path
 from typing import Any, List
 
+import numpy as np
+from rich.logging import RichHandler
+
+
+try:
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    has_arrow = True
+except ImportError:
+    has_arrow = False
+
+try:
+    import h5py
+
+    has_h5py = True
+except ImportError:
+    has_h5py = False
+
+try:
+    from asammdf import MDF, Signal
+    from asammdf.blocks.v4_blocks import HeaderBlock
+    from asammdf.blocks.v4_constants import FLAG_HD_TIME_OFFSET_VALID
+
+    has_asammdf = True
+except ImportError:
+    has_asammdf = False
+
+try:
+    import xlsxwriter
+
+    has_xlsxwriter = True
+
+except ImportError:
+    has_xlsxwriter = False
+
+from pyxcp import console
+from pyxcp.recorder.rekorder import XcpLogFileDecoder as _XcpLogFileDecoder
+
+
+FORMAT = "%(message)s"
+logging.basicConfig(level="NOTSET", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
+
+log = logging.getLogger("rich")
 
 MAP_TO_ARRAY = {
     "U8": "B",
@@ -19,13 +71,26 @@ MAP_TO_ARRAY = {
     "BF16": "f",
 }
 
-logger = logging.getLogger("PyXCP")
+MAP_TO_NP = {
+    "U8": np.uint8,
+    "I8": np.int8,
+    "U16": np.uint16,
+    "I16": np.int16,
+    "U32": np.uint32,
+    "I32": np.int32,
+    "U64": np.uint64,
+    "I64": np.int64,
+    "F32": np.float32,
+    "F64": np.float64,
+    "F16": np.float16,
+    "BF16": np.float16,
+}
 
 
 @dataclass
 class Storage:
     name: str
-    arrow_type: Any
+    target_type: Any
     arr: array
 
 
@@ -35,3 +100,352 @@ class StorageContainer:
     arr: List[Storage] = field(default_factory=[])
     ts0: List[int] = field(default_factory=lambda: array("Q"))
     ts1: List[int] = field(default_factory=lambda: array("Q"))
+
+
+class XcpLogFileDecoder(_XcpLogFileDecoder):
+    """"""
+
+    def __init__(
+        self,
+        recording_file_name: str,
+        out_file_suffix: str,
+        remove_file: bool = True,
+        target_type_map: dict = None,
+        target_file_name: str = "",
+    ):
+        super().__init__(recording_file_name)
+        self.logger = logging.getLogger("PyXCP")
+        self.logger.setLevel(logging.DEBUG)
+        self.out_file_name = Path(recording_file_name).with_suffix(out_file_suffix)
+        self.out_file_suffix = out_file_suffix
+        self.target_type_map = target_type_map or {}
+        if remove_file:
+            try:
+                os.unlink(self.out_file_name)
+            except FileNotFoundError:
+                pass
+
+    def initialize(self) -> None:
+        self.on_initialize()
+
+    def on_initialize(self) -> None:
+        self.setup_containers()
+
+    def finalize(self) -> None:
+        self.on_finalize()
+
+    def on_finalize(self) -> None:
+        pass
+
+    def setup_containers(self) -> None:
+        self.tables = []
+        for dl in self.daq_lists:
+            result = []
+            for name, type_str in dl.headers:
+                array_txpe = MAP_TO_ARRAY[type_str]
+                target_type = self.target_type_map.get(type_str)
+                sd = Storage(name, target_type, array(array_txpe))
+                result.append(sd)
+            sc = StorageContainer(dl.name, result)
+            self.tables.append(sc)
+            self.on_container(sc)
+
+    def on_container(self, sc: StorageContainer) -> None:
+        pass
+
+
+class CollectRows:
+
+    def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
+        storage_container = self.tables[daq_list_num]
+        storage_container.ts0.append(timestamp0)
+        storage_container.ts1.append(timestamp1)
+        for idx, elem in enumerate(measurements):
+            storage = storage_container.arr[idx]
+            storage.arr.append(elem)
+
+
+class ArrowConverter(CollectRows, XcpLogFileDecoder):
+    """"""
+
+    MAP_TO_ARROW = {
+        "U8": pa.uint8(),
+        "I8": pa.int8(),
+        "U16": pa.uint16(),
+        "I16": pa.int16(),
+        "U32": pa.uint32(),
+        "I32": pa.int32(),
+        "U64": pa.uint64(),
+        "I64": pa.int64(),
+        "F32": pa.float32(),
+        "F64": pa.float64(),
+        "F16": pa.float16(),
+        "BF16": pa.float16(),
+    }
+
+    def __init__(self, recording_file_name: str, target_file_name: str = ""):
+        super().__init__(
+            recording_file_name=recording_file_name,
+            out_file_suffix=".parquet",
+            remove_file=False,
+            target_type_map=self.MAP_TO_ARROW,
+            target_file_name=target_file_name,
+        )
+
+    def on_initialize(self) -> None:
+        super().on_initialize()
+
+    def on_finalize(self) -> None:
+        result = []
+        for arr in self.tables:
+            timestamp0 = arr.ts0
+            timestamp1 = arr.ts1
+            names = ["timestamp0", "timestamp1"]
+            data = [timestamp0, timestamp1]
+            for sd in arr.arr:
+                adt = pa.array(sd.arr, type=sd.target_type)
+                names.append(sd.name)
+                data.append(adt)
+            table = pa.Table.from_arrays(data, names=names)
+            fname = f"{arr.name}{self.out_file_suffix}"
+            self.logger.info(f"Writing file {fname!r}")
+            pq.write_table(table, fname)
+            result.append(table)
+        return result
+
+
+class CsvConverter(XcpLogFileDecoder):
+
+    def __init__(self, recording_file_name: str, target_file_name: str = ""):
+        super().__init__(
+            recording_file_name=recording_file_name, out_file_suffix=".csv", remove_file=False, target_file_name=target_file_name
+        )
+
+    def on_initialize(self) -> None:
+        self.csv_writers = []
+        super().on_initialize()
+
+    def on_container(self, sc: StorageContainer) -> None:
+        fname = f"{sc.name}{self.out_file_suffix}"
+        self.logger.info(f"Creating file {fname!r}.")
+        writer = csv.writer(open(fname, "w", newline=""), dialect="excel")
+        headers = ["ts0", "ts1"] + [e.name for e in sc.arr]
+        writer.writerow(headers)
+        self.csv_writers.append(writer)
+
+    def on_finalize(self) -> None:
+        self.logger.info("Done.")
+
+    def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
+        writer = self.csv_writers[daq_list_num]
+        data = [timestamp0, timestamp1, *measurements]
+        writer.writerow(data)
+
+
+class ExcelConverter(XcpLogFileDecoder):
+
+    def __init__(self, recording_file_name: str, target_file_name: str = ""):
+        super().__init__(recording_file_name=recording_file_name, out_file_suffix=".xlsx", target_file_name=target_file_name)
+
+    def on_initialize(self) -> None:
+        self.logger.info(f"Creating file {str(self.out_file_name)!r}.")
+        self.xls_workbook = xlsxwriter.Workbook(self.out_file_name)
+        self.xls_sheets = []
+        self.rows = []
+        super().on_initialize()
+
+    def on_container(self, sc: StorageContainer) -> None:
+        sheet = self.xls_workbook.add_worksheet(sc.name)
+        self.xls_sheets.append(sheet)
+        headers = ["ts0", "ts1"] + [e.name for e in sc.arr]
+        sheet.write_row(0, 0, headers)
+        self.rows.append(1)
+
+    def on_finalize(self) -> None:
+        self.xls_workbook.close()
+        self.logger.info("Done.")
+
+    def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
+        sheet = self.xls_sheets[daq_list_num]
+        row = self.rows[daq_list_num]
+        data = [timestamp0, timestamp1] + measurements
+        sheet.write_row(row, 0, data)
+        self.rows[daq_list_num] += 1
+
+
+class HdfConverter(CollectRows, XcpLogFileDecoder):
+
+    def __init__(self, recording_file_name: str, target_file_name: str = ""):
+        super().__init__(recording_file_name=recording_file_name, out_file_suffix=".h5", target_file_name=target_file_name)
+
+    def on_initialize(self) -> None:
+        self.logger.info(f"Creating file {str(self.out_file_name)!r}")
+        self.out_file = h5py.File(self.out_file_name, "w")
+        super().on_initialize()
+
+    def on_finalize(self) -> None:
+        for arr in self.tables:
+            timestamp0 = arr.ts0
+            timestamp1 = arr.ts1
+            self.out_file[f"/{arr.name}/timestamp0"] = timestamp0
+            self.out_file[f"/{arr.name}/timestamp1"] = timestamp1
+            for sd in arr.arr:
+                self.out_file[f"/{arr.name}/{sd.name}"] = sd.arr
+            self.logger.info(f"Writing table {arr.name!r}")
+            self.logger.info("Done.")
+        self.out_file.close()
+
+
+class MdfConverter(CollectRows, XcpLogFileDecoder):
+
+    def __init__(self, recording_file_name: str, target_file_name: str = ""):
+        super().__init__(
+            recording_file_name=recording_file_name,
+            out_file_suffix=".mf4",
+            target_type_map=MAP_TO_NP,
+            target_file_name=target_file_name,
+        )
+
+    def on_initialize(self) -> None:
+        super().on_initialize()
+
+    def on_finalize(self) -> None:
+        timestamp_info = self.parameters.timestamp_info
+        hdr = HeaderBlock(
+            abs_time=timestamp_info.timestamp_ns,
+            tz_offset=timestamp_info.utc_offset,
+            daylight_save_time=timestamp_info.dst_offset,
+            time_flags=FLAG_HD_TIME_OFFSET_VALID,
+        )
+        hdr.comment = f"""<HDcomment><TX>Timezone: {timestamp_info.timezone}</TX></HDcomment>"""  # Test-Comment.
+        mdf4 = MDF(version="4.10")
+        mdf4.header = hdr
+        for idx, arr in enumerate(self.tables):
+            signals = []
+            timestamps = arr.ts0
+            for sd in arr.arr:
+                signal = Signal(samples=sd.arr, name=sd.name, timestamps=timestamps)
+                signals.append(signal)
+            self.logger.info(f"Appending data-group {arr.name!r}")
+            mdf4.append(signals, acq_name=arr.name, comment="Created by pyXCP recorder")
+        self.logger.info(f"Writing {str(self.out_file_name)!r}")
+        mdf4.save(self.out_file_name, compression=2, overwrite=True)
+        self.logger.info("Done.")
+
+
+class SqliteConverter(XcpLogFileDecoder):
+    """ """
+
+    MAP_TO_SQL = {
+        "U8": "INTEGER",
+        "I8": "INTEGER",
+        "U16": "INTEGER",
+        "I16": "INTEGER",
+        "U32": "INTEGER",
+        "I32": "INTEGER",
+        "U64": "INTEGER",
+        "I64": "INTEGER",
+        "F32": "FLOAT",
+        "F64": "FLOAT",
+        "F16": "FLOAT",
+        "BF16": "FLOAT",
+    }
+
+    def __init__(self, recording_file_name: str, target_file_name: str = ""):
+        super().__init__(
+            recording_file_name=recording_file_name,
+            out_file_suffix=".sq3",
+            target_type_map=self.MAP_TO_SQL,
+            target_file_name=target_file_name,
+        )
+
+    def on_initialize(self) -> None:
+        self.logger.info(f"Creating database {str(self.out_file_name)!r}.")
+        self.create_database(self.out_file_name)
+        self.insert_stmt = {}
+        super().on_initialize()
+
+    def on_container(self, sc: StorageContainer) -> None:
+        self.create_table(sc)
+        self.logger.info(f"Creating table {sc.name!r}.")
+        self.insert_stmt[sc.name] = (
+            f"""INSERT INTO {sc.name}({', '.join(['ts0', 'ts1'] + [r.name for r in sc.arr])}) VALUES({', '.join(["?" for _ in range(len(sc.arr) + 2)])})"""
+        )
+
+    def on_finalize(self) -> None:
+        self.conn.commit()
+        self.conn.close()
+        print("Done.")
+
+    def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
+        sc = self.tables[daq_list_num]
+        insert_stmt = self.insert_stmt[sc.name]
+        data = [timestamp0, timestamp1, *measurements]
+        self.execute(insert_stmt, data)
+
+    def create_database(self, db_name: str) -> None:
+        self.conn = sqlite3.Connection(db_name)
+        self.cursor = self.conn.cursor()
+        self.execute("PRAGMA FOREIGN_KEYS=ON")
+        self.execute(f"PRAGMA PAGE_SIZE={PAGESIZE}")
+        self.execute("PRAGMA SYNCHRONOUS=OFF")
+        self.execute("PRAGMA LOCKING_MODE=EXCLUSIVE")
+        self.execute("PRAGMA TEMP_STORE=MEMORY")
+
+        timestamp_info = self.parameters.timestamp_info
+        self.execute(
+            "CREATE TABLE timestamp_info(timestamp_ns INTEGER, utc_offset INTEGER, dst_offset INTEGER, timezone VARCHAR(255))"
+        )
+        self.execute("CREATE TABLE table_names(name VARCHAR(255))")
+        self.execute(
+            "INSERT INTO timestamp_info VALUES(?, ?, ?, ?)",
+            [timestamp_info.timestamp_ns, timestamp_info.utc_offset, timestamp_info.dst_offset, timestamp_info.timezone],
+        )
+
+    def create_table(self, sc: StorageContainer) -> None:
+        columns = ["ts0 INTEGER", "ts1 INTEGER"]
+        for elem in sc.arr:
+            columns.append(f"{elem.name} {elem.target_type}")
+        ddl = f"CREATE TABLE {sc.name}({', '.join(columns)})"
+        self.execute(ddl)
+        self.execute("INSERT INTO table_names VALUES(?)", [sc.name])
+
+    def execute(self, *args: List[str]) -> None:
+        try:
+            self.cursor.execute(*args)
+        except Exception as e:
+            print(e)
+
+
+CONVERTERS = {
+    "arrow": ArrowConverter,
+    "csv": CsvConverter,
+    "excel": ExcelConverter,
+    "hdf5": HdfConverter,
+    "mdf": MdfConverter,
+    "sqlite3": SqliteConverter,
+}
+
+CONVERTER_REQUIREMENTS = {
+    "arrow": (has_arrow, "pyarrow"),
+    "csv": (True, "csv"),
+    "excel": (has_xlsxwriter, "xlsxwriter"),
+    "hdf5": (has_h5py, "h5py"),
+    "mdf": (has_asammdf, "asammdf"),
+    "sqlite3": (True, "csv"),
+}
+
+
+def convert_xmraw(converter_name: str, recording_file_name: str, target_file_name: str, *args, **kwargs) -> None:
+    converter_class = CONVERTERS.get(converter_name.lower())
+    if converter_class is None:
+        console.print(f"Invalid converter name: {converter_name!r}")
+        return
+    available, pck_name = CONVERTER_REQUIREMENTS.get(converter_name.lower(), (True, ""))
+    if not available:
+        console.print(f"Converter {converter_name!r} requires package {pck_name!r}.")
+        console.print(f"Please run [green]pip install {pck_name}[/green] to install it.")
+        return
+    # Path(*p.parts[:-1], p.stem)
+    converter = converter_class(recording_file_name)
+    converter.run()