pyxcp 0.25.2__cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl

pyxcp/recorder/reader.hpp

@@ -0,0 +1,138 @@
+
+#ifndef RECORDER_READER_HPP
+#define RECORDER_READER_HPP
+
+#include <iostream>
+
+class XcpLogFileReader {
+   public:
+
+    explicit XcpLogFileReader(const std::string &file_name) {
+        if (!file_name.ends_with(detail::FILE_EXTENSION)) {
+            m_file_name = file_name + detail::FILE_EXTENSION;
+        } else {
+            m_file_name = file_name;
+        }
+
+        m_mmap = new mio::mmap_source(m_file_name);
+        blob_t magic[detail::MAGIC_SIZE + 1];
+
+        read_bytes(0UL, detail::MAGIC_SIZE, magic);
+        if (memcmp(detail::MAGIC.c_str(), magic, detail::MAGIC_SIZE) != 0) {
+            throw std::runtime_error("Invalid file magic.");
+        }
+        m_offset = detail::MAGIC_SIZE;
+
+        read_bytes(m_offset, detail::FILE_HEADER_SIZE, reinterpret_cast<blob_t *>(&m_header));
+        // printf("Sizes: %u %u %.3f\n", m_header.size_uncompressed,
+        //        m_header.size_compressed,
+        //        float(m_header.size_uncompressed) / float(m_header.size_compressed));
+        if (m_header.hdr_size != detail::FILE_HEADER_SIZE + detail::MAGIC_SIZE) {
+            throw std::runtime_error("File header size does not match.");
+        }
+        if (detail::VERSION != m_header.version) {
+            throw std::runtime_error("File version mismatch.");
+        }
+#if 0
+        if (m_header.num_containers < 1) {
+            throw std::runtime_error("At least one container required.");
+        }
+#endif
+        m_offset += detail::FILE_HEADER_SIZE;
+
+        if ((m_header.options & XMRAW_HAS_METADATA) == XMRAW_HAS_METADATA) {
+            std::size_t metadata_length = 0;
+            std::size_t data_start      = m_offset + sizeof(std::size_t);
+
+            read_bytes(m_offset, sizeof(std::size_t), reinterpret_cast<blob_t *>(&metadata_length));
+
+            std::copy(ptr(data_start), ptr(data_start + metadata_length), std::back_inserter(m_metadata));
+            m_offset += (metadata_length + sizeof(std::size_t));
+        }
+    }
+
+    [[nodiscard]] FileHeaderType get_header() const noexcept {
+        return m_header;
+    }
+
+    [[nodiscard]] auto get_header_as_tuple() const noexcept -> HeaderTuple {
+        auto hdr = get_header();
+
+        return std::make_tuple(
+            hdr.version, hdr.options, hdr.num_containers, hdr.record_count, hdr.size_uncompressed, hdr.size_compressed,
+            (double)((std::uint64_t)(((double)hdr.size_uncompressed / (double)hdr.size_compressed * 100.0) + 0.5)) / 100.0
+        );
+    }
+
+    [[nodiscard]] auto get_metadata() const noexcept {
+        return m_metadata;
+    }
+
+    void reset() noexcept {
+        m_current_container = 0;
+        m_offset            = file_header_size();
+    }
+
+    std::optional<FrameVector> next_block() {
+        auto          container = ContainerHeaderType{};
+        auto          frame     = frame_header_t{};
+        std::uint64_t boffs     = 0;
+        auto          result    = FrameVector{};
+        payload_t     payload;
+
+        if (m_current_container >= m_header.num_containers) {
+            return std::nullopt;
+        }
+        read_bytes(m_offset, detail::CONTAINER_SIZE, reinterpret_cast<blob_t *>(&container));
+        __ALIGN auto buffer = new blob_t[container.size_uncompressed];
+        m_offset += detail::CONTAINER_SIZE;
+        result.reserve(container.record_count);
+        const int uc_size = ::LZ4_decompress_safe(
+            reinterpret_cast<char const *>(ptr(m_offset)), reinterpret_cast<char *>(buffer), container.size_compressed,
+            container.size_uncompressed
+        );
+        if (uc_size < 0) {
+            throw std::runtime_error("LZ4 decompression failed.");
+        }
+        boffs = 0;
+        for (std::uint64_t idx = 0; idx < container.record_count; ++idx) {
+            _fcopy(reinterpret_cast<char *>(&frame), reinterpret_cast<char const *>(&(buffer[boffs])), sizeof(frame_header_t));
+            boffs += sizeof(frame_header_t);
+            result.emplace_back(
+                frame.category, frame.counter, frame.timestamp, frame.length, create_payload(frame.length, &buffer[boffs])
+            );
+            boffs += frame.length;
+        }
+        m_offset += container.size_compressed;
+        m_current_container += 1;
+        delete[] buffer;
+
+        return std::optional<FrameVector>{ result };
+    }
+
+    ~XcpLogFileReader() noexcept {
+        delete m_mmap;
+    }
+
+   protected:
+
+    [[nodiscard]] blob_t const *ptr(std::uint64_t pos = 0) const {
+        return reinterpret_cast<blob_t const *>(m_mmap->data() + pos);
+    }
+
+    void read_bytes(std::uint64_t pos, std::uint64_t count, blob_t *buf) const {
+        auto addr = reinterpret_cast<char const *>(ptr(pos));
+        _fcopy(reinterpret_cast<char *>(buf), addr, count);
+    }
+
+   private:
+
+    std::string       m_file_name;
+    std::uint64_t     m_offset{ 0 };
+    std::uint64_t     m_current_container{ 0 };
+    mio::mmap_source *m_mmap{ nullptr };
+    FileHeaderType    m_header;
+    std::string       m_metadata;
+};
+
+#endif  // RECORDER_READER_HPP

pyxcp/recorder/reco.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python
+"""Raw XCP traffic recorder.
+
+Data is stored in LZ4 compressed containers.
+
+Examples
+--------
+
+See
+
+- ``_ for recording / writing
+
+-  ``_ for reading.
+"""
+
+import enum
+import mmap
+import struct
+from collections import namedtuple
+
+import lz4.block as lz4block
+
+
+FILE_EXTENSION = ".xmraw"  # XCP Measurement / raw data.
+
+MAGIC = b"ASAMINT::XCP_RAW"
+
+FILE_HEADER_STRUCT = struct.Struct(f"<{len(MAGIC):d}sHHHLLLL")
+FileHeader = namedtuple(
+    "FileHeader",
+    "magic hdr_size version options num_containers record_count size_compressed size_uncompressed",
+)
+
+CONTAINER_HEADER_STRUCT = struct.Struct("<LLL")
+ContainerHeader = namedtuple("ContainerHeader", "record_count size_compressed size_uncompressed")
+
+DAQ_RECORD_STRUCT = struct.Struct("<BHdL")
+DAQRecord = namedtuple("DAQRecord", "category counter timestamp payload")
+
+
+class XcpLogCategory(enum.IntEnum):
+    """ """
+
+    DAQ = 1
+
+
+class XcpLogFileParseError(Exception):
+    """Log file is damaged is some way."""
+
+    pass
+
+
+class XcpLogFileCapacityExceededError(Exception):
+    pass
+
+
+class XcpLogFileWriter:
+    """
+    Parameters
+    ----------
+    file_name: str
+        Don't specify extension.
+
+    prealloc: int
+        Pre-allocate a sparse file (size in MB).
+
+    chunk_size: int
+        Number of kilobytes to collect before compressing.
+
+    compression_level: int
+        s. LZ4 documentation.
+    """
+
+    def __init__(
+        self,
+        file_name: str,
+        prealloc: int = 10,
+        chunk_size: int = 1024,
+        compression_level: int = 9,
+    ):
+        self._is_closed = True
+        try:
+            self._of = open(f"{file_name}{FILE_EXTENSION}", "w+b")
+        except Exception:
+            raise
+        else:
+            self._of.truncate(1024 * 1024 * prealloc)  # Create sparse file (hopefully).
+            self._mapping = mmap.mmap(self._of.fileno(), 0)
+        self.container_header_offset = FILE_HEADER_STRUCT.size
+        self.current_offset = self.container_header_offset + CONTAINER_HEADER_STRUCT.size
+        self.total_size_uncompressed = self.total_size_compressed = 0
+        self.container_size_uncompressed = self.container_size_compressed = 0
+        self.total_record_count = 0
+        self.chunk_size = chunk_size * 1024
+        self.num_containers = 0
+        self.intermediate_storage = []
+        self.compression_level = compression_level
+        self.prealloc = prealloc
+        self._is_closed = False
+
+    def add_xcp_frames(self, xcp_frames: list):
+        for counter, timestamp, raw_data in xcp_frames:
+            length = len(raw_data)
+            item = DAQ_RECORD_STRUCT.pack(1, counter, timestamp, length) + raw_data
+            self.intermediate_storage.append(item)
+            self.container_size_uncompressed += len(item)
+            if self.container_size_uncompressed > self.chunk_size:
+                self._compress_framez()
+
+    def _compress_framez(self):
+        compressed_data = lz4block.compress(b"".join(self.intermediate_storage), compression=self.compression_level)
+        record_count = len(self.intermediate_storage)
+        hdr = CONTAINER_HEADER_STRUCT.pack(record_count, len(compressed_data), self.container_size_uncompressed)
+        self.set(self.current_offset, compressed_data)
+        self.set(self.container_header_offset, hdr)
+        self.container_header_offset = self.current_offset + len(compressed_data)
+        self.current_offset = self.container_header_offset + CONTAINER_HEADER_STRUCT.size
+        self.intermediate_storage = []
+        self.total_record_count += record_count
+        self.num_containers += 1
+        self.total_size_uncompressed += self.container_size_uncompressed
+        self.total_size_compressed += len(compressed_data)
+        self.container_size_uncompressed = 0
+        self.container_size_compressed = 0
+
+    def __del__(self):
+        if not self._is_closed:
+            self.close()
+
+    def close(self):
+        if not self._is_closed:
+            if hasattr(self, "_mapping"):
+                if self.intermediate_storage:
+                    self._compress_framez()
+                self._write_header(
+                    version=0x0100,
+                    options=0x0000,
+                    num_containers=self.num_containers,
+                    record_count=self.total_record_count,
+                    size_compressed=self.total_size_compressed,
+                    size_uncompressed=self.total_size_uncompressed,
+                )
+                self._mapping.flush()
+                self._mapping.close()
+                self._of.truncate(self.current_offset)
+            self._of.close()
+            self._is_closed = True
+
+    def set(self, address: int, data: bytes):
+        """Write to memory mapped file.
+
+        Parameters
+        ----------
+        address: int
+
+        data: bytes-like
+        """
+        length = len(data)
+        try:
+            self._mapping[address : address + length] = data
+        except IndexError:
+            raise XcpLogFileCapacityExceededError(f"Maximum file size of {self.prealloc} MBytes exceeded.") from None
+
+    def _write_header(
+        self,
+        version,
+        options,
+        num_containers,
+        record_count,
+        size_compressed,
+        size_uncompressed,
+    ):
+        hdr = FILE_HEADER_STRUCT.pack(
+            MAGIC,
+            FILE_HEADER_STRUCT.size,
+            version,
+            options,
+            num_containers,
+            record_count,
+            size_compressed,
+            size_uncompressed,
+        )
+        self.set(0x00000000, hdr)
+
+    @property
+    def compression_ratio(self):
+        if self.total_size_compressed:
+            return self.total_size_uncompressed / self.total_size_compressed
+
+
+class XcpLogFileReader:
+    """
+    Parameters
+    ----------
+    file_name: str
+        Don't specify extension.
+    """
+
+    def __init__(self, file_name):
+        self._is_closed = True
+        try:
+            self._log_file = open(f"{file_name}{FILE_EXTENSION}", "r+b")
+        except Exception:
+            raise
+        else:
+            self._mapping = mmap.mmap(self._log_file.fileno(), 0)
+        self._is_closed = False
+        (
+            magic,
+            _,
+            _,
+            _,
+            self.num_containers,
+            self.total_record_count,
+            self.total_size_compressed,
+            self.total_size_uncompressed,
+        ) = FILE_HEADER_STRUCT.unpack(self.get(0, FILE_HEADER_STRUCT.size))
+        if magic != MAGIC:
+            raise XcpLogFileParseError(f"Invalid file magic: {magic!r}.")
+
+    def __del__(self):
+        if not self._is_closed:
+            self.close()
+
+    @property
+    def frames(self):
+        """Iterate over all frames in file.
+
+        Yields
+        ------
+        DAQRecord
+        """
+        offset = FILE_HEADER_STRUCT.size
+        for _ in range(self.num_containers):
+            (
+                record_count,
+                size_compressed,
+                size_uncompressed,
+            ) = CONTAINER_HEADER_STRUCT.unpack(self.get(offset, CONTAINER_HEADER_STRUCT.size))
+            offset += CONTAINER_HEADER_STRUCT.size
+            uncompressed_data = memoryview(lz4block.decompress(self.get(offset, size_compressed)))
+            frame_offset = 0
+            for _ in range(record_count):
+                category, counter, timestamp, frame_length = DAQ_RECORD_STRUCT.unpack(
+                    uncompressed_data[frame_offset : frame_offset + DAQ_RECORD_STRUCT.size]
+                )
+                frame_offset += DAQ_RECORD_STRUCT.size
+                frame_data = uncompressed_data[frame_offset : frame_offset + frame_length]  # .tobytes()
+                frame_offset += len(frame_data)
+                frame = DAQRecord(category, counter, timestamp, frame_data)
+                yield frame
+            offset += size_compressed
+
+    def get(self, address: int, length: int):
+        """Read from memory mapped file.
+
+        Parameters
+        ----------
+        address: int
+
+        length: int
+
+        Returns
+        -------
+        memoryview
+        """
+        return self._mapping[address : address + length]
+
+    def close(self):
+        if hasattr(self, "self._mapping"):
+            self._mapping.close()
+        self._log_file.close()
+        self._is_closed = True
+
+    @property
+    def compression_ratio(self):
+        if self.total_size_compressed:
+            return self.total_size_uncompressed / self.total_size_compressed

pyxcp/recorder/rekorder.cpp

@@ -0,0 +1,59 @@
+
+#define STANDALONE_REKORDER (1)
+
+#include "rekorder.hpp"
+
+void some_records(XcpLogFileWriter& writer) {
+    const auto COUNT  = 1024 * 100 * 5;
+    unsigned   filler = 0x00;
+    char       buffer[1024];
+
+    for (auto idx = 0; idx < COUNT; ++idx) {
+        auto fr      = frame_header_t{};
+        fr.category  = 1;
+        fr.counter   = idx;
+        fr.timestamp = std::clock();
+        fr.length    = 10 + (rand() % 240);
+        filler       = (filler + 1) % 16;
+        memset(buffer, filler, fr.length);
+        writer.add_frame(fr.category, fr.counter, fr.timestamp, fr.length, std::bit_cast<char const *>(&buffer));
+    }
+}
+
+int main() {
+    srand(42);
+
+    printf("\nWRITER\n");
+    printf("======\n");
+
+    auto writer = XcpLogFileWriter("test_logger", 250, 1);
+    some_records(writer);
+    writer.finalize();
+
+    printf("\nREADER\n");
+    printf("======\n");
+
+    auto reader = XcpLogFileReader("test_logger");
+    auto header = reader.get_header();
+
+    printf("size:               %u\n", header.hdr_size);
+    printf("version:            %u\n", header.version);
+    printf("options:            %u\n", header.options);
+    printf("containers:         %u\n", header.num_containers);
+    printf("records:            %u\n", header.record_count);
+    printf("size/compressed:    %u\n", header.size_compressed);
+    printf("size/uncompressed:  %u\n", header.size_uncompressed);
+    printf("compression ratio:  %.2f\n", static_cast<float>(header.size_uncompressed) / static_cast<float>(header.size_compressed));
+
+    while (true) {
+        const auto& frames = reader.next_block();
+        if (!frames) {
+            break;
+        }
+        for (const auto& frame : frames.value()) {
+            auto const& [category, counter, timestamp, length, payload] = frame;
+        }
+    }
+    printf("---\n");
+    printf("Finished.\n");
+}