odxtools 6.7.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

odxtools/dtcdop.py

@@ -4,6 +4,8 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
 from xml.etree import ElementTree
 
+from typing_extensions import override
+
 from .compumethods.compumethod import CompuMethod
 from .compumethods.createanycompumethod import create_any_compu_method_from_et
 from .createanydiagcodedtype import create_any_diag_coded_type_from_et
@@ -12,7 +14,7 @@ from .diagcodedtype import DiagCodedType
 from .diagnostictroublecode import DiagnosticTroubleCode
 from .dopbase import DopBase
 from .encodestate import EncodeState
-from .exceptions import DecodeError, EncodeError, odxassert, odxrequire
+from .exceptions import DecodeError, EncodeError, odxassert, odxraise, odxrequire
 from .nameditemlist import NamedItemList
 from .odxlink import OdxDocFragment, OdxLinkDatabase, OdxLinkId, OdxLinkRef
 from .odxtypes import ParameterValue, odxstr_to_bool
@@ -87,17 +89,20 @@ class DtcDop(DopBase):
     def linked_dtc_dops(self) -> NamedItemList["DtcDop"]:
         return self._linked_dtc_dops
 
+    @override
     def decode_from_pdu(self, decode_state: DecodeState) -> ParameterValue:
 
         int_trouble_code = self.diag_coded_type.decode_from_pdu(decode_state)
 
-        if self.compu_method.is_valid_internal_value(int_trouble_code):
-            trouble_code = self.compu_method.convert_internal_to_physical(int_trouble_code)
-        else:
+        if not self.compu_method.is_valid_internal_value(int_trouble_code):
             # TODO: How to prevent this?
-            raise DecodeError(
+            odxraise(
                 f"DTC-DOP {self.short_name} could not convert the coded value "
-                f" {repr(int_trouble_code)} to physical type {self.physical_type.base_data_type}.")
+                f" {repr(int_trouble_code)} to physical type {self.physical_type.base_data_type}.",
+                DecodeError)
+            return
+
+        trouble_code = self.compu_method.convert_internal_to_physical(int_trouble_code)
 
         assert isinstance(trouble_code, int)
 
@@ -110,10 +115,12 @@ class DtcDop(DopBase):
             return dtcs[0]
 
         # the DTC was not specified. This probably means that the
-        # diagnostic description file is incomplete. We do not bail
-        # out but we cannot provide an interpretation for it out of the
-        # box...
-        dtc = DiagnosticTroubleCode(
+        # diagnostic description file is incomplete.
+        odxraise(
+            f"Encountered DTC 0x{trouble_code:06x} which has not been defined "
+            f"by the database", DecodeError)
+
+        return DiagnosticTroubleCode(
             trouble_code=trouble_code,
             odx_id=cast(OdxLinkId, None),
             short_name=f'DTC_{trouble_code:06x}',
@@ -126,12 +133,9 @@ class DtcDop(DopBase):
             sdgs=[],
         )
 
-        return dtc
-
-    def convert_physical_to_bytes(self,
-                                  physical_value: ParameterValue,
-                                  encode_state: EncodeState,
-                                  bit_position: int = 0) -> bytes:
+    @override
+    def encode_into_pdu(self, physical_value: Optional[ParameterValue],
+                        encode_state: EncodeState) -> None:
         if isinstance(physical_value, DiagnosticTroubleCode):
             trouble_code = physical_value.trouble_code
         elif isinstance(physical_value, int):
@@ -147,9 +151,7 @@ class DtcDop(DopBase):
                               f" DiagnosticTroubleCode but got {physical_value!r}.")
 
         internal_trouble_code = self.compu_method.convert_physical_to_internal(trouble_code)
-
-        return self.diag_coded_type.convert_internal_to_bytes(
-            internal_trouble_code, encode_state=encode_state, bit_position=bit_position)
+        self.diag_coded_type.encode_into_pdu(internal_trouble_code, encode_state)
 
     def _build_odxlinks(self) -> Dict[OdxLinkId, Any]:
         odxlinks = super()._build_odxlinks()

odxtools/dynamiclengthfield.py

@@ -3,6 +3,8 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Dict, List
 from xml.etree import ElementTree
 
+from typing_extensions import override
+
 from .decodestate import DecodeState
 from .determinenumberofitems import DetermineNumberOfItems
 from .encodestate import EncodeState
@@ -47,44 +49,46 @@ class DynamicLengthField(Field):
         super()._resolve_snrefs(diag_layer)
         self.determine_number_of_items._resolve_snrefs(diag_layer)
 
-    def convert_physical_to_bytes(
-        self,
-        physical_value: ParameterValue,
-        encode_state: EncodeState,
-        bit_position: int = 0,
-    ) -> bytes:
+    @override
+    def encode_into_pdu(self, physical_value: ParameterValue, encode_state: EncodeState) -> None:
+
+        odxassert(encode_state.cursor_bit_position == 0,
+                  "No bit position can be specified for dynamic length fields!")
 
-        odxassert(bit_position == 0, "No bit position can be specified for dynamic length fields!")
         if not isinstance(physical_value, list):
             odxraise(
                 f"Expected a list of values for dynamic length field {self.short_name}, "
                 f"got {type(physical_value)}", EncodeError)
 
+        # move the origin to the cursor position
+        orig_cursor = encode_state.cursor_byte_position
+        orig_origin = encode_state.origin_byte_position
+        encode_state.origin_byte_position = encode_state.cursor_byte_position
+
         det_num_items = self.determine_number_of_items
-        field_len = det_num_items.dop.convert_physical_to_bytes(
-            len(physical_value), encode_state, det_num_items.bit_position or 0)
-
-        # hack to emplace the length specifier at the correct location
-        tmp = encode_state.coded_message
-        encode_state.coded_message = bytearray()
-        encode_state.emplace_atomic_value(field_len, self.short_name + ".num_items",
-                                          det_num_items.byte_position)
-        result = encode_state.coded_message
-        encode_state.coded_message = tmp
-
-        # if required, add padding between the length specifier and
-        # the first item
-        if len(result) < self.offset:
-            result.extend([0] * (self.offset - len(result)))
-        elif len(result) > self.offset:
+        encode_state.cursor_bit_position = self.determine_number_of_items.bit_position or 0
+        encode_state.cursor_byte_position = encode_state.origin_byte_position + det_num_items.byte_position
+        det_num_items.dop.encode_into_pdu(len(physical_value), encode_state)
+
+        if encode_state.cursor_byte_position - encode_state.origin_byte_position > self.offset:
             odxraise(f"The length specifier of field {self.short_name} overlaps "
-                     f"with the first item!")
+                     f"with its first item!")
+
+        encode_state.cursor_byte_position = encode_state.origin_byte_position + self.offset
+        encode_state.cursor_bit_position = 0
 
         for value in physical_value:
-            result += self.structure.convert_physical_to_bytes(value, encode_state)
+            self.structure.encode_into_pdu(value, encode_state)
 
-        return result
+        # ensure the correct message size if the field is empty
+        if len(physical_value) == 0:
+            encode_state.emplace_bytes(b"")
+
+        # move cursor and origin positions
+        encode_state.origin_byte_position = orig_origin
+        encode_state.cursor_byte_position = max(orig_cursor, encode_state.cursor_byte_position)
 
+    @override
     def decode_from_pdu(self, decode_state: DecodeState) -> ParameterValue:
 
         odxassert(decode_state.cursor_bit_position == 0,

odxtools/encodestate.py

@@ -1,12 +1,15 @@
 # SPDX-License-Identifier: MIT
 import warnings
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Dict, Optional
+from typing import Dict, Optional
 
-from .exceptions import OdxWarning
+from .exceptions import EncodeError, OdxWarning, odxassert, odxraise
+from .odxtypes import AtomicOdxType, DataType
 
-if TYPE_CHECKING:
-    from .tablerow import TableRow
+try:
+    import bitstruct.c as bitstruct
+except ImportError:
+    import bitstruct
 
 
 @dataclass
@@ -15,10 +18,23 @@ class EncodeState:
     """
 
     #: payload that has been constructed so far
-    coded_message: bytearray
+    coded_message: bytearray = field(default_factory=bytearray)
 
-    #: a mapping from short name to value for each parameter
-    parameter_values: Dict[str, Any]
+    #: the bits of the payload that are used
+    used_mask: bytearray = field(default_factory=bytearray)
+
+    #: The absolute position in bytes from the beginning of the PDU to
+    #: which relative positions refer to, e.g., the beginning of the
+    #: structure.
+    origin_byte_position: int = 0
+
+    #: The absolute position in bytes from the beginning of the PDU
+    #: where the next object ought to be placed into the PDU
+    cursor_byte_position: int = 0
+
+    #: The bit position [0-7] where the next object ought to be
+    #: placed into the PDU
+    cursor_bit_position: int = 0
 
     #: If encoding a response: request that triggered the response
     triggering_request: Optional[bytes] = None
@@ -28,31 +44,180 @@ class EncodeState:
     length_keys: Dict[str, int] = field(default_factory=dict)
 
     #: Mapping from the short name of a table-key parameter to the
-    #: corresponding row of the table (specified by TableKeyParameter)
-    table_keys: Dict[str, "TableRow"] = field(default_factory=dict)
+    #: short name of the corresponding row of the table (specified by
+    #: TableKeyParameter)
+    table_keys: Dict[str, str] = field(default_factory=dict)
+
+    #: The cursor position where a given length- or table key is located
+    #: in the PDU
+    key_pos: Dict[str, int] = field(default_factory=dict)
 
     #: Flag whether we are currently the last parameter of the PDU
-    #: (needed for MinMaxLengthType)
-    is_end_of_pdu: bool = False
+    #: (needed for MinMaxLengthType, EndOfPduField, etc.)
+    is_end_of_pdu: bool = True
+
+    def __post_init__(self) -> None:
+        # if a coded message has been specified, but no used_mask, we
+        # assume that all of the bits of the coded message are
+        # currently used.
+        if len(self.coded_message) > len(self.used_mask):
+            self.used_mask += b'\xff' * (len(self.coded_message) - len(self.used_mask))
+        if len(self.coded_message) < len(self.used_mask):
+            odxraise(f"The specified bit mask 0x{self.used_mask.hex()} for used bits "
+                     f"is not suitable for representing the coded_message "
+                     f"0x{self.coded_message.hex()}")
+            self.used_mask = self.used_mask[:len(self.coded_message)]
+
+    def emplace_atomic_value(
+        self,
+        *,
+        internal_value: AtomicOdxType,
+        bit_length: int,
+        base_data_type: DataType,
+        is_highlow_byte_order: bool,
+        used_mask: Optional[bytes],
+    ) -> None:
+        """Convert the internal_value to bytes and emplace this into the PDU"""
+
+        raw_value: AtomicOdxType
+
+        # Check that bytes and strings actually fit into the bit length
+        if base_data_type == DataType.A_BYTEFIELD:
+            if not isinstance(internal_value, bytes):
+                odxraise()
+            if 8 * len(internal_value) > bit_length:
+                raise EncodeError(f"The bytefield {internal_value.hex()} is too large "
+                                  f"({len(internal_value)} bytes)."
+                                  f" The maximum length is {bit_length//8}.")
+            raw_value = internal_value
+        elif base_data_type == DataType.A_ASCIISTRING:
+            if not isinstance(internal_value, str):
+                odxraise()
+
+            # The spec says ASCII, meaning only byte values 0-127.
+            # But in practice, vendors use iso-8859-1, aka latin-1
+            # reason being iso-8859-1 never fails since it has a valid
+            # character mapping for every possible byte sequence.
+            raw_value = internal_value.encode("iso-8859-1")
+
+            if 8 * len(raw_value) > bit_length:
+                raise EncodeError(f"The string {repr(internal_value)} is too large."
+                                  f" The maximum number of characters is {bit_length//8}.")
+        elif base_data_type == DataType.A_UTF8STRING:
+            if not isinstance(internal_value, str):
+                odxraise()
+
+            raw_value = internal_value.encode("utf-8")
+
+            if 8 * len(raw_value) > bit_length:
+                raise EncodeError(f"The string {repr(internal_value)} is too large."
+                                  f" The maximum number of bytes is {bit_length//8}.")
+
+        elif base_data_type == DataType.A_UNICODE2STRING:
+            if not isinstance(internal_value, str):
+                odxraise()
 
-    def emplace_atomic_value(self,
-                             new_data: bytes,
-                             param_name: str,
-                             pos: Optional[int] = None) -> None:
-        if pos is None:
-            pos = len(self.coded_message)
+            text_encoding = "utf-16-be" if is_highlow_byte_order else "utf-16-le"
+            raw_value = internal_value.encode(text_encoding)
+
+            if 8 * len(raw_value) > bit_length:
+                raise EncodeError(f"The string {repr(internal_value)} is too large."
+                                  f" The maximum number of characters is {bit_length//16}.")
+        else:
+            raw_value = internal_value
+
+        # If the bit length is zero, return empty bytes
+        if bit_length == 0:
+            if (base_data_type.value in [
+                    DataType.A_INT32, DataType.A_UINT32, DataType.A_FLOAT32, DataType.A_FLOAT64
+            ] and base_data_type.value != 0):
+                odxraise(
+                    f"The number {repr(internal_value)} cannot be encoded into {bit_length} bits.",
+                    EncodeError)
+            self.emplace_bytes(b'')
+            return
+
+        char = base_data_type.bitstruct_format_letter
+        padding = (8 - ((bit_length + self.cursor_bit_position) % 8)) % 8
+        odxassert((0 <= padding and padding < 8 and
+                   (padding + bit_length + self.cursor_bit_position) % 8 == 0),
+                  f"Incorrect padding {padding}")
+        left_pad = f"p{padding}" if padding > 0 else ""
+
+        # actually encode the value
+        coded = bitstruct.pack(f"{left_pad}{char}{bit_length}", raw_value)
+
+        # create the raw mask of used bits for numeric objects
+        used_mask_raw = used_mask
+        if base_data_type in [DataType.A_INT32, DataType.A_UINT32
+                             ] and (self.cursor_bit_position != 0 or
+                                    (self.cursor_bit_position + bit_length) % 8 != 0):
+            if used_mask is None:
+                tmp = (1 << bit_length) - 1
+            else:
+                tmp = int.from_bytes(used_mask, "big")
+            tmp <<= self.cursor_bit_position
+
+            used_mask_raw = tmp.to_bytes((self.cursor_bit_position + bit_length + 7) // 8, "big")
+
+        # apply byte order to numeric objects
+        if not is_highlow_byte_order and base_data_type in [
+                DataType.A_INT32, DataType.A_UINT32, DataType.A_FLOAT32, DataType.A_FLOAT64
+        ]:
+            coded = coded[::-1]
+
+            if used_mask_raw is not None:
+                used_mask_raw = used_mask_raw[::-1]
+
+        self.cursor_bit_position = 0
+        self.emplace_bytes(coded, obj_used_mask=used_mask_raw)
+
+    def emplace_bytes(self,
+                      new_data: bytes,
+                      obj_name: Optional[str] = None,
+                      obj_used_mask: Optional[bytes] = None) -> None:
+        if self.cursor_bit_position != 0:
+            odxraise("EncodeState.emplace_bytes can only be called "
+                     "for a bit position of 0!", RuntimeError)
+
+        pos = self.cursor_byte_position
 
         # Make blob longer if necessary
         min_length = pos + len(new_data)
         if len(self.coded_message) < min_length:
-            self.coded_message.extend([0] * (min_length - len(self.coded_message)))
+            pad = b'\x00' * (min_length - len(self.coded_message))
+            self.coded_message += pad
+            self.used_mask += pad
+
+        if obj_used_mask is None:
+            # Happy path for when no obj_used_mask has been
+            # specified. In this case we assume that all bits of the
+            # new data to be emplaced are used.
+            n = len(new_data)
 
-        for byte_idx_val, byte_idx_rpc in enumerate(range(pos, pos + len(new_data))):
-            # insert byte value
-            if self.coded_message[byte_idx_rpc] & new_data[byte_idx_val] != 0:
+            if self.used_mask[pos:pos + n] != b'\x00' * n:
                 warnings.warn(
-                    f"Object '{param_name}' overlaps with another parameter (bytes are already set)",
+                    f"Overlapping objects detected in between bytes {pos} and "
+                    f"{pos+n}",
                     OdxWarning,
                     stacklevel=1,
                 )
-            self.coded_message[byte_idx_rpc] |= new_data[byte_idx_val]
+            self.coded_message[pos:pos + n] = new_data
+            self.used_mask[pos:pos + n] = b'\xff' * n
+        else:
+            # insert data the hard way, i.e. we have to look at each
+            # individual byte to determine if it has already been used
+            # somewhere else (it would be nice if bytearrays supported
+            # bitwise operations!)
+            for i in range(len(new_data)):
+                if self.used_mask[pos + i] & obj_used_mask[i] != 0:
+                    warnings.warn(
+                        f"Overlapping objects detected at position {pos + i}",
+                        OdxWarning,
+                        stacklevel=1,
+                    )
+                self.coded_message[pos + i] &= ~obj_used_mask[i]
+                self.coded_message[pos + i] |= new_data[i] & obj_used_mask[i]
+                self.used_mask[pos + i] |= obj_used_mask[i]
+
+        self.cursor_byte_position += len(new_data)

odxtools/endofpdufield.py

@@ -3,6 +3,8 @@ from dataclasses import dataclass
 from typing import List, Optional
 from xml.etree import ElementTree
 
+from typing_extensions import override
+
 from .decodestate import DecodeState
 from .encodestate import EncodeState
 from .exceptions import EncodeError, odxassert, odxraise
@@ -39,26 +41,22 @@ class EndOfPduField(Field):
 
         return eopf
 
-    def convert_physical_to_bytes(
-        self,
-        physical_value: ParameterValue,
-        encode_state: EncodeState,
-        bit_position: int = 0,
-    ) -> bytes:
+    @override
+    def encode_into_pdu(self, physical_value: Optional[ParameterValue],
+                        encode_state: EncodeState) -> None:
+        odxassert(not encode_state.cursor_bit_position,
+                  "No bit position can be specified for end-of-pdu fields!")
 
-        odxassert(
-            bit_position == 0, "End of PDU field must be byte aligned. "
-            "Is there an error in reading the .odx?", EncodeError)
         if not isinstance(physical_value, list):
             odxraise(
-                f"Expected a list of values for end-of-pdu field {self.short_name}, "
-                f"got {type(physical_value)}", EncodeError)
+                f"Invalid type {type(physical_value).__name__} of physical "
+                f"value for end-of-pdu field, expected a list", EncodeError)
+            return
 
-        coded_message = b''
         for value in physical_value:
-            coded_message += self.structure.convert_physical_to_bytes(value, encode_state)
-        return coded_message
+            self.structure.encode_into_pdu(value, encode_state)
 
+    @override
     def decode_from_pdu(self, decode_state: DecodeState) -> ParameterValue:
         odxassert(not decode_state.cursor_bit_position,
                   "No bit position can be specified for end-of-pdu fields!")

odxtools/environmentdatadescription.py

@@ -1,13 +1,15 @@
 # SPDX-License-Identifier: MIT
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 from xml.etree import ElementTree
 
+from typing_extensions import override
+
 from .complexdop import ComplexDop
 from .decodestate import DecodeState
 from .encodestate import EncodeState
 from .environmentdata import EnvironmentData
-from .exceptions import DecodeError, EncodeError, odxrequire
+from .exceptions import odxraise, odxrequire
 from .odxlink import OdxDocFragment, OdxLinkDatabase, OdxLinkId, OdxLinkRef
 from .odxtypes import ParameterValue
 from .utils import dataclass_fields_asdict
@@ -86,23 +88,21 @@ class EnvironmentDataDescription(ComplexDop):
             for ed in self.env_datas:
                 ed._resolve_snrefs(diag_layer)
 
-    def convert_physical_to_bytes(self,
-                                  physical_value: ParameterValue,
-                                  encode_state: EncodeState,
-                                  bit_position: int = 0) -> bytes:
-        """Convert the physical value into bytes.
+    @override
+    def encode_into_pdu(self, physical_value: Optional[ParameterValue],
+                        encode_state: EncodeState) -> None:
+        """Convert a physical value into bytes and emplace them into a PDU.
 
         Since environmental data is supposed to never appear on the
         wire, this method just raises an EncodeError exception.
         """
-        raise EncodeError("EnvironmentDataDescription DOPs cannot be encoded or decoded")
+        odxraise("EnvironmentDataDescription DOPs cannot be encoded or decoded")
 
-    def convert_bytes_to_physical(self,
-                                  decode_state: DecodeState,
-                                  bit_position: int = 0) -> Tuple[ParameterValue, int]:
-        """Extract the bytes from the PDU and convert them to the physical value.
+    @override
+    def decode_from_pdu(self, decode_state: DecodeState) -> ParameterValue:
+        """Extract the bytes from a PDU and convert them to a physical value.
 
         Since environmental data is supposed to never appear on the
         wire, this method just raises an DecodeError exception.
         """
-        raise DecodeError("EnvironmentDataDescription DOPs cannot be encoded or decoded")
+        odxraise("EnvironmentDataDescription DOPs cannot be encoded or decoded")

odxtools/leadinglengthinfotype.py

@@ -1,11 +1,13 @@
 # SPDX-License-Identifier: MIT
 from dataclasses import dataclass
-from typing import Any, Optional
+from typing import Optional
+
+from typing_extensions import override
 
 from .decodestate import DecodeState
 from .diagcodedtype import DctType, DiagCodedType
 from .encodestate import EncodeState
-from .exceptions import odxassert, odxraise
+from .exceptions import EncodeError, odxassert, odxraise
 from .odxtypes import AtomicOdxType, DataType
 
 
@@ -42,29 +44,40 @@ class LeadingLengthInfoType(DiagCodedType):
         # DCT is dynamic!
         return None
 
-    def convert_internal_to_bytes(self, internal_value: Any, encode_state: EncodeState,
-                                  bit_position: int) -> bytes:
+    @override
+    def encode_into_pdu(self, internal_value: AtomicOdxType, encode_state: EncodeState) -> None:
+
+        if not isinstance(internal_value, (str, bytes)):
+            odxraise(
+                f"LEADING-LENGTH-INFO types can only be used for strings and byte fields, "
+                f"not {type(internal_value).__name__}", EncodeError)
+            return
 
         byte_length = self._minimal_byte_length_of(internal_value)
 
-        length_bytes = self._encode_internal_value(
-            byte_length,
-            bit_position=bit_position,
+        used_mask = None
+        bit_pos = encode_state.cursor_bit_position
+        if encode_state.cursor_bit_position != 0 or (bit_pos + self.bit_length) % 8 != 0:
+            used_mask = (1 << self.bit_length) - 1
+            used_mask <<= bit_pos
+
+        encode_state.emplace_atomic_value(
+            internal_value=byte_length,
+            used_mask=None,
             bit_length=self.bit_length,
             base_data_type=DataType.A_UINT32,
             is_highlow_byte_order=self.is_highlow_byte_order,
         )
 
-        value_bytes = self._encode_internal_value(
-            internal_value,
-            bit_position=0,
+        encode_state.emplace_atomic_value(
+            internal_value=internal_value,
+            used_mask=None,
             bit_length=8 * byte_length,
             base_data_type=self.base_data_type,
             is_highlow_byte_order=self.is_highlow_byte_order,
         )
 
-        return length_bytes + value_bytes
-
+    @override
     def decode_from_pdu(self, decode_state: DecodeState) -> AtomicOdxType:
 
         # Extract length of the parameter value