objutils 0.10.2__tar.gz → 0.10.4__tar.gz

{objutils-0.10.2 → objutils-0.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: objutils
-Version: 0.10.2
+Version: 0.10.4
 Summary: Objectfile library for Python
 License: GPLv2
 License-File: LICENSE
@@ -36,7 +36,7 @@ Readme
 .. image:: https://github.com/christoph2/objutils/raw/master/docs/objutils_banner.png
     :align: center
 
-|PyPI| |Python Versions| |License: GPLv2| |Code style: black|
+|PyPI| |Python Versions| |License: GPLv2| |Code style: black| |Ask DeepWiki| |PDF Manual|
 
 
 Binary data stored in hex-files is in widespread use especially in embedded systems applications.
@@ -467,4 +467,8 @@ If you contribute code to this project, you are implicitly allowing your code to
    :target: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 .. |Code style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg
    :target: https://github.com/psf/black
+.. |Ask DeepWiki| image:: https://deepwiki.com/badge.svg
+   :target: https://deepwiki.com/christoph2/objutils
+.. |PDF Manual| image:: https://img.shields.io/badge/docs-PDF%20manual-blue.svg
+   :target: https://objutils.readthedocs.io/_/downloads/en/latest/pdf/
 

{objutils-0.10.2 → objutils-0.10.4}/docs/README.rst

@@ -5,7 +5,7 @@ Readme
 .. image:: https://github.com/christoph2/objutils/raw/master/docs/objutils_banner.png
     :align: center
 
-|PyPI| |Python Versions| |License: GPLv2| |Code style: black|
+|PyPI| |Python Versions| |License: GPLv2| |Code style: black| |Ask DeepWiki| |PDF Manual|
 
 
 Binary data stored in hex-files is in widespread use especially in embedded systems applications.
@@ -436,3 +436,7 @@ If you contribute code to this project, you are implicitly allowing your code to
    :target: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 .. |Code style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg
    :target: https://github.com/psf/black
+.. |Ask DeepWiki| image:: https://deepwiki.com/badge.svg
+   :target: https://deepwiki.com/christoph2/objutils
+.. |PDF Manual| image:: https://img.shields.io/badge/docs-PDF%20manual-blue.svg
+   :target: https://objutils.readthedocs.io/_/downloads/en/latest/pdf/

{objutils-0.10.2 → objutils-0.10.4}/objutils/__init__.py

@@ -13,7 +13,7 @@ Registers CODECS and implements an interface to them.
 The first parameter is always the codec name.
 """
 
-__version__ = "0.10.2"
+__version__ = "0.10.4"
 
 __all__ = [
     "Image",

{objutils-0.10.2 → objutils-0.10.4}/objutils/elf/model.py

@@ -130,7 +130,7 @@ from sqlalchemy.engine import Engine
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
 from sqlalchemy.orm import Session, declarative_base, relationship
-from sqlalchemy.pool import NullPool
+from sqlalchemy.pool import NullPool, StaticPool
 from sqlalchemy.sql import func
 
 from objutils.elf import defs
@@ -1312,6 +1312,11 @@ class Model:
             # else:
             self.dbname = filename
 
+        # NullPool creates a fresh connection each time, which doesn't work for
+        # :memory: SQLite (each connection gets its own empty database).
+        # Use StaticPool for in-memory databases to reuse the same connection.
+        pool_class = StaticPool if self.dbname == ":memory:" else NullPool
+
         self._engine = create_engine(
             f"sqlite:///{self.dbname}",
             echo=debug,
@@ -1319,7 +1324,7 @@ class Model:
                 "detect_types": sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
                 "timeout": SQLITE_TIMEOUT_SECONDS,
             },
-            poolclass=NullPool,
+            poolclass=pool_class,
             native_datetime=True,
         )
 

{objutils-0.10.2 → objutils-0.10.4}/objutils/etek.py

@@ -67,9 +67,9 @@ class Reader(hexfile.Reader):
     VALID_CHARS = re.compile(r"^[a-zA-Z0-9_ %\n\r]*$")
 
     FORMAT_SPEC = (
-        (DATA, "%LL6CCAAAAADD"),
+        (DATA, "%LL6CCAAAAAADD"),
         (SYMBOL, "%LL3CCU"),
-        (EOF, "%LL8CCAAAAADD"),
+        (EOF, "%LL8CCAAAAAADD"),
     )
 
     def check_line(self, line: Any, format_type: int) -> None:

{objutils-0.10.2 → objutils-0.10.4}/objutils/hexfile.py

@@ -508,6 +508,7 @@ class Container:
     length: Optional[int] = None
     type: Optional[int] = None
     checksum: Optional[int] = None
+    addrChecksum: Optional[int] = None
     chunk: Optional[bytearray] = None
     junk: Optional[str] = None
     processing_instructions: list[Any] = field(default_factory=list)
@@ -790,7 +791,7 @@ class Reader(BaseType):
         self.stats = Statistics()
         self.valid = True
         self.formats: list[tuple[int, re.Pattern[str]]] = []
-        self.base_address = 0   # Base address for relative addressing (if applicable - mainly Intel HEX)
+        self.base_address = 0  # Base address for relative addressing (if applicable - mainly Intel HEX)
 
         # Parse FORMAT_SPEC into compiled patterns
         if isinstance(self.FORMAT_SPEC, str):
@@ -892,6 +893,7 @@ class Reader(BaseType):
                 length = self._parse_optional_int(dict_, "length")
                 record_type = self._parse_optional_int(dict_, "type")
                 checksum = self._parse_optional_int(dict_, "checksum")
+                addr_checksum = self._parse_optional_int(dict_, "addrChecksum")
                 junk = dict_.get("junk")
 
                 # Parse data chunk
@@ -912,6 +914,7 @@ class Reader(BaseType):
                     length=length,
                     type=record_type,
                     checksum=checksum,
+                    addrChecksum=addr_checksum,
                     chunk=chunk,
                     junk=junk,
                 )

{objutils-0.10.2 → objutils-0.10.4}/objutils/ihex.py

@@ -25,10 +25,9 @@ __copyright__ = """
   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 """
 
-import operator
 from collections.abc import Mapping, Sequence
 from functools import partial
-from typing import Any, Callable, Optional
+from typing import Any, Optional
 
 import objutils.checksums as checksums
 import objutils.hexfile as hexfile

{objutils-0.10.2 → objutils-0.10.4}/objutils/image.py

@@ -716,11 +716,20 @@ class Image:
         array: Any,
         dtype: str,
         byte_order: str = "MSB_LAST",
-        order: Optional[str] = None,
+        index_mode: str = "ROW_DIR",
         **kws: Any,
     ) -> None:
-        """Write a NumPy ndarray using ASAM datatype and ECU byte order semantics."""
-        self._call_address_function("write_asam_ndarray", addr, array, dtype, byte_order, order=order, **kws)
+        """Write a NumPy ndarray using ASAM datatype and ECU byte order semantics.
+
+        Args:
+            addr: Absolute memory address to write to.
+            array: NumPy ndarray (shape in numpy convention).
+            dtype: ASAM datatype name.
+            byte_order: ASAM byte order string.
+            index_mode: ``"ROW_DIR"`` (default) or ``"COLUMN_DIR"``.
+            **kws: Passed through to section method.
+        """
+        self._call_address_function("write_asam_ndarray", addr, array, dtype, byte_order, index_mode=index_mode, **kws)
 
     def read_ndarray(
         self,
@@ -755,12 +764,27 @@ class Image:
         length: int,
         dtype: str,
         shape: Optional[tuple[int, ...]] = None,
-        order: Optional[str] = None,
         byte_order: str = "MSB_LAST",
+        index_mode: str = "ROW_DIR",
         **kws: Any,
     ) -> Any:
-        """Read a NumPy ndarray using ASAM datatype and ECU byte order semantics."""
-        return self._call_address_function("read_asam_ndarray", addr, length, dtype, shape, order, byte_order, **kws)
+        """Read a NumPy ndarray using ASAM datatype and ECU byte order semantics.
+
+        Args:
+            addr: Start address to read from.
+            length: Number of **elements** (not bytes).
+            dtype: ASAM datatype name.
+            shape: Dimensions in **ASAM** order ``(X, Y, Z, ...)``.
+            byte_order: ASAM byte order string.
+            index_mode: ``"ROW_DIR"`` (default) or ``"COLUMN_DIR"``.
+            **kws: Passed through to section method.
+
+        Returns:
+            NumPy ndarray with shape in numpy convention.
+        """
+        return self._call_address_function(
+            "read_asam_ndarray", addr, length, dtype, shape=shape, byte_order=byte_order, index_mode=index_mode, **kws
+        )
 
     def read_string(self, addr: int, encoding: str = "latin1", length: int = -1, **kws: Any) -> str:
         """Read string from image.

{objutils-0.10.2 → objutils-0.10.4}/objutils/pecoff/__init__.py

@@ -739,12 +739,12 @@ class SectionAPI:
     def __init__(self, parent: PeParser):
         self.parent = parent
 
-        def fetch(self, name_pattern: str | None = None) -> list[model.Pe_Section]:
-            self.parent.create_db_on_demand()
-            if self.parent.db is None:
-                raise RuntimeError("PE database is not initialized")
-            with self.parent.db.session() as ses:
-                q = ses.query(model.Pe_Section)
+    def fetch(self, name_pattern: str | None = None) -> list[model.Pe_Section]:
+        self.parent.create_db_on_demand()
+        if self.parent.db is None:
+            raise RuntimeError("PE database is not initialized")
+        with self.parent.db.session() as ses:
+            q = ses.query(model.Pe_Section)
             if name_pattern:
                 q = q.filter(model.Pe_Section.name.like(f"%{name_pattern}%"))
             return q.order_by(model.Pe_Section.vaddr).all()
@@ -754,12 +754,12 @@ class SymbolAPI:
     def __init__(self, parent: PeParser):
         self.parent = parent
 
-        def fetch(self, name_pattern: str | None = None) -> list[model.Pe_Symbol]:
-            self.parent.create_db_on_demand()
-            if self.parent.db is None:
-                raise RuntimeError("PE database is not initialized")
-            with self.parent.db.session() as ses:
-                q = ses.query(model.Pe_Symbol)
+    def fetch(self, name_pattern: str | None = None) -> list[model.Pe_Symbol]:
+        self.parent.create_db_on_demand()
+        if self.parent.db is None:
+            raise RuntimeError("PE database is not initialized")
+        with self.parent.db.session() as ses:
+            q = ses.query(model.Pe_Symbol)
             if name_pattern:
                 q = q.filter(model.Pe_Symbol.name.like(f"%{name_pattern}%"))
             return q.order_by(model.Pe_Symbol.value).all()

{objutils-0.10.2 → objutils-0.10.4}/objutils/pecoff/model.py

@@ -114,7 +114,10 @@ def StdInteger(default=0, primary_key=False, unique=False, nullable=False, index
             count = StdInteger(default=0, index=True)
         ```
     """
-    return Column(Integer, primary_key=primary_key, unique=unique, nullable=nullable, index=index, default=default)
+    kw = dict(primary_key=primary_key, unique=unique, nullable=nullable, index=index)
+    if not primary_key:
+        kw["default"] = default
+    return Column(Integer, **kw)
 
 
 class MixInBase:

{objutils-0.10.2 → objutils-0.10.4}/objutils/scripts/oj_coff_extract.py

@@ -52,6 +52,14 @@ def main(argv: list[str] | None = None) -> int:
         dest="include",
         default=None,
     )
+    parser.add_argument(
+        "-r",
+        "--no-image-base",
+        help="Use relative virtual addresses (RVAs) instead of absolute addresses. Required for 64-bit PE files "
+        "whose image base pushes addresses beyond 32-bit hex format limits.",
+        dest="no_image_base",
+        action="store_true",
+    )
     parser.add_argument("-n", help="Number of data bytes per line", dest="row_length", default=16, type=int)
     args = parser.parse_args(argv)
 
@@ -60,17 +68,33 @@ def main(argv: list[str] | None = None) -> int:
     except (OSError, ValueError, RuntimeError) as e:
         print(f"\n'{args.pe_file}' is not valid PE/COFF file. Raised exception: '{repr(e)}'.")
         return 1
-    print("\nExtracting from...\n")
+
+    add_image_base = not args.no_image_base
+    image_base = pp.image_base()
+    if args.no_image_base:
+        print(f"\nUsing relative addresses (image base {image_base:#010x} subtracted).\n")
+    else:
+        print(f"\nUsing absolute addresses (image base: {image_base:#010x}).\n")
+
+    print("Extracting from...\n")
     print("Section                   Address    Length")
     print("-" * 45)
     img = pp.create_image(
         callback=callback,
         join=args.join,
+        add_image_base=add_image_base,
         exclude_pattern=args.exclude or "",
         include_pattern=args.include or "",
     )
     if img:
-        dump(args.file_type, args.output_file_name, img, row_length=args.row_length)
+        try:
+            dump(args.file_type, args.output_file_name, img, row_length=args.row_length)
+        except Exception as e:
+            if "address too large" in str(e).lower():
+                print(f"\nError: {e}")
+                print(f"Hint: Try using --no-image-base / -r to use relative addresses (subtract image base {image_base:#010x}).")
+                return 1
+            raise
         print(f"HEX image written to: '{args.output_file_name}' [{len(img)} total bytes]")
     return 0
 

{objutils-0.10.2 → objutils-0.10.4}/objutils/scripts/oj_coff_syms.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 
 import argparse
 
-from objutils.pecoff import PeParser
+from objutils.pecoff import PeParser, SymbolAPI
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -35,7 +35,7 @@ def main(argv: list[str] | None = None) -> int:
         return 1
 
     # Fetch via SymbolAPI to ensure DB is created/reused
-    syms = pp.SymbolAPI(pp).fetch(name_pattern=args.pattern)  # type: ignore[attr-defined]
+    syms = SymbolAPI(pp).fetch(name_pattern=args.pattern)
     # Fallback: if SymbolAPI attr is not present (static type), use direct list
     if not syms and pp.symbols:
         syms = [type("_S", (), s) for s in pp.symbols]  # quick adapter for printing

{objutils-0.10.2 → objutils-0.10.4}/objutils/section.py

@@ -419,6 +419,78 @@ def fortran_array_to_buffer(array: np.ndarray) -> bytearray:
         return result
 
 
+ASAM_INDEX_MODES = {"ROW_DIR", "COLUMN_DIR"}
+
+
+def _asam_column_dir_from_buffer(data: bytes, numpy_shape: tuple, dtype: Any) -> np.ndarray:
+    """Reconstruct array from ASAM COLUMN_DIR memory layout.
+
+    In COLUMN_DIR only the X and Y dimensions (last two in numpy convention)
+    are swapped compared to ROW_DIR.  Higher dimensions (Z, Z4, Z5) keep
+    C-order.  This is **not** true column-major (Fortran) order for dims > 2.
+
+    Args:
+        data: Raw byte buffer.
+        numpy_shape: Target shape in numpy convention (rows, cols, …).
+        dtype: NumPy dtype.
+
+    Returns:
+        NumPy array with the specified shape.
+    """
+    dt = np.dtype(dtype)
+    flat = np.frombuffer(data, dtype=dt)
+
+    if numpy_shape is None or len(numpy_shape) <= 1:
+        return flat.reshape(numpy_shape) if numpy_shape else flat
+
+    if len(numpy_shape) == 2:
+        return flat.reshape(numpy_shape, order="F")
+
+    # 3D+: only the last two dims (Y, X) are Fortran-ordered per slice.
+    higher_dims = numpy_shape[:-2]
+    yx_shape = numpy_shape[-2:]
+    num_slices = reduce(mul, higher_dims, 1)
+    slice_size = reduce(mul, yx_shape, 1)
+
+    sliced = flat.reshape(num_slices, slice_size)
+    out = np.empty((num_slices, *yx_shape), dtype=dt)
+    for idx in range(num_slices):
+        out[idx] = sliced[idx].reshape(yx_shape, order="F")
+    return out.reshape(numpy_shape)
+
+
+def _asam_column_dir_to_buffer(arr: np.ndarray) -> bytearray:
+    """Serialize array for ASAM COLUMN_DIR memory layout.
+
+    Only the X and Y dimensions (last two in numpy convention) are
+    transposed.  Higher dimensions keep C-order.
+
+    Args:
+        arr: NumPy array to serialize.
+
+    Returns:
+        Byte buffer with COLUMN_DIR memory layout.
+    """
+    if arr.ndim <= 1:
+        return bytearray(arr.tobytes())
+
+    if arr.ndim == 2:
+        return bytearray(arr.tobytes("F"))
+
+    shape = arr.shape
+    higher_dims = shape[:-2]
+    yx_shape = shape[-2:]
+    num_slices = reduce(mul, higher_dims, 1)
+    slice_bytes = reduce(mul, yx_shape, 1) * arr.dtype.itemsize
+    rs_arr = arr.reshape(num_slices, *yx_shape)
+    result = bytearray(arr.nbytes)
+    offset = 0
+    for idx in range(num_slices):
+        result[offset : offset + slice_bytes] = rs_arr[idx].tobytes("F")
+        offset += slice_bytes
+    return result
+
+
 @dataclass(repr=False, order=True)
 class Section:
     """Continuous block of bytes with start address and known length.
@@ -978,17 +1050,44 @@ class Section:
         array: np.ndarray,
         dtype: str,
         byte_order: str = "MSB_LAST",
-        order: str = None,
+        index_mode: str = "ROW_DIR",
         **kws,
     ) -> None:
+        """Write a NumPy ndarray using ASAM datatype and byte order semantics.
+
+        Unlike :meth:`write_ndarray`, this method uses ASAM conventions:
+
+        - ``index_mode`` controls the memory layout (``ROW_DIR`` or
+          ``COLUMN_DIR``) instead of the generic ``order`` parameter.
+        - Byte-order permutations (word-swap variants) are applied per
+          element.
+
+        Args:
+            addr: Absolute memory address to write to.
+            array: NumPy array to write (shape in **numpy** convention,
+                i.e. rows × columns).
+            dtype: ASAM datatype name (e.g. ``"UWORD"``, ``"ULONG"``).
+            byte_order: ASAM byte order string.
+            index_mode: ``"ROW_DIR"`` (default) – X increments fastest
+                (C-like row-major).  ``"COLUMN_DIR"`` – Y increments
+                fastest; only X and Y are swapped (not true column-major
+                for dims > 2).
+            **kws: Passed through to :meth:`write`.
+
+        Raises:
+            TypeError: If *array* is not an ndarray.
+            ValueError: If *index_mode* is unsupported.
+        """
         if not isinstance(array, np.ndarray):
             raise TypeError("array must be of type numpy.ndarray.")
+        if index_mode not in ASAM_INDEX_MODES:
+            raise ValueError(f"Unsupported index_mode {index_mode!r}; use ROW_DIR or COLUMN_DIR.")
 
         asam_byte_order = self._resolve_asam_byteorder(byte_order)
         internal_dtype = self._asam_numeric_dtype_to_internal(dtype, asam_byte_order)
         typed_array = np.asarray(array, dtype=self._numpy_dtype_from_internal(internal_dtype))
-        if order is not None and order == "F":
-            raw_data = fortran_array_to_buffer(array=typed_array)
+        if index_mode == "COLUMN_DIR":
+            raw_data = _asam_column_dir_to_buffer(typed_array)
         else:
             raw_data = typed_array.tobytes()
         permuted = self._permute_asam_buffer(raw_data, internal_dtype, asam_byte_order)
@@ -1015,20 +1114,63 @@ class Section:
         length: int,
         dtype: str,
         shape: tuple = None,
-        order: str = None,
         byte_order: str = "MSB_LAST",
+        index_mode: str = "ROW_DIR",
         **kws,
     ) -> np.ndarray:
+        """Read a NumPy ndarray using ASAM datatype and byte order semantics.
+
+        Unlike :meth:`read_ndarray`, this method uses ASAM conventions:
+
+        - ``length`` is the **element count** (not byte count).
+        - ``shape`` uses ASAM dimension order ``(X, Y, Z, Z4, Z5)``
+          which is **reversed** compared to numpy ``(…, Z, Y, X)``.
+        - ``index_mode`` controls the memory layout instead of the
+          generic ``order`` parameter.
+
+        Args:
+            addr: Absolute memory address to read from.
+            length: Number of **elements** to read.
+            dtype: ASAM datatype name (e.g. ``"UWORD"``, ``"ULONG"``).
+            shape: Array dimensions in **ASAM** convention ``(X, Y, …)``.
+                Converted internally to numpy convention ``(…, Y, X)``.
+            byte_order: ASAM byte order string.
+            index_mode: ``"ROW_DIR"`` (default) – X increments fastest
+                (C-like row-major).  ``"COLUMN_DIR"`` – Y increments
+                fastest; only X and Y are swapped (not true column-major
+                for dims > 2).
+            **kws: Passed through to :meth:`read`.
+
+        Returns:
+            NumPy array with shape in **numpy** convention.
+
+        Raises:
+            ValueError: If *index_mode* is unsupported.
+        """
+        if index_mode not in ASAM_INDEX_MODES:
+            raise ValueError(f"Unsupported index_mode {index_mode!r}; use ROW_DIR or COLUMN_DIR.")
+
         asam_byte_order = self._resolve_asam_byteorder(byte_order)
         internal_dtype = self._asam_numeric_dtype_to_internal(dtype, asam_byte_order)
         dt = self._numpy_dtype_from_internal(internal_dtype)
-        raw_data = self.read(addr, length, **kws)
+
+        # Compute byte count from element count.
+        type_name = internal_dtype.split("_")[0]
+        type_size = TYPE_SIZES[type_name]
+        byte_count = length * type_size
+
+        # Convert ASAM shape (X, Y, Z …) → numpy shape (… Z, Y, X).
+        numpy_shape = tuple(reversed(shape)) if shape else None
+
+        raw_data = self.read(addr, byte_count, **kws)
         permuted = self._permute_asam_buffer(raw_data, internal_dtype, asam_byte_order)
-        if order is not None and order == "F":
-            return fortran_array_from_buffer(arr=permuted, shape=shape, dtype=dt)
 
+        if index_mode == "COLUMN_DIR":
+            return _asam_column_dir_from_buffer(permuted, numpy_shape, dt)
+
+        # ROW_DIR: standard C-order.
         flat = np.frombuffer(permuted, dtype=dt)
-        return flat.reshape(shape) if shape else flat
+        return flat.reshape(numpy_shape) if numpy_shape else flat
 
     """
     def write_timestamp():

{objutils-0.10.2 → objutils-0.10.4}/objutils/shf.py

@@ -30,7 +30,7 @@ from hashlib import sha1
 
 from objutils.image import Image
 from objutils.logger import Logger
-from objutils.section import Section
+from objutils.section import Section, join_sections
 from objutils.utils import create_string_buffer
 
 SHF_DTD = """<!--

{objutils-0.10.2 → objutils-0.10.4}/objutils/srec.py

@@ -151,14 +151,14 @@ class Reader(hexfile.Reader):
             )
         else:
             raise TypeError(f"Invalid format type {format_type}.")
-        if hasattr(line, "chunk"):
+        if hasattr(line, "chunk") and line.chunk is not None:
             checksum = (~(sum([line.length, checksum_of_address]) + sum(line.chunk))) & 0xFF
         else:
             checksum = (~(sum([line.length, checksum_of_address]))) & 0xFF
         if line.checksum != checksum:
             raise hexfile.InvalidRecordChecksumError()
         line.length -= BIAS[format_type]  # calculate actual data length.
-        if hasattr(line, "chunk") and line.length and (line.length != len(line.chunk)):
+        if hasattr(line, "chunk") and line.chunk is not None and line.length and (line.length != len(line.chunk)):
             raise hexfile.InvalidRecordLengthError("Byte count doesn't match length of actual data.")
 
     def is_data_line(self, line: Any, format_type: int) -> bool: