objutils 0.10.19__tar.gz → 0.10.20__tar.gz

{objutils-0.10.19 → objutils-0.10.20}/.idea/objutils@1.iml

@@ -8,4 +8,4 @@
     <orderEntry type="jdk" jdkName="Poetry (objutils) (2)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
-</module>
+</module>

{objutils-0.10.19 → objutils-0.10.20}/.pre-commit-config.yaml

@@ -35,21 +35,21 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 26.5.1
+    rev: 26.10.1
     hooks:
       - id: black
         language_version: python3.13
         args: ["--line-length=132"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.16.6
+    rev: v0.17.0
     hooks:
       - id: ruff
         args: ["--fix", "--exit-non-zero-on-fix"]
 
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.21.2
+    rev: v3.22.0
     hooks:
       - id: pyupgrade
         args: ["--py39-plus"]

{objutils-0.10.19 → objutils-0.10.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: objutils
-Version: 0.10.19
+Version: 0.10.20
 Summary: Objectfile library for Python
 Keywords: hex files,intel hex,s19,srec,srecords,object files,map files,embedded,microcontroller,ECU,shf,rfc4194
 Author-Email: Christoph Schueler <cpu12.gems@googlemail.com>
@@ -24,6 +24,7 @@ Requires-Dist: sqlalchemy>=2.0.29
 Requires-Dist: rich>=14.2
 Requires-Dist: numpy>=2.2.5
 Requires-Dist: ufilename<0.2.0,>=0.1.1
+Requires-Dist: coverage<8.0.0,>=7.16.2
 Provides-Extra: dev
 Requires-Dist: bumpver>=2023.1129; extra == "dev"
 Requires-Dist: pre-commit>=3.7.0; extra == "dev"

{objutils-0.10.19 → objutils-0.10.20}/objutils/__init__.py

@@ -13,7 +13,7 @@ Registers CODECS and implements an interface to them.
 The first parameter is always the codec name.
 """
 
-__version__ = "0.10.19"
+__version__ = "0.10.20"
 
 __all__ = [
     "Image",

{objutils-0.10.19 → objutils-0.10.20}/objutils/dwarf/__init__.py

@@ -186,7 +186,7 @@ import io
 import json
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Any, Optional
+from typing import Any
 
 from construct import (
     Array,
@@ -198,8 +198,6 @@ from construct import (
     If,
     IfThenElse,
     Int8ul,
-    Int32ub,
-    Int32ul,
     Padding,
     Pass,
     Struct,
@@ -433,6 +431,10 @@ class Abbrevations:
         "stop" / Tell,
     )
 
+    @staticmethod
+    def _tag_name_fallback(tag: int) -> str:
+        return f"User TAG value: 0x{tag:04x}"
+
     def __init__(self, section):
         """Initialize abbreviation parser.
 
@@ -442,6 +444,67 @@ class Abbrevations:
         self.image = section.image
         self.length = len(section.image)
         self.abbrevations = {}
+        self._parsed_offsets: set[int] = set()
+        self._tag_name_cache: dict[int, str] = {}
+        self._encoding_cache: dict[int, Any] = {}
+        self._form_cache: dict[int, Any] = {}
+
+    def _tag_name(self, tag: int) -> str:
+        cached = self._tag_name_cache.get(tag)
+        if cached is not None:
+            return cached
+        try:
+            value = constants.Tag(tag).name
+        except (AttributeError, TypeError, ValueError):
+            value = self._tag_name_fallback(tag)
+        self._tag_name_cache[tag] = value
+        return value
+
+    def _encoding(self, attr_value: int):
+        cached = self._encoding_cache.get(attr_value)
+        if cached is not None:
+            return cached
+        try:
+            value = constants.AttributeEncoding(attr_value)
+        except (TypeError, ValueError):
+            value = constants.FakeEncoding(attr_value)
+        self._encoding_cache[attr_value] = value
+        return value
+
+    def _form(self, form_value: int):
+        cached = self._form_cache.get(form_value)
+        if cached is not None:
+            return cached
+        value = constants.AttributeForm(form_value)
+        self._form_cache[form_value] = value
+        return value
+
+    def _build_abbrev(self, tag: int, children: bool, attrs: list[Any]) -> Abbrevation:
+        abbrev = Abbrevation(self._tag_name(tag), bool(children), [])
+        append_attr = abbrev.attrs.append
+        implicit_const = constants.AttributeForm.DW_FORM_implicit_const
+        for attr_value, form_value, special_value in attrs:
+            encoding = self._encoding(attr_value)
+            form = self._form(form_value)
+            if form == implicit_const:
+                append_attr(Attribute(encoding, form, special_value))
+            else:
+                append_attr(Attribute(encoding, form))
+        return abbrev
+
+    def _read_uleb(self, image: bytes, offset: int) -> tuple[int, int]:
+        """Decode one ULEB128 value from image[offset:] and return (value, new_offset)."""
+        result = 0
+        shift = 0
+        length = self.length
+        while offset < length:
+            bval = image[offset]
+            offset += 1
+            result |= (bval & 0x7F) << shift
+            if (bval & 0x80) == 0:
+                return result, offset
+            shift += 7
+        raise ValueError("Truncated ULEB128 in .debug_abbrev section.")
 
     def get(self, abbr_offset, item):
         """Retrieve abbreviation schema by code.
@@ -455,96 +518,53 @@ class Abbrevations:
         """
         if item == 0:
             return None
-        if (abbr_offset, item) in self.abbrevations:
-            return self.abbrevations.get(
-                (
-                    abbr_offset,
-                    item,
-                )
-            )
-        else:
-            return self._fetch(abbr_offset, item)
+        if abbr_offset not in self._parsed_offsets:
+            self._fetch(abbr_offset)
+        return self.abbrevations.get((abbr_offset, item))
 
-    def _fetch(self, abbr_offset, item):
+    def _fetch(self, abbr_offset):
         """Parse abbreviation from raw bytes and cache it.
 
-        Parses abbreviation table starting at abbr_offset, caching all
-        entries encountered until the requested item is found.
+        Parses a complete abbreviation table starting at abbr_offset and
+        caches all entries for O(1) subsequent lookups.
 
         Args:
             abbr_offset: Start offset in abbreviation section
-            item: Target abbreviation code
-
-        Returns:
-            Abbrevation instance for the requested code
         """
+        if abbr_offset in self._parsed_offsets:
+            return
+
+        image = self.image
         offset = abbr_offset
-        while True:
-            abbrv_header = self.AbbrevationHeader.parse(self.image[offset:])
-            # print("ABBREV-CODE", hex(offset), abbrv_header)
-            code = abbrv_header.code
-            offset += abbrv_header.stop - abbrv_header.start
-            if code == 0:
-                continue
-            key = (
-                abbr_offset,
-                code,
-            )
-            if offset >= self.length:
-                break
-            abbrv_body = self.AbbrevationBody.parse(self.image[offset:])
-            tag = abbrv_body.tag
-            children = abbrv_body.children
-            offset += abbrv_body.stop - abbrv_body.start
+        try:
+            while offset < self.length:
+                code, offset = self._read_uleb(image, offset)
+                if code == 0:
+                    break
+                tag, offset = self._read_uleb(image, offset)
+                if offset >= self.length:
+                    break
+                children = image[offset]
+                offset += 1
 
-            try:
-                self.abbrevations[key] = Abbrevation(
-                    constants.Tag(tag).name,
-                    (children == 1),
-                    [],
-                )
-            except TypeError:
-                self.abbrevations[key] = Abbrevation(
-                    f"User TAG value: 0x{tag:04x}",
-                    (children == 1),
-                    [],
-                )
+                attrs: list[tuple[int, int, Any]] = []
+                while True:
+                    attr_value, offset = self._read_uleb(image, offset)
+                    form_value, offset = self._read_uleb(image, offset)
+                    if attr_value == 0 or form_value == 0:
+                        break
+                    if form_value == constants.AttributeForm.DW_FORM_implicit_const:
+                        special_value, offset = self._read_uleb(image, offset)
+                        attrs.append((attr_value, form_value, special_value))
+                    else:
+                        attrs.append((attr_value, form_value, None))
 
-            while True:
-                attr = self.AttributeStruct.parse(self.image[offset:])
-                if not attr.next:
-                    # key_offset = offset
-                    break
-                if attr.formValue == constants.AttributeForm.DW_FORM_implicit_const:
-                    offset += attr.stop - attr.start
-                    special_value = self.SpecialValueStruct.parse(self.image[offset:])
-                    offset += special_value.stop - special_value.start
-                    self.abbrevations[key].attrs.append(
-                        Attribute(
-                            constants.AttributeEncoding(attr.attrValue),
-                            constants.AttributeForm(attr.formValue),
-                            special_value.value,
-                        )
-                    )
-                    continue
-                if attr.attrValue != 0 and attr.formValue != 0:
-                    try:
-                        self.abbrevations[key].attrs.append(
-                            Attribute(
-                                constants.AttributeEncoding(attr.attrValue),
-                                constants.AttributeForm(attr.formValue),
-                            )
-                        )
-                    except TypeError:
-                        self.abbrevations[key].attrs.append(
-                            Attribute(
-                                constants.FakeEncoding(attr.attrValue),
-                                constants.AttributeForm(attr.formValue),
-                            )
-                        )
-                offset += attr.stop - attr.start
-            if code == item:
-                return self.abbrevations[key]
+                self.abbrevations[(abbr_offset, code)] = self._build_abbrev(tag, (children == 1), attrs)
+        except ValueError:
+            # Graceful fallback: keep already cached entries and mark offset parsed.
+            pass
+
+        self._parsed_offsets.add(abbr_offset)
 
 
 class DwarfProcessor:
@@ -619,9 +639,6 @@ class DwarfProcessor:
         Earlier or later DWARF versions may not parse correctly.
     """
 
-    DATATYPES32 = {
-        "Addr": (Int32ul, Int32ub),  # 4 - Unsigned program address
-    }
 
     UTF8String = CString(encoding="utf8")
 
@@ -672,6 +689,7 @@ class DwarfProcessor:
             strings=self.strings,
             line_strings=self.line_strings,
         )
+        self.elf_address_size = address_size
         self.readers = factory.readers
         self.stack_machine = factory.stack_machine
 
@@ -1030,135 +1048,180 @@ class DwarfProcessor:
             "stop" / Tell,
             "size" / Computed(this.stop - this.start),
         )
+        from contextlib import nullcontext
+
+        from sqlalchemy import func
+
         offset = 0
         result = []
         die_map = {}
         idx = 0
+        pending_die_rows: list[dict[str, Any]] = []
+        pending_attr_rows: list[dict[str, Any]] = []
         session = self.db_session
         try:
             no_autoflush = session.no_autoflush
         except AttributeError:
-            from contextlib import nullcontext
-
             no_autoflush = nullcontext()
 
-        with no_autoflush:
-            while True:
-                section_pos = image.tell()
-                if section_pos >= section_length - 1:
-                    break
-                root_element = DebugInformationEntry("root")
-                die_stack = [root_element]
-                db_die_stack = []
-                pending_dies = []
-                pending_attrs = []
-
-                dbgInfo = DbgInfo.parse_stream(image)  # CU
-                cu_length = dbgInfo.header.unit_length
-                version = dbgInfo.header.version
-                if version < 5:
-                    debug_abbrev_offset = dbgInfo.body.debug_abbrev_offset
-                    address_size = dbgInfo.body.address_size
-                else:
-                    debug_abbrev_offset = dbgInfo.body.debug_abbrev_offset
-                    address_size = dbgInfo.body.address_size
-                idx += 1
-                if self.verbose:
-                    print(
-                        f"   Compilation Unit #{idx:05d}  offset: 0x{dbgInfo.start:08x}  length: 0x{cu_length:08x}  abbrev-offset: 0x{debug_abbrev_offset:08x}  pointer-size:  {address_size}  version: {version}"
-                    )
-                level = 0
-                pos = 0
-                offset += dbgInfo.stop - dbgInfo.start
-                form_readers = self.get_form_readers(address_size)
-                if pos >= cu_length:
-                    break
+        max_rid = session.query(func.max(model.DebugInformationEntry.rid)).scalar() or 0
+        next_die_rid = int(max_rid) + 1
+
+        # Chunk sizes tuned for SQLite bulk inserts to avoid oversized execute payloads.
+        DIE_BATCH_SIZE = 10000
+        ATTR_BATCH_SIZE = 20000
+
+        def _iter_chunks(rows: list[dict[str, Any]], size: int):
+            for start_idx in range(0, len(rows), size):
+                yield rows[start_idx : start_idx + size]
+
+        def _coerce_tag_value(tag: Any) -> Any:
+            try:
+                return int(constants.Tag[str(tag)])
+            except (KeyError, TypeError, ValueError):
+                try:
+                    return int(tag)
+                except (TypeError, ValueError):
+                    return tag
+
+        SQLITE_INT64_MIN = -(2**63)
+        SQLITE_INT64_MAX = 2**63 - 1
+
+        def _normalize_raw_value(value: Any) -> Any:
+            """Keep DB writes SQLite-safe while preserving semantics for consumers."""
+            if value is None:
+                return None
+            if isinstance(value, int) and not (SQLITE_INT64_MIN <= value <= SQLITE_INT64_MAX):
+                return str(value)
+            return value
+
+        try:
+            with no_autoflush:
                 while True:
-                    die_start = None
-                    start = image.tell()
-                    if start >= dbgInfo.start + cu_length + 4:
+                    section_pos = image.tell()
+                    if section_pos >= section_length - 1:
                         break
-                    if start >= section_length - 1:
-                        break
-                    attr = Attribute.parse_stream(image)
-                    abbr = self.abbrevations.get(debug_abbrev_offset, attr.attr)
-                    if not abbr:
-                        # End of a sibling list: pop one level on both in-memory and DB stacks
-                        if level > 0:
-                            level -= 1
-                        # Pop in-memory DIE stack if we have a nested context (keep root at index 0)
-                        if len(die_stack) > 1:
-                            die_stack.pop()
-                        # Pop DB parent stack so subsequent siblings attach to the correct parent
-                        if db_die_stack:
-                            db_die_stack.pop()
+                    root_element = DebugInformationEntry("root")
+                    die_stack = [root_element]
+                    db_parent_stack: list[int] = []
+
+                    dbgInfo = DbgInfo.parse_stream(image)  # CU
+                    cu_length = dbgInfo.header.unit_length
+                    version = dbgInfo.header.version
+                    if version < 5:
+                        debug_abbrev_offset = dbgInfo.body.debug_abbrev_offset
+                        address_size = dbgInfo.body.address_size
                     else:
-                        die = DebugInformationEntry(abbr.tag)
-                        db_die = model.DebugInformationEntry(tag=abbr.tag)
-                        # Cache-friendly: store DIE offset for quick lookups by external tools
-                        db_die.offset = start
-                        # Store CU start for reference resolution later
-                        db_die.cu_start = dbgInfo.start
-                        # Set parent on DB DIE if we have one on stack
-                        if db_die_stack:
-                            db_die.parent = db_die_stack[-1]
-                        pending_dies.append(db_die)
-                        if attr.attr != 0:
-                            die_stack[-1].children.append(die)
-                        die_start = start
-                        if abbr.children:
-                            die_stack.append(die)
-                            # Push this db_die as the current parent for subsequent entries until a null DIE is encountered
-                            db_die_stack.append(db_die)
-                            level += 1
-                        for enc, form, special_value in abbr.attrs:
-                            reader = form_readers.get(form)
-                            start = image.tell()
-                            if reader is None:
-                                if form == constants.AttributeForm.DW_FORM_implicit_const:
-                                    value = special_value
-                                    # In-memory representation (keep minimal to preserve structure)
-                                    die.attributes.append((enc.name, DIEAttribute(value, value)))
-                                    pending_attrs.append((db_die, enc, form, value))
-                                    offset += attr.stop - attr.start
-                                    pos = image.tell()
-                                    if pos >= dbgInfo.start + cu_length + 4:
-                                        break
-                                    continue
-                                else:
-                                    # Unknown reader; skip gracefully
-                                    continue
-                            value = reader.parse_stream(image)
-                            # In-memory representation: do not compute heavy display, keep simple
-                            die.attributes.append((enc.name, DIEAttribute(value, value)))
-                            # DB attribute: store for lazy display
-                            pending_attrs.append((db_die, enc, form, value))
-                        offset += attr.stop - attr.start
-                        pos = image.tell()
-                        if pos >= dbgInfo.start + cu_length + 4:
+                        if version > 5: # Handle messed up version numbers in some binaries (e.g. 39424)
+                            address_size = self.elf_address_size
+                        else:
+                            address_size = dbgInfo.body.address_size
+                        debug_abbrev_offset = dbgInfo.body.debug_abbrev_offset
+                    idx += 1
+                    if self.verbose:
+                        print(
+                            f"   Compilation Unit #{idx:05d}  offset: 0x{dbgInfo.start:08x}  length: 0x{cu_length:08x}  abbrev-offset: 0x{debug_abbrev_offset:08x}"
+                        )
+                    level = 0
+                    pos = 0
+                    offset += dbgInfo.stop - dbgInfo.start
+                    form_readers = self.get_form_readers(address_size)
+                    if pos >= cu_length:
+                        break
+                    while True:
+                        die_start = None
+                        start = image.tell()
+                        if start >= dbgInfo.start + cu_length + 4:
                             break
-                    if die_start is not None:
-                        die_map[die_start] = die
-
-                if pending_dies:
-                    session.add_all(pending_dies)
-                    session.flush()
-                    if pending_attrs:
-                        attr_rows = []
-                        for db_die, enc, form, value in pending_attrs:
-                            attr_rows.append(
+                        if start >= section_length - 1:
+                            break
+                        attr = Attribute.parse_stream(image)
+                        abbr = self.abbrevations.get(debug_abbrev_offset, attr.attr)
+                        if not abbr:
+                            if level > 0:
+                                level -= 1
+                            if len(die_stack) > 1:
+                                die_stack.pop()
+                            if db_parent_stack:
+                                db_parent_stack.pop()
+                        else:
+                            die = DebugInformationEntry(abbr.tag)
+                            die_rid = next_die_rid
+                            next_die_rid += 1
+                            parent_rid = db_parent_stack[-1] if db_parent_stack else None
+
+                            pending_die_rows.append(
                                 {
-                                    "name": int(enc),
-                                    "form": int(form) if form is not None else None,
-                                    "raw_value": value,
-                                    "entry_id": db_die.rid,
+                                    "rid": die_rid,
+                                    "tag": _coerce_tag_value(abbr.tag),
+                                    "offset": start,
+                                    "cu_start": dbgInfo.start,
+                                    "parent_id": parent_rid,
                                 }
                             )
-                        if attr_rows:
-                            session.execute(model.DIEAttribute.__table__.insert(), attr_rows)
-                    session.flush()
-                result.append(root_element)
-                session.commit()
+
+                            if attr.attr != 0:
+                                die_stack[-1].children.append(die)
+                            die_start = start
+                            if abbr.children:
+                                die_stack.append(die)
+                                db_parent_stack.append(die_rid)
+                                level += 1
+                            for enc, form, special_value in abbr.attrs:
+                                reader = form_readers.get(form)
+                                start = image.tell()
+                                if reader is None:
+                                    if form == constants.AttributeForm.DW_FORM_implicit_const:
+                                        value = special_value
+                                        normalized_value = _normalize_raw_value(value)
+                                        die.attributes.append((enc.name, DIEAttribute(normalized_value, normalized_value)))
+                                        pending_attr_rows.append(
+                                            {
+                                                "name": int(enc),
+                                                "form": int(form),
+                                                "raw_value": normalized_value,
+                                                "entry_id": die_rid,
+                                            }
+                                        )
+                                        offset += attr.stop - attr.start
+                                        pos = image.tell()
+                                        if pos >= dbgInfo.start + cu_length + 4:
+                                            break
+                                        continue
+                                    else:
+                                        continue
+                                value = reader.parse_stream(image)
+                                normalized_value = _normalize_raw_value(value)
+                                die.attributes.append((enc.name, DIEAttribute(normalized_value, normalized_value)))
+                                pending_attr_rows.append(
+                                    {
+                                        "name": int(enc),
+                                        "form": int(form) if form is not None else None,
+                                        "raw_value": normalized_value,
+                                        "entry_id": die_rid,
+                                    }
+                                )
+                            offset += attr.stop - attr.start
+                            pos = image.tell()
+                            if pos >= dbgInfo.start + cu_length + 4:
+                                break
+                        if die_start is not None:
+                            die_map[die_start] = die
+
+                    result.append(root_element)
+
+                if pending_die_rows:
+                    die_insert = model.DebugInformationEntry.__table__.insert()
+                    for chunk in _iter_chunks(pending_die_rows, DIE_BATCH_SIZE):
+                        session.execute(die_insert, chunk)
+                if pending_attr_rows:
+                    attr_insert = model.DIEAttribute.__table__.insert()
+                    for chunk in _iter_chunks(pending_attr_rows, ATTR_BATCH_SIZE):
+                        session.execute(attr_insert, chunk)
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
         return DebugInformation(die_map, [d.children[0] for d in result])
 
     def pubnames(self):