dissect.cstruct 4.4.dev3__tar.gz → 4.5.dev1__tar.gz

{dissect_cstruct-4.4.dev3 → dissect_cstruct-4.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dissect.cstruct
-Version: 4.4.dev3
+Version: 4.5.dev1
 Summary: A Dissect module implementing a parser for C-like structures: structure parsing in Python made easy
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Apache License 2.0
@@ -22,6 +22,9 @@ Requires-Python: ~=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: COPYRIGHT
+Provides-Extra: dev
+Requires-Dist: typing_extensions; extra == "dev"
+Dynamic: license-file
 
 # dissect.cstruct
 

{dissect_cstruct-4.4.dev3 → dissect_cstruct-4.5.dev1}/dissect/cstruct/bitbuffer.py

@@ -13,12 +13,15 @@ class BitBuffer:
         self.stream = stream
         self.endian = endian
 
-        self._type = None
+        self._type: type[BaseType] | None = None
         self._buffer = 0
         self._remaining = 0
 
-    def read(self, field_type: BaseType, bits: int) -> int:
+    def read(self, field_type: type[BaseType], bits: int) -> int:
         if self._remaining == 0 or self._type != field_type:
+            if field_type.size is None:
+                raise ValueError("Reading variable-length fields is unsupported")
+
             self._type = field_type
             self._remaining = field_type.size * 8
             self._buffer = field_type._read(self.stream)
@@ -43,13 +46,20 @@ class BitBuffer:
 
         return v
 
-    def write(self, field_type: BaseType, data: int, bits: int) -> None:
+    def write(self, field_type: type[BaseType], data: int, bits: int) -> None:
         if self._remaining == 0 or self._type != field_type:
             if self._type:
                 self.flush()
+
+            if field_type.size is None:
+                raise ValueError("Writing variable-length fields is unsupported")
+
             self._remaining = field_type.size * 8
             self._type = field_type
 
+        if self._type is None or self._type.size is None:
+            raise ValueError("Invalid state")
+
         if self.endian == "<":
             self._buffer |= data << (self._type.size * 8 - self._remaining)
         else:
@@ -60,7 +70,8 @@ class BitBuffer:
             self.flush()
 
     def flush(self) -> None:
-        self._type._write(self.stream, self._buffer)
+        if self._type is not None:
+            self._type._write(self.stream, self._buffer)
         self._type = None
         self._remaining = 0
         self._buffer = 0

{dissect_cstruct-4.4.dev3 → dissect_cstruct-4.5.dev1}/dissect/cstruct/compiler.py

@@ -11,12 +11,11 @@ from typing import TYPE_CHECKING
 from dissect.cstruct.bitbuffer import BitBuffer
 from dissect.cstruct.types import (
     Array,
-    ArrayMetaType,
+    BaseType,
     Char,
     CharArray,
     Flag,
     Int,
-    MetaType,
     Packed,
     Pointer,
     Structure,
@@ -25,6 +24,7 @@ from dissect.cstruct.types import (
     Wchar,
     WcharArray,
 )
+from dissect.cstruct.types.base import BaseArray
 from dissect.cstruct.types.enum import EnumMetaType
 from dissect.cstruct.types.packed import _struct
 
@@ -159,7 +159,7 @@ class _ReadSourceGenerator:
                 yield f"stream.seek(-stream.tell() & ({field.alignment} - 1), {io.SEEK_CUR})"
 
         for field in self.fields:
-            field_type = self.cs.resolve(field.type)
+            field_type = field.type
 
             if isinstance(field_type, EnumMetaType):
                 field_type = field_type.type
@@ -187,7 +187,7 @@ class _ReadSourceGenerator:
 
             # Array of structures and multi-dimensional arrays
             elif issubclass(field_type, (Array, CharArray, WcharArray)) and (
-                issubclass(field_type.type, Structure) or isinstance(field_type.type, ArrayMetaType) or is_dynamic
+                issubclass(field_type.type, Structure) or issubclass(field_type.type, BaseArray) or is_dynamic
             ):
                 yield from flush()
                 yield from align_to_field(field)
@@ -195,6 +195,9 @@ class _ReadSourceGenerator:
 
             # Bit fields
             elif field.bits:
+                if size is None:
+                    raise TypeError(f"Unsupported type for bit field: {field_type}")
+
                 if not prev_was_bits:
                     prev_bits_type = field_type
                     prev_was_bits = True
@@ -223,8 +226,8 @@ class _ReadSourceGenerator:
     def _generate_structure(self, field: Field) -> Iterator[str]:
         template = f"""
         _s = stream.tell()
-        r["{field.name}"] = {self._map_field(field)}._read(stream, context=r)
-        s["{field.name}"] = stream.tell() - _s
+        r["{field._name}"] = {self._map_field(field)}._read(stream, context=r)
+        s["{field._name}"] = stream.tell() - _s
         """
 
         yield dedent(template)
@@ -232,8 +235,8 @@ class _ReadSourceGenerator:
     def _generate_array(self, field: Field) -> Iterator[str]:
         template = f"""
         _s = stream.tell()
-        r["{field.name}"] = {self._map_field(field)}._read(stream, context=r)
-        s["{field.name}"] = stream.tell() - _s
+        r["{field._name}"] = {self._map_field(field)}._read(stream, context=r)
+        s["{field._name}"] = stream.tell() - _s
         """
 
         yield dedent(template)
@@ -252,7 +255,7 @@ class _ReadSourceGenerator:
 
         template = f"""
         _t = {lookup}
-        r["{field.name}"] = type.__call__(_t, bit_reader.read({read_type}, {field.bits}))
+        r["{field._name}"] = type.__call__(_t, bit_reader.read({read_type}, {field.bits}))
         """
 
         yield dedent(template)
@@ -269,7 +272,7 @@ class _ReadSourceGenerator:
                 size += count
                 continue
 
-            field_type = self.cs.resolve(field.type)
+            field_type = field.type
             read_type = _get_read_type(self.cs, field_type)
 
             if issubclass(field_type, (Array, CharArray, WcharArray)):
@@ -320,8 +323,8 @@ class _ReadSourceGenerator:
             else:
                 parser = parser_template.format(type=self._map_field(field), getter=getter)
 
-            reads.append(f'r["{field.name}"] = {parser}')
-            reads.append(f's["{field.name}"] = {field_type.size}')
+            reads.append(f'r["{field._name}"] = {parser}')
+            reads.append(f's["{field._name}"] = {field_type.size}')
             reads.append("")  # Generates a newline in the resulting code
 
             size += field_type.size
@@ -411,9 +414,7 @@ def _optimize_struct_fmt(info: Iterator[tuple[Field, int, str]]) -> str:
     return "".join(f"{count if count > 1 else ''}{char}" for count, char in chars)
 
 
-def _get_read_type(cs: cstruct, type_: MetaType | str) -> MetaType:
-    type_ = cs.resolve(type_)
-
+def _get_read_type(cs: cstruct, type_: type[BaseType]) -> type[BaseType]:
     if issubclass(type_, (Enum, Flag)):
         type_ = type_.type
 

{dissect_cstruct-4.4.dev3 → dissect_cstruct-4.5.dev1}/dissect/cstruct/cstruct.py

@@ -5,21 +5,21 @@ import struct
 import sys
 import types
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, BinaryIO
+from typing import TYPE_CHECKING, Any, BinaryIO, TypeVar, cast
 
 from dissect.cstruct.exceptions import ResolveError
 from dissect.cstruct.expression import Expression
 from dissect.cstruct.parser import CStyleParser, TokenParser
 from dissect.cstruct.types import (
     LEB128,
-    ArrayMetaType,
+    Array,
+    BaseArray,
     BaseType,
     Char,
     Enum,
     Field,
     Flag,
     Int,
-    MetaType,
     Packed,
     Pointer,
     Structure,
@@ -29,7 +29,12 @@ from dissect.cstruct.types import (
 )
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Iterable
+
+    from typing_extensions import TypeAlias
+
+
+T = TypeVar("T", bound=BaseType)
 
 
 class cstruct:
@@ -37,7 +42,7 @@ class cstruct:
 
     Args:
         endian: The endianness to use when parsing.
-        pointer: The pointer type to use for Pointers.
+        pointer: The pointer type to use for pointers.
     """
 
     DEF_CSTYLE = 1
@@ -81,16 +86,16 @@ class cstruct:
             "signed char": "int8",
             "unsigned char": "char",
             "short": "int16",
-            "signed short": "short",
+            "signed short": "int16",
             "unsigned short": "uint16",
             "int": "int32",
-            "signed int": "int",
+            "signed int": "int32",
             "unsigned int": "uint32",
             "long": "int32",
-            "signed long": "long",
+            "signed long": "int32",
             "unsigned long": "uint32",
             "long long": "int64",
-            "signed long long": "long long",
+            "signed long long": "int64",
             "unsigned long long": "uint64",
 
             # Windows types
@@ -113,14 +118,14 @@ class cstruct:
             "ULONG64": "uint64",
             "ULONGLONG": "uint64",
 
-            "INT": "int",
+            "INT": "int32",
             "INT8": "int8",
             "INT16": "int16",
             "INT32": "int32",
             "INT64": "int64",
             "INT128": "int128",
 
-            "UINT": "uint",
+            "UINT": "uint32",
             "UINT8": "uint8",
             "UINT16": "uint16",
             "UINT32": "uint32",
@@ -172,14 +177,14 @@ class cstruct:
             "__u32": "uint32",
             "__u64": "uint64",
             "uchar": "uint8",
-            "ushort": "unsigned short",
-            "uint": "unsigned int",
-            "ulong": "unsigned long",
+            "ushort": "uint16",
+            "uint": "uint32",
+            "ulong": "uint32",
         }
         # fmt: on
 
         pointer = pointer or ("uint64" if sys.maxsize > 2**32 else "uint32")
-        self.pointer = self.resolve(pointer)
+        self.pointer: type[BaseType] = self.resolve(pointer)
         self._anonymous_count = 0
 
     def __getattr__(self, attr: str) -> Any:
@@ -200,7 +205,7 @@ class cstruct:
         self._anonymous_count += 1
         return name
 
-    def add_type(self, name: str, type_: MetaType | str, replace: bool = False) -> None:
+    def add_type(self, name: str, type_: type[BaseType] | str, replace: bool = False) -> None:
         """Add a type or type reference.
 
         Only use this method when creating type aliases or adding already bound types.
@@ -220,7 +225,7 @@ class cstruct:
     addtype = add_type
 
     def add_custom_type(
-        self, name: str, type_: MetaType, size: int | None = None, alignment: int | None = None, **kwargs
+        self, name: str, type_: type[BaseType], size: int | None = None, alignment: int | None = None, **kwargs
     ) -> None:
         """Add a custom type.
 
@@ -265,7 +270,7 @@ class cstruct:
     def loadfile(self, path: str, deftype: int | None = None, **kwargs) -> None:
         """Load structure definitions from a file.
 
-        The given path will be read and parsed using the .load() function.
+        The given path will be read and parsed using the :meth:`~cstruct.load` function.
 
         Args:
             path: The path to load definitions from.
@@ -287,7 +292,7 @@ class cstruct:
         """
         return self.resolve(name).read(stream)
 
-    def resolve(self, name: str) -> MetaType:
+    def resolve(self, name: type[BaseType] | str) -> type[BaseType]:
         """Resolve a type name to get the actual type object.
 
         Types can be referenced using different names. When we want
@@ -320,7 +325,7 @@ class cstruct:
     def _make_type(
         self,
         name: str,
-        bases: Iterator[object],
+        bases: Iterable[object],
         size: int | None,
         *,
         alignment: int | None = None,
@@ -341,10 +346,18 @@ class cstruct:
         )
         return types.new_class(name, bases, {}, lambda ns: ns.update(attrs))
 
-    def _make_array(self, type_: MetaType, num_entries: int | Expression | None) -> ArrayMetaType:
-        null_terminated = num_entries is None
-        dynamic = isinstance(num_entries, Expression) or type_.dynamic
-        size = None if (null_terminated or dynamic) else (num_entries * type_.size)
+    def _make_array(self, type_: T, num_entries: int | Expression | None) -> type[Array[T]]:
+        null_terminated = False
+        if num_entries is None:
+            null_terminated = True
+            size = None
+        elif isinstance(num_entries, Expression) or type_.dynamic:
+            size = None
+        else:
+            if type_.size is None:
+                raise ValueError(f"Cannot create array of dynamic type: {type_.__name__}")
+            size = num_entries * type_.size
+
         name = f"{type_.__name__}[]" if null_terminated else f"{type_.__name__}[{num_entries}]"
 
         bases = (type_.ArrayType,)
@@ -355,27 +368,30 @@ class cstruct:
             "null_terminated": null_terminated,
         }
 
-        return self._make_type(name, bases, size, alignment=type_.alignment, attrs=attrs)
+        return cast(type[Array], self._make_type(name, bases, size, alignment=type_.alignment, attrs=attrs))
 
     def _make_int_type(self, name: str, size: int, signed: bool, *, alignment: int | None = None) -> type[Int]:
-        return self._make_type(name, (Int,), size, alignment=alignment, attrs={"signed": signed})
+        return cast(type[Int], self._make_type(name, (Int,), size, alignment=alignment, attrs={"signed": signed}))
 
     def _make_packed_type(self, name: str, packchar: str, base: type, *, alignment: int | None = None) -> type[Packed]:
-        return self._make_type(
-            name,
-            (base, Packed),
-            struct.calcsize(packchar),
-            alignment=alignment,
-            attrs={"packchar": packchar},
+        return cast(
+            type[Packed],
+            self._make_type(
+                name,
+                (base, Packed),
+                struct.calcsize(packchar),
+                alignment=alignment,
+                attrs={"packchar": packchar},
+            ),
         )
 
-    def _make_enum(self, name: str, type_: MetaType, values: dict[str, int]) -> type[Enum]:
+    def _make_enum(self, name: str, type_: type[BaseType], values: dict[str, int]) -> type[Enum]:
         return Enum(self, name, type_, values)
 
-    def _make_flag(self, name: str, type_: MetaType, values: dict[str, int]) -> type[Flag]:
+    def _make_flag(self, name: str, type_: type[BaseType], values: dict[str, int]) -> type[Flag]:
         return Flag(self, name, type_, values)
 
-    def _make_pointer(self, target: MetaType) -> type[Pointer]:
+    def _make_pointer(self, target: type[BaseType]) -> type[Pointer]:
         return self._make_type(
             f"{target.__name__}*",
             (Pointer,),
@@ -409,18 +425,164 @@ class cstruct:
     ) -> type[Structure]:
         return self._make_struct(name, fields, align=align, anonymous=anonymous, base=Union)
 
+    Z = TYPE_CHECKING
+
+    if TYPE_CHECKING:
+        A = 1
+        # ruff: noqa: PYI042
+        _int = int
+        _float = float
+
+        class int8(_int, Packed[_int]): ...
+
+        class uint8(_int, Packed[_int]): ...
+
+        class int16(_int, Packed[_int]): ...
+
+        class uint16(_int, Packed[_int]): ...
+
+        class int32(_int, Packed[_int]): ...
+
+        class uint32(_int, Packed[_int]): ...
+
+        class int64(_int, Packed[_int]): ...
+
+        class uint64(_int, Packed[_int]): ...
+
+        class float16(_float, Packed[_float]): ...
+
+        class float(_float, Packed[_float]): ...
+
+        class double(_float, Packed[_float]): ...
+
+        class char(Char): ...
+
+        class wchar(Wchar): ...
+
+        class int24(Int): ...
+
+        class uint24(Int): ...
+
+        class int48(Int): ...
+
+        class uint48(Int): ...
+
+        class int128(Int): ...
+
+        class uint128(Int): ...
+
+        class uleb128(LEB128): ...
+
+        class ileb128(LEB128): ...
+
+        class void(Void): ...
+
+        # signed char: TypeAlias = int8
+        # signed char: TypeAlias = char
+        short: TypeAlias = int16
+        # signed short: TypeAlias = int16
+        # unsigned short: TypeAlias = uint16
+        int: TypeAlias = int32
+        # signed int: TypeAlias = int32
+        # unsigned int: TypeAlias = uint32
+        long: TypeAlias = int32
+        # signed long: TypeAlias = int32
+        # unsigned long: TypeAlias = uint32
+        # long long: TypeAlias = int64
+        # signed long long: TypeAlias = int64
+        # unsigned long long: TypeAlias = uint64
+
+        BYTE: TypeAlias = uint8
+        CHAR: TypeAlias = char
+        SHORT: TypeAlias = int16
+        WORD: TypeAlias = uint16
+        DWORD: TypeAlias = uint32
+        LONG: TypeAlias = int32
+        LONG32: TypeAlias = int32
+        LONG64: TypeAlias = int64
+        LONGLONG: TypeAlias = int64
+        QWORD: TypeAlias = uint64
+        OWORD: TypeAlias = uint128
+        WCHAR: TypeAlias = wchar
+
+        UCHAR: TypeAlias = uint8
+        USHORT: TypeAlias = uint16
+        ULONG: TypeAlias = uint32
+        ULONG64: TypeAlias = uint64
+        ULONGLONG: TypeAlias = uint64
+
+        INT: TypeAlias = int32
+        INT8: TypeAlias = int8
+        INT16: TypeAlias = int16
+        INT32: TypeAlias = int32
+        INT64: TypeAlias = int64
+        INT128: TypeAlias = int128
+
+        UINT: TypeAlias = uint32
+        UINT8: TypeAlias = uint8
+        UINT16: TypeAlias = uint16
+        UINT32: TypeAlias = uint32
+        UINT64: TypeAlias = uint64
+        UINT128: TypeAlias = uint128
+
+        __int8: TypeAlias = int8
+        __int16: TypeAlias = int16
+        __int32: TypeAlias = int32
+        __int64: TypeAlias = int64
+        __int128: TypeAlias = int128
+
+        # unsigned __int8: TypeAlias = uint8
+        # unsigned __int16: TypeAlias = uint16
+        # unsigned __int32: TypeAlias = uint32
+        # unsigned __int64: TypeAlias = uint64
+        # unsigned __int128: TypeAlias = uint128
+
+        wchar_t: TypeAlias = wchar
 
-def ctypes(structure: Structure) -> _ctypes.Structure:
+        int8_t: TypeAlias = int8
+        int16_t: TypeAlias = int16
+        int32_t: TypeAlias = int32
+        int64_t: TypeAlias = int64
+        int128_t: TypeAlias = int128
+
+        uint8_t: TypeAlias = uint8
+        uint16_t: TypeAlias = uint16
+        uint32_t: TypeAlias = uint32
+        uint64_t: TypeAlias = uint64
+        uint128_t: TypeAlias = uint128
+
+        _BYTE: TypeAlias = uint8
+        _WORD: TypeAlias = uint16
+        _DWORD: TypeAlias = uint32
+        _QWORD: TypeAlias = uint64
+        _OWORD: TypeAlias = uint128
+
+        u1: TypeAlias = uint8
+        u2: TypeAlias = uint16
+        u4: TypeAlias = uint32
+        u8: TypeAlias = uint64
+        u16: TypeAlias = uint128
+        __u8: TypeAlias = uint8
+        __u16: TypeAlias = uint16
+        __u32: TypeAlias = uint32
+        __u64: TypeAlias = uint64
+        uchar: TypeAlias = uint8
+        ushort: TypeAlias = uint16
+        uint: TypeAlias = uint32
+        ulong: TypeAlias = uint32
+
+
+def ctypes(structure: type[Structure]) -> type[_ctypes.Structure]:
     """Create ctypes structures from cstruct structures."""
     fields = []
     for field in structure.__fields__:
         t = ctypes_type(field.type)
-        fields.append((field.name, t))
+        fields.append((field._name, t))
 
-    return type(structure.name, (_ctypes.Structure,), {"_fields_": fields})
+    return type(structure.__name__, (_ctypes.Structure,), {"_fields_": fields})
 
 
-def ctypes_type(type_: MetaType) -> Any:
+def ctypes_type(type_: type[BaseType]) -> Any:
     mapping = {
         "b": _ctypes.c_int8,
         "B": _ctypes.c_uint8,
@@ -443,7 +605,7 @@ def ctypes_type(type_: MetaType) -> Any:
     if issubclass(type_, Wchar):
         return _ctypes.c_wchar
 
-    if isinstance(type_, ArrayMetaType):
+    if issubclass(type_, BaseArray):
         subtype = ctypes_type(type_.type)
         return subtype * type_.num_entries
 

{dissect_cstruct-4.4.dev3 → dissect_cstruct-4.5.dev1}/dissect/cstruct/parser.py

@@ -11,7 +11,7 @@ from dissect.cstruct.exceptions import (
     ParserError,
 )
 from dissect.cstruct.expression import Expression
-from dissect.cstruct.types import ArrayMetaType, Field, MetaType
+from dissect.cstruct.types import BaseArray, BaseType, Field, Structure
 
 if TYPE_CHECKING:
     from dissect.cstruct import cstruct
@@ -147,22 +147,28 @@ class TokenParser(Parser):
         tokens.consume()
         type_ = None
 
+        names = []
+
         if tokens.next == self.TOK.IDENTIFIER:
             type_ = self.cstruct.resolve(self._identifier(tokens))
         elif tokens.next == self.TOK.STRUCT:
-            # The register thing is a bit dirty
-            # Basically consumes all NAME tokens and
-            # registers the struct
-            type_ = self._struct(tokens, register=True)
+            type_ = self._struct(tokens)
+            if not type_.__anonymous__:
+                names.append(type_.__name__)
 
-        names = self._names(tokens)
+        names.extend(self._names(tokens))
         for name in names:
+            if issubclass(type_, Structure) and type_.__anonymous__:
+                type_.__anonymous__ = False
+                type_.__name__ = name
+                type_.__qualname__ = name
+
             type_, name, bits = self._parse_field_type(type_, name)
             if bits is not None:
                 raise ParserError(f"line {self._lineno(tokens.previous)}: typedefs cannot have bitfields")
             self.cstruct.add_type(name, type_)
 
-    def _struct(self, tokens: TokenConsumer, register: bool = False) -> None:
+    def _struct(self, tokens: TokenConsumer, register: bool = False) -> type[Structure]:
         stype = tokens.consume()
 
         factory = self.cstruct._make_union if stype.value.startswith("union") else self.cstruct._make_struct
@@ -204,9 +210,14 @@ class TokenParser(Parser):
             field = self._parse_field(tokens)
             fields.append(field)
 
-        # All names from here on are from typedef's
-        # Parsing names consumes the EOL token
-        names.extend(self._names(tokens))
+        if register:
+            names.extend(self._names(tokens))
+
+        # If the next token is EOL, consume it
+        # Otherwise we're part of a typedef or field definition
+        if tokens.next == self.TOK.EOL:
+            tokens.eol()
+
         name = names[0] if names else None
 
         if st is None:
@@ -251,8 +262,7 @@ class TokenParser(Parser):
             type_ = self._struct(tokens)
 
             if tokens.next != self.TOK.NAME:
-                type_, name, bits = self._parse_field_type(type_, type_.__name__)
-                return Field(name.strip(), type_, bits)
+                return Field(None, type_, None)
 
         if tokens.next != self.TOK.NAME:
             raise ParserError(f"line {self._lineno(tokens.next)}: expected name")
@@ -263,7 +273,7 @@ class TokenParser(Parser):
         tokens.eol()
         return Field(name.strip(), type_, bits)
 
-    def _parse_field_type(self, type_: MetaType, name: str) -> tuple[MetaType, str, int | None]:
+    def _parse_field_type(self, type_: type[BaseType], name: str) -> tuple[type[BaseType], str, int | None]:
         pattern = self.TOK.patterns[self.TOK.NAME]
         # Dirty trick because the regex expects a ; but we don't want it to be part of the value
         d = pattern.match(name + ";").groupdict()
@@ -289,7 +299,7 @@ class TokenParser(Parser):
                     except Exception:
                         pass
 
-                if isinstance(type_, ArrayMetaType) and count is None:
+                if issubclass(type_, BaseArray) and count is None:
                     raise ParserError("Depth required for multi-dimensional array")
 
                 type_ = self.cstruct._make_array(type_, count)
@@ -571,7 +581,7 @@ class TokenCollection:
 
         return object.__getattribute__(self, attr)
 
-    def add(self, regex: str, name: str) -> None:
+    def add(self, regex: str, name: str | None) -> None:
         if name is None:
             self.tokens.append((regex, None))
         else: