dissect.cstruct 5.0.dev14__tar.gz → 5.0.dev15__tar.gz

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.cstruct
-Version: 5.0.dev14
+Version: 5.0.dev15
 Summary: A Dissect module implementing a parser for C-like structures: structure parsing in Python made easy
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/dissect/cstruct/lexer.py

@@ -125,6 +125,7 @@ _SINGLE_CHARS = {
 
 _RE_IDENTIFIER = re.compile(r"[a-zA-Z_.][a-zA-Z0-9_.]*")
 _RE_WHITESPACE = re.compile(r"[ \t\r\n]+")
+_RE_DEFINE_LITERAL = re.compile(r"""(["'])(?:\\.|(?!\1)[^\\\n])*\1""")
 
 IDENTIFIER_TYPES = set(_C_KEYWORDS.values()) | {TokenType.IDENTIFIER}
 
@@ -292,10 +293,10 @@ class Lexer:
             if peek == "*":
                 self._take(2)  # Consume /*
                 end = self.data.find("*/", self._pos)
-                if end != -1:
-                    self._take(end - self._pos + 2)
-                else:
-                    self._take(len(self.data) - self._pos)
+                if end == -1:
+                    raise self._error("unterminated comment")
+
+                self._take(end - self._pos + 2)
 
             elif peek == "/":
                 self._take(2)  # Consume //
@@ -390,6 +391,23 @@ class Lexer:
 
         return result
 
+    def _read_define_value(self) -> str:
+        """Read the value of a ``#define`` up to the end of the line."""
+        value = []
+
+        while not self.eof and (ch := self._current()) != "\n":
+            if ch == "/" and self._peek() in ("/", "*"):
+                self._skip_comment()
+                value.append(" ")
+            elif ch in ('"', "'"):
+                if not (match := _RE_DEFINE_LITERAL.match(self.data, self._pos)):
+                    raise self._error("unterminated string literal")
+                value.append(self._take(match.end() - self._pos))
+            else:
+                value.append(self._read_until("\n\"'/") or self._take())
+
+        return "".join(value).strip()
+
     def _read_preprocessor(self) -> None:
         """Read a preprocessor directive starting with ``#``."""
         line = self._line
@@ -423,12 +441,7 @@ class Lexer:
                 raise self._error("expected identifier after '#define'", lineno=line)
             self._emit(TokenType.IDENTIFIER, name, line)
 
-            self._skip_whitespace_and_comments()
-            if self.eof or self._line != line:
-                # No value, just a simple macro definition
-                return
-
-            if (value := self._read_until("\n")).strip():
+            if value := self._read_define_value():
                 self._emit(TokenType.STRING, value, line)
 
         elif token_type == TokenType.PP_INCLUDE:

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/dissect/cstruct/parser.py

@@ -20,6 +20,10 @@ if TYPE_CHECKING:
     from dissect.cstruct.types import BaseType, Enum, Flag
 
 
+# A string or the name of a constant in the value of a define
+_RE_DEFINE_STRING = re.compile(r"""\s*(?:([bB]?)(["'])((?:\\.|(?!\2)[^\\\n])*)\2|([a-zA-Z_.][a-zA-Z0-9_.]*))""")
+
+
 class Parser(TokenCursor):
     """Base class for definition parsers.
 
@@ -42,7 +46,7 @@ class Parser(TokenCursor):
 
 def _join_line_continuations(string: str) -> str:
     # Join lines ending with backslash
-    return re.sub(r"\\\n", "", string)
+    return re.sub(r"\\\r?\n", "", string)
 
 
 class CStyleParser(Parser):
@@ -138,11 +142,16 @@ class CStyleParser(Parser):
             elif token.type == TokenType.TYPEDEF:
                 self._parse_typedef()
             elif token.type in (TokenType.STRUCT, TokenType.UNION):
-                self._parse_struct_or_union()
+                type_ = self._parse_struct_or_union()
 
                 # Skip variable declarations after struct/union definitions
-                while not self._at(TokenType.SEMICOLON, TokenType.EOF):
-                    self._pos += 1
+                has_variable = self._at(TokenType.IDENTIFIER, TokenType.STAR)
+                while has_variable:
+                    _, _, bits = self._parse_field_name(type_)
+                    if bits is not None:
+                        raise self._error("variables cannot have bitfields")
+
+                    has_variable = self._match(TokenType.COMMA)
 
                 self._expect(TokenType.SEMICOLON)
             elif token.type in (TokenType.ENUM, TokenType.FLAG):
@@ -174,21 +183,28 @@ class CStyleParser(Parser):
             value = self._take().value
 
         if value:
-            if value[0] in ('"', "'"):
-                quote = value[0]
-                if value[-1] != quote:
-                    raise self._error("unterminated string literal", token=token)
-
-                # Remove the surrounding quotes and flatten escape sequences
-                value = value[1:-1].encode().decode("unicode_escape")
-            elif value[:2].lower() in ("b'", 'b"'):
-                quote = value[1]
-                if value[-1] != quote:
-                    raise self._error("unterminated bytes literal", token=token)
-
-                # Remove the leading b and surrounding quotes and flatten escape sequences
-                value = value[2:-1].encode().decode("unicode_escape")
-                value = ast.literal_eval(f"b{value!r}")
+            # Adjacent strings and constants that are strings are concatenated like in C
+            strings = []
+            pos = 0
+            if '"' in value or "'" in value:
+                while pos < len(value) and (match := _RE_DEFINE_STRING.match(value, pos)):
+                    is_bytes, _, string, name = match.groups()
+                    if name is None:
+                        # Flatten escape sequences
+                        string = string.encode().decode("unicode_escape")
+                        strings.append(string.encode("latin-1") if is_bytes else string)
+                    elif isinstance(constant := self.cs.consts.get(name), (str, bytes)):
+                        strings.append(constant)
+                    else:
+                        break
+
+                    pos = match.end()
+
+            if strings and pos == len(value):
+                try:
+                    value = strings[0][:0].join(strings)
+                except TypeError:
+                    raise self._error("cannot concatenate strings and bytes", token=token) from None
             else:
                 try:
                     # Lazy mode, try to evaluate as a Python literal first (for simple constants)

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/dissect.cstruct.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.cstruct
-Version: 5.0.dev14
+Version: 5.0.dev15
 Summary: A Dissect module implementing a parser for C-like structures: structure parsing in Python made easy
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/dissect.cstruct.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
   "tag": "5.0.dev0",
-  "distance": 14,
-  "node": "g2688580e0f466b79d4fc7fb94562f0defb978796",
+  "distance": 15,
+  "node": "ge066596a738ce1f58096078ed54b60e62f4032fc",
   "dirty": true,
   "branch": "main",
   "node_date": "2026-11-04"

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/tests/test_lexer.py

@@ -90,14 +90,6 @@ from dissect.cstruct.lexer import TokenType, tokenize
             [TokenType.PP_DEFINE, TokenType.IDENTIFIER, TokenType.STRING],
             ["define", "FOO", "ADCRYPT\00"],
         ),
-        (
-            """
-            #define FOO (1 \
-            + 2)
-            """,
-            [TokenType.PP_DEFINE, TokenType.IDENTIFIER, TokenType.STRING],
-            ["define", "FOO", "(1             + 2)"],
-        ),
         ("#undef", [TokenType.PP_UNDEF], ["undef"]),
         ("#ifdef", [TokenType.PP_IFDEF], ["ifdef"]),
         ("#ifndef", [TokenType.PP_IFNDEF], ["ifndef"]),
@@ -174,6 +166,14 @@ def test_token(src: str, types: list[TokenType], values: list[str]) -> None:
         ("#include", "unexpected end of input"),
         ("#include <missing_end", "unexpected end of input"),
         ('#include "missing_end', "unexpected end of input"),
+        ('#define A "x\ny"\n#define B 2\n', "line 1: unterminated string literal"),
+        ('#define A "x\n#define B "y"\n', "line 1: unterminated string literal"),
+        ('#define A "x', "line 1: unterminated string literal"),
+        ("#define A 'x\n", "line 1: unterminated string literal"),
+        ("#define A it's\n", "line 1: unterminated string literal"),
+        ("/* unterminated", "line 1: unterminated comment"),
+        ("struct a {\n    uint8 b; /* c\n};\n", "line 2: unterminated comment"),
+        ("#define A 1 /* c\n#define B 2\n", "line 1: unterminated comment"),
         ("@", "unexpected character"),
         ("`", "unexpected character"),
     ],

{dissect_cstruct-5.0.dev14 → dissect_cstruct-5.0.dev15}/tests/test_parser.py

@@ -41,6 +41,65 @@ def test_struct(cs: cstruct, compiled: bool) -> None:
             cs.resolve(name)
 
 
+def test_struct_variables(cs: cstruct) -> None:
+    """Test that all forms of variable declarations after a struct or union definition are silently ignored."""
+    cdef = """
+    #define SIZE 2
+
+    struct test {
+        uint8 a;
+    } b[4], *c[SIZE], d[2][3], e[];
+
+    union test1 {
+        uint8 a;
+    } f;
+
+    struct test g, *h;
+
+    typedef uint8 test2;
+    """
+    cs.load(cdef)
+
+    assert cs.resolve("test") is cs.test
+    assert cs.resolve("test1") is cs.test1
+    assert cs.resolve("test2") is cs.uint8
+
+    for name in ("b", "c", "d", "e", "f", "g", "h"):
+        with pytest.raises(ResolveError, match=f"Unknown type {name}"):
+            cs.resolve(name)
+
+
+@pytest.mark.parametrize(
+    ("cdef", "match"),
+    [
+        # A missing semicolon before the next definition
+        ("struct a { uint8 x; }\nstruct b { uint8 x; };", "line 2: expected ;, got struct"),
+        ("struct a { uint8 x; }\nunion b { uint8 x; };", "line 2: expected ;, got union"),
+        ("struct a { uint8 x; }\nenum b : uint8 { C };", "line 2: expected ;, got enum"),
+        ("struct a { uint8 x; }\nflag b : uint8 { C };", "line 2: expected ;, got flag"),
+        ("struct a { uint8 x; }\ntypedef uint8 b;", "line 2: expected ;, got typedef"),
+        ("struct a { uint8 x; }\n#define B 1", "line 2: expected ;, got define"),
+        ("struct a { uint8 x; }\n#[nocompile]\nstruct b { uint8 x; };", "line 2: expected ;, got nocompile"),
+        ("union a { uint8 x; }\ntypedef uint8 b;", "line 2: expected ;, got typedef"),
+        ("struct a { uint8 x; } b\ntypedef uint8 c;", "line 2: expected ;, got typedef"),
+        ("struct a { uint8 x; }", "line 1: expected ;, got end of input"),
+        ("struct a { uint8 x; } b", "line 1: expected ;, got end of input"),
+        # Invalid variable declarations
+        ("struct a { uint8 x; } b c;", "line 1: expected ;, got c"),
+        ("struct a { uint8 x; } 1;", "line 1: expected ;, got 1"),
+        ("struct a { uint8 x; } , b;", "line 1: expected ;, got ,"),
+        ("struct a { uint8 x; } b,;", "line 1: expected one of .*, got ;"),
+        ("struct a { uint8 x; } *;", "line 1: expected one of .*, got ;"),
+        ("struct a { uint8 x; } b[2;", "line 1: expected ], got end of input"),
+        ("struct a { uint8 x; } b : 4;", "line 1: variables cannot have bitfields"),
+    ],
+)
+def test_struct_invalid_variables(cs: cstruct, cdef: str, match: str) -> None:
+    """Test that nothing other than variable declarations and a semicolon can follow a struct or union definition."""
+    with pytest.raises(ParserError, match=match):
+        cs.load(cdef)
+
+
 def test_nested_structs(cs: cstruct, compiled: bool) -> None:
     """Test parsing of nested structs, including anonymous ones."""
     cdef = """
@@ -248,14 +307,18 @@ def test_define(cs: cstruct) -> None:
     #define NULLSTR "ADCRYPT\00"
     #define NULLBYTES b"ADCRYPT\00"
     #define ARBITRARYBYTES b"\x00\x01\x02"
-    #define MULTILINE (1 + \
-                        2 + \
+    #define MULTILINE (1 + \\
+                        2 + \\
                         3)
-    #define QUOTES "\'\"a'b\""
+    #define QUOTES "'\\"a'b\\""
     #define ESCAPE "\\'\\"a'b\\"\\n"
     #define BYTES_ESCAPE b"`\\n"
     #define FUNC(x) ( x == 0 )
     #define TERNARY(x) ( x ? 1 : 0 )
+    #define STR_CONCAT "say " STR
+    #define BYTES_CONCAT BYTES b"!"
+    #define STR_AND_NUMBER "a" CONST
+    #define RAW_AND_RAW RAW RAW
     """
     cs.load(cdef)
 
@@ -275,6 +338,16 @@ def test_define(cs: cstruct) -> None:
     # We don't evaluate function-like macros yet, so they should be stored as their raw string representation
     assert cs.consts["FUNC"] == "(x) ( x == 0 )"
     assert cs.consts["TERNARY"] == "(x) ( x ? 1 : 0 )"
+    # Adjacent strings and constants that are strings are concatenated like in C
+    assert cs.consts["STR_CONCAT"] == "say hello"
+    assert cs.consts["BYTES_CONCAT"] == b"world!"
+    # A constant that isn't a string can't be concatenated, so the value is kept as it is
+    assert cs.consts["STR_AND_NUMBER"] == '"a" CONST'
+    # Without a string in the value we don't know whether these are strings, they could also just be raw values
+    assert cs.consts["RAW_AND_RAW"] == "RAW RAW"
+
+    with pytest.raises(ParserError, match="line 1: cannot concatenate strings and bytes"):
+        cs.load('#define MIXED "a" b"b"')
 
 
 def test_define_flag_value(cs: cstruct) -> None:
@@ -427,19 +500,6 @@ def test_conditional_parsing_error(cs: cstruct) -> None:
         cs.load(cdef)
 
 
-def test_multiline_define(cs: cstruct) -> None:
-    """Test parsing of multi-line ``#define`` directives."""
-    cdef = """
-    #define MULTILINE_DEF (1 + \\
-                          2 + \\
-                          3)
-    """
-    cs.load(cdef)
-
-    assert "MULTILINE_DEF" in cs.consts
-    assert cs.consts["MULTILINE_DEF"] == 6
-
-
 def test_multiple_declarators(cs: cstruct) -> None:
     """Test parsing of multiple declarators in a single struct field declaration."""
     cdef = """
@@ -595,3 +655,51 @@ def test_error_context(cs: cstruct) -> None:
           3: 69
         """.rstrip()
     )
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        # A comment is replaced by a space, after which the value continues
+        ("data /* c */", "data"),
+        ("a/* c */b", "a b"),
+        ("data /* c\n#define C 3\n d */", "data"),
+        ("data // c", "data"),
+        ("1 // it's", 1),
+        ("http://x", "http:"),
+        ("1 /* c\n d */ + 2", 3),
+        ("/* c\n d */ 1", 1),
+        ("/* c */", ""),
+        # Comments after and in strings
+        ('"a" /* c\n d */', "a"),
+        ('"a" // it\'s "b"', "a"),
+        ("'b' /* c\n d */", "b"),
+        ('b"c" // c', b"c"),
+        ("b'd' /* c */", b"d"),
+        ('"\\"f\\"" // c', '"f"'),
+        ('"it\'s /* c */ // d"', "it's /* c */ // d"),
+        # Escapes and spaces in strings
+        ('"a\\\\"', "a\\"),
+        ('"a   b"', "a   b"),
+        ('b"\\xff"', b"\xff"),
+        # Line continuations are joined first
+        ('"a\\\nb"', "ab"),
+        ("1 /* c */ + \\\n 2", 3),
+        ('"a" // c \\\n "b"', "a"),
+        ("1 + \\\r\n 2\r", 3),
+        # Adjacent strings are concatenated
+        ('"bar" /* c */ \\\n "zomg"', "barzomg"),
+        # CRLF line endings
+        ('"a"\r', "a"),
+        ("data\r", "data"),
+        # Anything that isn't only strings is kept as it is, like other macro-like constants
+        ('"a" + 1', '"a" + 1'),
+    ],
+)
+def test_define_value(cs: cstruct, value: str, expected: int | str | bytes) -> None:
+    """Test that the value of a ``#define`` follows the rules of C for comments, continuations and strings."""
+    cs.load(f"#define A {value}\n#define B 2\n")
+
+    assert cs.consts["A"] == expected
+    assert cs.consts["B"] == 2
+    assert "C" not in cs.consts