dissect.cstruct 5.0.dev9__py3-none-any.whl → 5.0.dev11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dissect/cstruct/compiler.py +683 -221
- dissect/cstruct/cstruct.py +19 -4
- dissect/cstruct/expression.py +139 -40
- dissect/cstruct/parser.py +2 -2
- dissect/cstruct/types/base.py +88 -30
- dissect/cstruct/types/char.py +4 -1
- dissect/cstruct/types/enum.py +20 -7
- dissect/cstruct/types/flag.py +40 -7
- dissect/cstruct/types/int.py +21 -2
- dissect/cstruct/types/packed.py +32 -3
- dissect/cstruct/types/structure.py +53 -34
- dissect/cstruct/types/wchar.py +5 -2
- {dissect_cstruct-5.0.dev9.dist-info → dissect_cstruct-5.0.dev11.dist-info}/METADATA +1 -1
- dissect_cstruct-5.0.dev11.dist-info/RECORD +30 -0
- {dissect_cstruct-5.0.dev9.dist-info → dissect_cstruct-5.0.dev11.dist-info}/WHEEL +1 -1
- dissect_cstruct-5.0.dev9.dist-info/RECORD +0 -30
- {dissect_cstruct-5.0.dev9.dist-info → dissect_cstruct-5.0.dev11.dist-info}/entry_points.txt +0 -0
- {dissect_cstruct-5.0.dev9.dist-info → dissect_cstruct-5.0.dev11.dist-info}/licenses/COPYRIGHT +0 -0
- {dissect_cstruct-5.0.dev9.dist-info → dissect_cstruct-5.0.dev11.dist-info}/licenses/LICENSE +0 -0
- {dissect_cstruct-5.0.dev9.dist-info → dissect_cstruct-5.0.dev11.dist-info}/top_level.txt +0 -0
dissect/cstruct/cstruct.py
CHANGED
|
@@ -8,6 +8,7 @@ import sys
|
|
|
8
8
|
import types
|
|
9
9
|
import warnings
|
|
10
10
|
from contextlib import contextmanager
|
|
11
|
+
from functools import lru_cache
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
from typing import TYPE_CHECKING, Any, BinaryIO, Literal, TypeVar, cast
|
|
13
14
|
|
|
@@ -32,7 +33,7 @@ from dissect.cstruct.types import (
|
|
|
32
33
|
from dissect.cstruct.types.base import normalize_endianness
|
|
33
34
|
|
|
34
35
|
if TYPE_CHECKING:
|
|
35
|
-
from collections.abc import
|
|
36
|
+
from collections.abc import Generator, Iterable
|
|
36
37
|
from typing import TypeAlias
|
|
37
38
|
|
|
38
39
|
from dissect.cstruct.types import (
|
|
@@ -203,6 +204,9 @@ class cstruct:
|
|
|
203
204
|
self.pointer: type[BaseType] = self.resolve(pointer)
|
|
204
205
|
self._anonymous_count = 0
|
|
205
206
|
|
|
207
|
+
# Array types are frequently created at runtime (e.g. ``cs.uint32[count]``), so cache them per instance
|
|
208
|
+
self._make_array_cached = lru_cache(maxsize=4096)(self._make_array_cached)
|
|
209
|
+
|
|
206
210
|
if load:
|
|
207
211
|
self.load(load)
|
|
208
212
|
|
|
@@ -467,7 +471,7 @@ class cstruct:
|
|
|
467
471
|
return copy.copy(self)
|
|
468
472
|
|
|
469
473
|
@contextmanager
|
|
470
|
-
def endianness(self, endian: AllowedEndianness) ->
|
|
474
|
+
def endianness(self, endian: AllowedEndianness) -> Generator[cstruct]:
|
|
471
475
|
"""Context manager for temporarily changing the endianness of this cstruct instance."""
|
|
472
476
|
original = self.endian
|
|
473
477
|
self.endian = normalize_endianness(endian)
|
|
@@ -477,13 +481,13 @@ class cstruct:
|
|
|
477
481
|
self.endian = original
|
|
478
482
|
|
|
479
483
|
@contextmanager
|
|
480
|
-
def big_endian(self) ->
|
|
484
|
+
def big_endian(self) -> Generator[cstruct]:
|
|
481
485
|
"""Context manager for temporarily changing the endianness to big endian."""
|
|
482
486
|
with self.endianness(">") as cs:
|
|
483
487
|
yield cs
|
|
484
488
|
|
|
485
489
|
@contextmanager
|
|
486
|
-
def little_endian(self) ->
|
|
490
|
+
def little_endian(self) -> Generator[cstruct]:
|
|
487
491
|
"""Context manager for temporarily changing the endianness to little endian."""
|
|
488
492
|
with self.endianness("<") as cs:
|
|
489
493
|
yield cs
|
|
@@ -513,6 +517,17 @@ class cstruct:
|
|
|
513
517
|
return types.new_class(name, bases, {}, lambda ns: ns.update(attrs))
|
|
514
518
|
|
|
515
519
|
def _make_array(self, type_: T, num_entries: int | Expression | None) -> type[Array[T]]:
|
|
520
|
+
if isinstance(num_entries, Expression):
|
|
521
|
+
# Only created when parsing a definition, so no point in caching
|
|
522
|
+
return self._make_array_type(type_, num_entries)
|
|
523
|
+
|
|
524
|
+
# Structures can be dynamically modified, so include the size and alignment in the cache key
|
|
525
|
+
return self._make_array_cached(type_, num_entries, type_.__size__, type_.__alignment__)
|
|
526
|
+
|
|
527
|
+
def _make_array_cached(self, type_: T, num_entries: int | None, size: int | None, alignment: int) -> type[Array[T]]:
|
|
528
|
+
return self._make_array_type(type_, num_entries)
|
|
529
|
+
|
|
530
|
+
def _make_array_type(self, type_: T, num_entries: int | Expression | None) -> type[Array[T]]:
|
|
516
531
|
null_terminated = False
|
|
517
532
|
if num_entries is None:
|
|
518
533
|
null_terminated = True
|
dissect/cstruct/expression.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
4
|
|
|
5
5
|
from dissect.cstruct.exception import ExpressionParserError
|
|
6
6
|
from dissect.cstruct.lexer import IDENTIFIER_TYPES, Lexer, TokenCursor, TokenType
|
|
@@ -34,6 +34,21 @@ UNARY_OPERATORS: dict[TokenType, Callable[[int], int]] = {
|
|
|
34
34
|
|
|
35
35
|
OPERATORS = set(BINARY_OPERATORS.keys()) | set(UNARY_OPERATORS.keys())
|
|
36
36
|
|
|
37
|
+
OPERATOR_SOURCE: dict[TokenType, str] = {
|
|
38
|
+
TokenType.PIPE: "|",
|
|
39
|
+
TokenType.CARET: "^",
|
|
40
|
+
TokenType.AMPERSAND: "&",
|
|
41
|
+
TokenType.LSHIFT: "<<",
|
|
42
|
+
TokenType.RSHIFT: ">>",
|
|
43
|
+
TokenType.PLUS: "+",
|
|
44
|
+
TokenType.MINUS: "-",
|
|
45
|
+
TokenType.STAR: "*",
|
|
46
|
+
TokenType.SLASH: "//",
|
|
47
|
+
TokenType.PERCENT: "%",
|
|
48
|
+
TokenType.UNARY_MINUS: "-",
|
|
49
|
+
TokenType.TILDE: "~",
|
|
50
|
+
}
|
|
51
|
+
|
|
37
52
|
FUNCTION_TOKENS = {
|
|
38
53
|
TokenType.SIZEOF: 1,
|
|
39
54
|
TokenType.OFFSETOF: 2,
|
|
@@ -58,6 +73,100 @@ PRECEDENCE_LEVELS = {
|
|
|
58
73
|
}
|
|
59
74
|
|
|
60
75
|
|
|
76
|
+
def _sizeof(cs: cstruct, name: str) -> int:
|
|
77
|
+
return sizeof(cs.resolve(name))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _offsetof(cs: cstruct, name: str, field: str) -> int:
|
|
81
|
+
return offsetof(cs.resolve(name), field)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
FUNCTIONS: dict[TokenType, Callable[..., int]] = {
|
|
85
|
+
TokenType.SIZEOF: _sizeof,
|
|
86
|
+
TokenType.OFFSETOF: _offsetof,
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _error(msg: str, line: int) -> ExpressionParserError:
|
|
91
|
+
return ExpressionParserError(f"line {line}: {msg}")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _identifier(cs: cstruct, context: dict[str, Any], ident: str, line: int) -> int:
|
|
95
|
+
"""Resolve an identifier to an integer value."""
|
|
96
|
+
obj = None
|
|
97
|
+
for i, part in enumerate(ident.split(".")):
|
|
98
|
+
if i == 0:
|
|
99
|
+
if part in context:
|
|
100
|
+
obj = context[part]
|
|
101
|
+
elif part in cs.consts:
|
|
102
|
+
obj = cs.consts[part]
|
|
103
|
+
elif part in cs.types:
|
|
104
|
+
obj = cs.types[part]
|
|
105
|
+
else:
|
|
106
|
+
raise _error(f"Unknown identifier: '{ident}'", line)
|
|
107
|
+
else:
|
|
108
|
+
obj = obj.get(part, _MISSING) if isinstance(obj, dict) else getattr(obj, part, _MISSING)
|
|
109
|
+
if obj is _MISSING:
|
|
110
|
+
raise _error(f"Unknown identifier: '{ident}'", line)
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
return int(obj)
|
|
114
|
+
except (ValueError, TypeError):
|
|
115
|
+
raise _error(f"Identifier '{ident}' does not resolve to an integer", line)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# Globals used by the source code of an expression
|
|
119
|
+
SOURCE_GLOBALS = {func.__name__: func for func in (_identifier, *FUNCTIONS.values())}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class _Evaluator:
|
|
123
|
+
"""Calculates the value of an expression."""
|
|
124
|
+
|
|
125
|
+
def __init__(self, cs: cstruct, context: dict[str, Any]):
|
|
126
|
+
self.cs = cs
|
|
127
|
+
self.context = context
|
|
128
|
+
|
|
129
|
+
def number(self, value: int) -> int:
|
|
130
|
+
return value
|
|
131
|
+
|
|
132
|
+
def identifier(self, ident: str, token: Token) -> int:
|
|
133
|
+
return _identifier(self.cs, self.context, ident, token.line)
|
|
134
|
+
|
|
135
|
+
def unary(self, operator: TokenType, value: int) -> int:
|
|
136
|
+
return UNARY_OPERATORS[operator](value)
|
|
137
|
+
|
|
138
|
+
def binary(self, operator: TokenType, left: int, right: int) -> int:
|
|
139
|
+
return BINARY_OPERATORS[operator](left, right)
|
|
140
|
+
|
|
141
|
+
def function(self, operator: TokenType, args: list[str]) -> int:
|
|
142
|
+
return FUNCTIONS[operator](self.cs, *args)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _SourceBuilder:
|
|
146
|
+
"""Builds the Python source code of an expression."""
|
|
147
|
+
|
|
148
|
+
def __init__(self, variables: dict[str, str], context: str):
|
|
149
|
+
self.variables = variables
|
|
150
|
+
self.context = context
|
|
151
|
+
|
|
152
|
+
def number(self, value: int) -> str:
|
|
153
|
+
return repr(value)
|
|
154
|
+
|
|
155
|
+
def identifier(self, ident: str, token: Token) -> str:
|
|
156
|
+
if ident in self.variables:
|
|
157
|
+
return self.variables[ident]
|
|
158
|
+
return f"_identifier(cs, {self.context}, {ident!r}, {token.line})"
|
|
159
|
+
|
|
160
|
+
def unary(self, operator: TokenType, value: str) -> str:
|
|
161
|
+
return f"({OPERATOR_SOURCE[operator]}{value})"
|
|
162
|
+
|
|
163
|
+
def binary(self, operator: TokenType, left: str, right: str) -> str:
|
|
164
|
+
return f"({left} {OPERATOR_SOURCE[operator]} {right})"
|
|
165
|
+
|
|
166
|
+
def function(self, operator: TokenType, args: list[str]) -> str:
|
|
167
|
+
return f"{FUNCTIONS[operator].__name__}(cs, {', '.join(map(repr, args))})"
|
|
168
|
+
|
|
169
|
+
|
|
61
170
|
def precedence(o1: TokenType, o2: TokenType) -> bool:
|
|
62
171
|
# Unary (prefix) operators are right-associative, so use strict greater-than
|
|
63
172
|
if o2 in UNARY_OPERATORS:
|
|
@@ -86,9 +195,9 @@ class Expression(TokenCursor):
|
|
|
86
195
|
self._queue = []
|
|
87
196
|
|
|
88
197
|
def _error(self, msg: str, *, token: Token | None = None) -> ExpressionParserError:
|
|
89
|
-
return
|
|
198
|
+
return _error(msg, (token if token is not None else self._current()).line)
|
|
90
199
|
|
|
91
|
-
def _evaluate_expression(self,
|
|
200
|
+
def _evaluate_expression(self, backend: _Evaluator | _SourceBuilder) -> None:
|
|
92
201
|
operator = self._stack.pop(-1)
|
|
93
202
|
result = 0
|
|
94
203
|
|
|
@@ -96,37 +205,48 @@ class Expression(TokenCursor):
|
|
|
96
205
|
if len(self._queue) < 1:
|
|
97
206
|
raise ExpressionParserError("Invalid expression: not enough operands")
|
|
98
207
|
|
|
99
|
-
result =
|
|
208
|
+
result = backend.unary(operator, self._queue.pop(-1))
|
|
100
209
|
elif operator in BINARY_OPERATORS:
|
|
101
210
|
if len(self._queue) < 2:
|
|
102
211
|
raise ExpressionParserError("Invalid expression: not enough operands")
|
|
103
212
|
|
|
104
213
|
right = self._queue.pop(-1)
|
|
105
214
|
left = self._queue.pop(-1)
|
|
106
|
-
result =
|
|
215
|
+
result = backend.binary(operator, left, right)
|
|
107
216
|
elif operator in FUNCTION_TOKENS:
|
|
108
217
|
num_args = FUNCTION_TOKENS[operator]
|
|
109
218
|
if len(self._queue) < num_args:
|
|
110
219
|
raise ExpressionParserError("Invalid expression: not enough operands")
|
|
111
220
|
|
|
112
221
|
args = [self._queue.pop(-1) for _ in range(num_args)][::-1]
|
|
113
|
-
|
|
114
|
-
type_ = cs.resolve(args[0])
|
|
115
|
-
result = sizeof(type_)
|
|
116
|
-
elif operator == TokenType.OFFSETOF:
|
|
117
|
-
type_ = cs.resolve(args[0])
|
|
118
|
-
result = offsetof(type_, args[1])
|
|
222
|
+
result = backend.function(operator, args)
|
|
119
223
|
|
|
120
224
|
self._queue.append(result)
|
|
121
225
|
|
|
122
226
|
def evaluate(self, cs: cstruct, context: dict[str, int] | None = None) -> int:
|
|
123
|
-
"""Evaluates an expression using a Shunting-Yard implementation.
|
|
227
|
+
"""Evaluates an expression using a Shunting-Yard implementation.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
cs: The cstruct instance to evaluate against.
|
|
231
|
+
context: The context to evaluate against.
|
|
232
|
+
"""
|
|
233
|
+
return self._evaluate(_Evaluator(cs, context or {}))
|
|
234
|
+
|
|
235
|
+
def source(self, variables: dict[str, str] | None = None, context: str = "context") -> str:
|
|
236
|
+
"""Return the Python source code of this expression.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
variables: The source code map for accessing local variables.
|
|
240
|
+
context: The symbol name of the context.
|
|
241
|
+
"""
|
|
242
|
+
return self._evaluate(_SourceBuilder(variables or {}, context))
|
|
243
|
+
|
|
244
|
+
def _evaluate(self, backend: _Evaluator | _SourceBuilder) -> int | str:
|
|
124
245
|
self._reset()
|
|
125
|
-
context = context or {}
|
|
126
246
|
|
|
127
247
|
while (token := self._current()).type != TokenType.EOF:
|
|
128
248
|
if token.type == TokenType.NUMBER:
|
|
129
|
-
self._queue.append(int(self._take().value, 0))
|
|
249
|
+
self._queue.append(backend.number(int(self._take().value, 0)))
|
|
130
250
|
|
|
131
251
|
elif token.type in OPERATORS:
|
|
132
252
|
while (
|
|
@@ -134,7 +254,7 @@ class Expression(TokenCursor):
|
|
|
134
254
|
and self._stack[-1] != TokenType.LPAREN
|
|
135
255
|
and precedence(self._stack[-1], token.type)
|
|
136
256
|
):
|
|
137
|
-
self._evaluate_expression(
|
|
257
|
+
self._evaluate_expression(backend)
|
|
138
258
|
|
|
139
259
|
self._stack.append(self._take().type)
|
|
140
260
|
|
|
@@ -154,31 +274,10 @@ class Expression(TokenCursor):
|
|
|
154
274
|
self._expect(TokenType.RPAREN)
|
|
155
275
|
|
|
156
276
|
# Evaluate immediately
|
|
157
|
-
self._evaluate_expression(
|
|
277
|
+
self._evaluate_expression(backend)
|
|
158
278
|
|
|
159
279
|
elif token.type in IDENTIFIER_TYPES:
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
obj = None
|
|
163
|
-
for i, part in enumerate(ident.split(".")):
|
|
164
|
-
if i == 0:
|
|
165
|
-
if part in context:
|
|
166
|
-
obj = context[part]
|
|
167
|
-
elif part in cs.consts:
|
|
168
|
-
obj = cs.consts[part]
|
|
169
|
-
elif part in cs.types:
|
|
170
|
-
obj = cs.types[part]
|
|
171
|
-
else:
|
|
172
|
-
raise self._error(f"Unknown identifier: '{ident}'", token=token)
|
|
173
|
-
else:
|
|
174
|
-
obj = obj.get(part, _MISSING) if isinstance(obj, dict) else getattr(obj, part, _MISSING)
|
|
175
|
-
if obj is _MISSING:
|
|
176
|
-
raise self._error(f"Unknown identifier: '{ident}'", token=token)
|
|
177
|
-
|
|
178
|
-
try:
|
|
179
|
-
self._queue.append(int(obj))
|
|
180
|
-
except (ValueError, TypeError):
|
|
181
|
-
raise self._error(f"Identifier '{ident}' does not resolve to an integer", token=token)
|
|
280
|
+
self._queue.append(backend.identifier(self._take().value, token))
|
|
182
281
|
|
|
183
282
|
elif token.type == TokenType.LPAREN:
|
|
184
283
|
if self._previous().type == TokenType.NUMBER:
|
|
@@ -200,7 +299,7 @@ class Expression(TokenCursor):
|
|
|
200
299
|
raise self._error("Mismatched parentheses")
|
|
201
300
|
|
|
202
301
|
while self._stack[-1] != TokenType.LPAREN:
|
|
203
|
-
self._evaluate_expression(
|
|
302
|
+
self._evaluate_expression(backend)
|
|
204
303
|
if len(self._stack) == 0:
|
|
205
304
|
raise self._error("Mismatched parentheses")
|
|
206
305
|
|
|
@@ -213,7 +312,7 @@ class Expression(TokenCursor):
|
|
|
213
312
|
while len(self._stack) != 0:
|
|
214
313
|
if TokenType.LPAREN in self._stack:
|
|
215
314
|
raise self._error("Mismatched parentheses")
|
|
216
|
-
self._evaluate_expression(
|
|
315
|
+
self._evaluate_expression(backend)
|
|
217
316
|
|
|
218
317
|
if len(self._queue) != 1:
|
|
219
318
|
raise self._error("Invalid expression: too many operands")
|
dissect/cstruct/parser.py
CHANGED
|
@@ -342,7 +342,7 @@ class CStyleParser(Parser):
|
|
|
342
342
|
# Named struct/union, empty pre-register for self-referencing
|
|
343
343
|
type = factory(name, [], align=self.align)
|
|
344
344
|
if self.compiled and "nocompile" not in self._flags:
|
|
345
|
-
type = compiler.
|
|
345
|
+
type = compiler.compile_structure(type)
|
|
346
346
|
self.cs.add_type(name, type)
|
|
347
347
|
else:
|
|
348
348
|
# struct typename ... (type reference)
|
|
@@ -359,7 +359,7 @@ class CStyleParser(Parser):
|
|
|
359
359
|
|
|
360
360
|
type = factory(name, fields, align=self.align, anonymous=is_anonymous)
|
|
361
361
|
if self.compiled and "nocompile" not in self._flags:
|
|
362
|
-
type = compiler.
|
|
362
|
+
type = compiler.compile_structure(type)
|
|
363
363
|
else:
|
|
364
364
|
type.__fields__.extend(fields)
|
|
365
365
|
type.commit()
|
dissect/cstruct/types/base.py
CHANGED
|
@@ -2,11 +2,10 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import functools
|
|
4
4
|
import warnings
|
|
5
|
-
from io import BytesIO
|
|
5
|
+
from io import SEEK_CUR, BytesIO
|
|
6
6
|
from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar
|
|
7
7
|
|
|
8
8
|
from dissect.cstruct.exception import ArraySizeError
|
|
9
|
-
from dissect.cstruct.expression import Expression
|
|
10
9
|
from dissect.cstruct.util import classproperty, normalize_endianness
|
|
11
10
|
|
|
12
11
|
if TYPE_CHECKING:
|
|
@@ -15,6 +14,7 @@ if TYPE_CHECKING:
|
|
|
15
14
|
from typing_extensions import Self
|
|
16
15
|
|
|
17
16
|
from dissect.cstruct.cstruct import AllowedEndianness, Endianness, cstruct
|
|
17
|
+
from dissect.cstruct.expression import Expression
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
EOF = -0xE0F # Negative counts are illegal anyway, so abuse that for our EOF sentinel
|
|
@@ -38,22 +38,35 @@ class MetaType(type):
|
|
|
38
38
|
|
|
39
39
|
def __call__(cls, *args, **kwargs) -> Self: # type: ignore
|
|
40
40
|
"""Adds support for ``TypeClass(bytes | file-like object)`` parsing syntax."""
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
41
|
+
if len(args) == 1 and not kwargs:
|
|
42
|
+
# Fast path for the most common case without keyword arguments
|
|
43
|
+
obj = args[0]
|
|
44
|
+
if type(obj) in _BUFFER_TYPES:
|
|
45
|
+
if len(obj) == cls.__size__ and isinstance(obj, bytes) and issubclass(cls, bytes):
|
|
46
|
+
# Shortcut for char/bytes type
|
|
47
|
+
return type.__call__(cls, obj)
|
|
48
|
+
return cls._read(BytesIO(obj), endian=cls.__cs__.endian)
|
|
49
|
+
|
|
50
|
+
if hasattr(obj, "read") and not isinstance(obj, cls):
|
|
51
|
+
return cls._read(obj, endian=cls.__cs__.endian)
|
|
52
|
+
|
|
53
|
+
if not _is_buffer_type(obj):
|
|
54
|
+
# E.g. initializing with a value
|
|
55
|
+
return type.__call__(cls, obj)
|
|
56
|
+
|
|
57
|
+
if len(args) == 1 and not isinstance(obj := args[0], cls):
|
|
58
|
+
if type(obj) not in _BUFFER_TYPES and hasattr(obj, "read"):
|
|
46
59
|
endian = (
|
|
47
60
|
normalize_endianness(endian) if (endian := kwargs.get("endian")) is not None else cls.__cs__.endian
|
|
48
61
|
)
|
|
49
|
-
return cls._read(
|
|
62
|
+
return cls._read(obj, endian=endian)
|
|
50
63
|
|
|
51
|
-
if
|
|
64
|
+
if isinstance(obj, bytes) and len(obj) == cls.__size__ and issubclass(cls, bytes):
|
|
52
65
|
# Shortcut for char/bytes type
|
|
53
66
|
return type.__call__(cls, *args, **kwargs)
|
|
54
67
|
|
|
55
|
-
if _is_buffer_type(
|
|
56
|
-
return cls.reads(
|
|
68
|
+
if _is_buffer_type(obj):
|
|
69
|
+
return cls.reads(obj, endian=kwargs.get("endian"))
|
|
57
70
|
|
|
58
71
|
return type.__call__(cls, *args, **kwargs)
|
|
59
72
|
|
|
@@ -104,7 +117,7 @@ class MetaType(type):
|
|
|
104
117
|
if _is_buffer_type(obj):
|
|
105
118
|
return cls.reads(obj, endian=endian)
|
|
106
119
|
|
|
107
|
-
if not
|
|
120
|
+
if not hasattr(obj, "read"):
|
|
108
121
|
raise TypeError("Invalid object type")
|
|
109
122
|
|
|
110
123
|
endian = normalize_endianness(endian) if endian is not None else cls.__cs__.endian
|
|
@@ -167,13 +180,18 @@ class MetaType(type):
|
|
|
167
180
|
context: Optional reading context.
|
|
168
181
|
endian: The endianness to use when parsing (little, big, network, <, >, !, @ or =).
|
|
169
182
|
"""
|
|
183
|
+
result = []
|
|
170
184
|
if count == EOF:
|
|
171
|
-
result = []
|
|
172
185
|
while not _is_eof(stream):
|
|
173
186
|
result.append(cls._read(stream, context=context, endian=endian))
|
|
174
187
|
return result
|
|
175
188
|
|
|
176
|
-
|
|
189
|
+
# This is purposefully a simple loop
|
|
190
|
+
# PyPy will pre-allocate the list for a list comprehension ahead of time, which will cause huge allocations in
|
|
191
|
+
# case wrong count is read, instead of causing an EOFError when trying to read the type.
|
|
192
|
+
for _ in range(count):
|
|
193
|
+
result.append(cls._read(stream, context=context, endian=endian)) # noqa: PERF401
|
|
194
|
+
return result
|
|
177
195
|
|
|
178
196
|
def _read_0(cls, stream: BinaryIO, *, context: dict[str, Any] | None = None, endian: Endianness) -> list[Self]:
|
|
179
197
|
"""Internal function for reading null-terminated data.
|
|
@@ -329,19 +347,23 @@ class BaseArray(BaseType):
|
|
|
329
347
|
if cls.__null_terminated__:
|
|
330
348
|
return cls.__type__._read_0(stream, context=context, endian=endian)
|
|
331
349
|
|
|
350
|
+
return cls.__type__._read_array(stream, cls._num_entries(context), context=context, endian=endian)
|
|
351
|
+
|
|
352
|
+
@classmethod
|
|
353
|
+
def _num_entries(cls, context: dict[str, Any] | None) -> int:
|
|
354
|
+
"""Return the number of entries to read for this (non null-terminated) array type."""
|
|
332
355
|
if isinstance(cls.__num_entries__, int):
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
return cls.__type__._read_array(stream, num, context=context, endian=endian)
|
|
356
|
+
return max(0, cls.__num_entries__)
|
|
357
|
+
|
|
358
|
+
if cls.__num_entries__ is None:
|
|
359
|
+
return EOF
|
|
360
|
+
|
|
361
|
+
try:
|
|
362
|
+
return max(0, cls.__num_entries__.evaluate(cls.__cs__, context))
|
|
363
|
+
except Exception:
|
|
364
|
+
if cls.__num_entries__.expression != "EOF":
|
|
365
|
+
raise
|
|
366
|
+
return EOF
|
|
345
367
|
|
|
346
368
|
@classmethod
|
|
347
369
|
def _write(cls, stream: BinaryIO, data: list[Any], *, endian: Endianness) -> int:
|
|
@@ -389,15 +411,51 @@ class BaseArray(BaseType):
|
|
|
389
411
|
class Array(list[T], BaseArray):
|
|
390
412
|
@classmethod
|
|
391
413
|
def _read(cls, stream: BinaryIO, *, context: dict[str, Any] | None = None, endian: Endianness) -> list[T]:
|
|
392
|
-
return cls
|
|
414
|
+
return type.__call__(cls, super()._read(stream, context=context, endian=endian))
|
|
393
415
|
|
|
394
416
|
|
|
395
|
-
|
|
396
|
-
return hasattr(value, "read")
|
|
417
|
+
_BUFFER_TYPES = (bytes, memoryview, bytearray)
|
|
397
418
|
|
|
398
419
|
|
|
399
420
|
def _is_buffer_type(value: object) -> bool:
|
|
400
|
-
return isinstance(value,
|
|
421
|
+
return isinstance(value, _BUFFER_TYPES)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _is_seekable(stream: BinaryIO) -> bool:
|
|
425
|
+
try:
|
|
426
|
+
return stream.seekable()
|
|
427
|
+
except Exception:
|
|
428
|
+
return False
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _read_null_terminated(stream: BinaryIO, width: int) -> bytes:
|
|
432
|
+
"""Read data up to a null terminator of the given width from a seekable stream, in chunks.
|
|
433
|
+
|
|
434
|
+
Leaves the stream just after the terminator.
|
|
435
|
+
"""
|
|
436
|
+
terminator = b"\x00" * width
|
|
437
|
+
buf = []
|
|
438
|
+
while True:
|
|
439
|
+
chunk = stream.read(128 * width)
|
|
440
|
+
if not isinstance(chunk, (bytes, bytearray)):
|
|
441
|
+
# E.g. a memoryview, which doesn't support find()
|
|
442
|
+
chunk = bytes(chunk)
|
|
443
|
+
|
|
444
|
+
# Find a terminator that is aligned on an item
|
|
445
|
+
idx = chunk.find(terminator)
|
|
446
|
+
while idx != -1 and idx % width:
|
|
447
|
+
idx = chunk.find(terminator, idx + 1)
|
|
448
|
+
|
|
449
|
+
if idx != -1:
|
|
450
|
+
buf.append(chunk[:idx])
|
|
451
|
+
stream.seek(idx + width - len(chunk), SEEK_CUR)
|
|
452
|
+
return b"".join(buf)
|
|
453
|
+
|
|
454
|
+
if not chunk or len(chunk) % width:
|
|
455
|
+
# The end of the stream, possibly with a partial item
|
|
456
|
+
raise EOFError(f"Read {len(chunk) % width} bytes, but expected {width}")
|
|
457
|
+
|
|
458
|
+
buf.append(chunk)
|
|
401
459
|
|
|
402
460
|
|
|
403
461
|
def _is_eof(stream: BinaryIO) -> bool:
|
dissect/cstruct/types/char.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
from typing import TYPE_CHECKING, Any, BinaryIO
|
|
4
4
|
|
|
5
5
|
from dissect.cstruct.exception import ArraySizeError
|
|
6
|
-
from dissect.cstruct.types.base import EOF, BaseArray, BaseType
|
|
6
|
+
from dissect.cstruct.types.base import EOF, BaseArray, BaseType, _is_seekable, _read_null_terminated
|
|
7
7
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
9
|
from typing_extensions import Self
|
|
@@ -68,6 +68,9 @@ class Char(bytes, BaseType):
|
|
|
68
68
|
|
|
69
69
|
@classmethod
|
|
70
70
|
def _read_0(cls, stream: BinaryIO, *, context: dict[str, Any] | None = None, endian: Endianness) -> Self:
|
|
71
|
+
if _is_seekable(stream):
|
|
72
|
+
return type.__call__(cls, _read_null_terminated(stream, 1))
|
|
73
|
+
|
|
71
74
|
buf = []
|
|
72
75
|
while True:
|
|
73
76
|
byte = stream.read(1)
|
dissect/cstruct/types/enum.py
CHANGED
|
@@ -45,6 +45,10 @@ class EnumMetaType(EnumMeta, MetaType):
|
|
|
45
45
|
# value is a parsable value
|
|
46
46
|
value = cls.__type__(value)
|
|
47
47
|
|
|
48
|
+
# Fast path for known values
|
|
49
|
+
if type(value) is not cls and (member := cls._value2member_map_.get(value)) is not None:
|
|
50
|
+
return member
|
|
51
|
+
|
|
48
52
|
return super().__call__(value)
|
|
49
53
|
|
|
50
54
|
# We are constructing a new Enum class
|
|
@@ -61,6 +65,7 @@ class EnumMetaType(EnumMeta, MetaType):
|
|
|
61
65
|
enum_cls.__alignment__ = type_.__alignment__
|
|
62
66
|
|
|
63
67
|
_fix_alias_members(enum_cls)
|
|
68
|
+
_fix_flag_operators(enum_cls)
|
|
64
69
|
|
|
65
70
|
return enum_cls
|
|
66
71
|
|
|
@@ -125,6 +130,14 @@ class EnumMetaType(EnumMeta, MetaType):
|
|
|
125
130
|
return "\n".join(lines)
|
|
126
131
|
|
|
127
132
|
|
|
133
|
+
def _fix_flag_operators(cls: type[Enum]) -> None:
|
|
134
|
+
# Since Python 3.11, the standard library copies its Flag operators onto every new Flag class
|
|
135
|
+
# Remove those copies, so that the (faster) operators of our Flag class are used
|
|
136
|
+
for name in ("__and__", "__or__", "__xor__", "__rand__", "__ror__", "__rxor__"):
|
|
137
|
+
if name in cls.__dict__:
|
|
138
|
+
delattr(cls, name)
|
|
139
|
+
|
|
140
|
+
|
|
128
141
|
def _fix_alias_members(cls: type[Enum]) -> None:
|
|
129
142
|
# Emulate aenum NoAlias behaviour
|
|
130
143
|
# https://github.com/ethanfurman/aenum/blob/master/aenum/doc/aenum.rst
|
|
@@ -204,20 +217,20 @@ class Enum(BaseType, IntEnum, metaclass=EnumMetaType):
|
|
|
204
217
|
return f"{base}{value}"
|
|
205
218
|
|
|
206
219
|
def __eq__(self, other: int | Enum) -> bool:
|
|
207
|
-
if isinstance(other, Enum) and other.__class__ is not self.__class__:
|
|
208
|
-
return False
|
|
209
|
-
|
|
210
|
-
# Python <= 3.10 compatibility
|
|
211
220
|
if isinstance(other, Enum):
|
|
212
|
-
other
|
|
221
|
+
if other.__class__ is not self.__class__:
|
|
222
|
+
return False
|
|
223
|
+
|
|
224
|
+
# Python <= 3.10 compatibility
|
|
225
|
+
other = other._value_
|
|
213
226
|
|
|
214
|
-
return self.
|
|
227
|
+
return self._value_ == other
|
|
215
228
|
|
|
216
229
|
def __ne__(self, value: int | Enum) -> bool:
|
|
217
230
|
return not self.__eq__(value)
|
|
218
231
|
|
|
219
232
|
def __hash__(self) -> int:
|
|
220
|
-
return hash((self.__class__, self.
|
|
233
|
+
return hash((self.__class__, self._name_, self._value_))
|
|
221
234
|
|
|
222
235
|
@classmethod
|
|
223
236
|
def _missing_(cls, value: int) -> Self:
|
dissect/cstruct/types/flag.py
CHANGED
|
@@ -1,10 +1,34 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import operator
|
|
4
|
+
from enum import Flag as _StdFlag
|
|
3
5
|
from enum import IntFlag
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
4
7
|
|
|
5
8
|
from dissect.cstruct.types.base import BaseType
|
|
6
9
|
from dissect.cstruct.types.enum import PY_311, EnumMetaType
|
|
7
10
|
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _operator(op: Callable[[int, int], int], fallback: Callable[[Flag, int], Flag]) -> Callable[[Flag, int], Flag]:
|
|
16
|
+
"""Create a flag operator with a fast path for integers and flags of the same class."""
|
|
17
|
+
|
|
18
|
+
def method(self: Flag, other: int | Flag) -> Flag:
|
|
19
|
+
if type(other) is self.__class__:
|
|
20
|
+
value = op(self._value_, other._value_)
|
|
21
|
+
elif isinstance(other, int) and not isinstance(other, _StdFlag):
|
|
22
|
+
value = op(self._value_, other)
|
|
23
|
+
else:
|
|
24
|
+
return fallback(self, other)
|
|
25
|
+
|
|
26
|
+
if (member := self.__class__._value2member_map_.get(value)) is not None:
|
|
27
|
+
return member
|
|
28
|
+
return self.__class__(value)
|
|
29
|
+
|
|
30
|
+
return method
|
|
31
|
+
|
|
8
32
|
|
|
9
33
|
class Flag(BaseType, IntFlag, metaclass=EnumMetaType):
|
|
10
34
|
"""Flag type supercharged with cstruct functionality.
|
|
@@ -56,17 +80,26 @@ class Flag(BaseType, IntFlag, metaclass=EnumMetaType):
|
|
|
56
80
|
return result
|
|
57
81
|
|
|
58
82
|
def __eq__(self, other: int | Flag) -> bool:
|
|
59
|
-
if isinstance(other, Flag) and other.__class__ is not self.__class__:
|
|
60
|
-
return False
|
|
61
|
-
|
|
62
|
-
# Python <= 3.10 compatibility
|
|
63
83
|
if isinstance(other, Flag):
|
|
64
|
-
other
|
|
84
|
+
if other.__class__ is not self.__class__:
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
# Python <= 3.10 compatibility
|
|
88
|
+
other = other._value_
|
|
65
89
|
|
|
66
|
-
return self.
|
|
90
|
+
return self._value_ == other
|
|
67
91
|
|
|
68
92
|
def __ne__(self, value: int | Flag) -> bool:
|
|
69
93
|
return not self.__eq__(value)
|
|
70
94
|
|
|
71
95
|
def __hash__(self) -> int:
|
|
72
|
-
return hash((self.__class__, self.
|
|
96
|
+
return hash((self.__class__, self._name_, self._value_))
|
|
97
|
+
|
|
98
|
+
# Faster implementations of the standard library Flag operators
|
|
99
|
+
__and__ = _operator(operator.and_, IntFlag.__and__)
|
|
100
|
+
__or__ = _operator(operator.or_, IntFlag.__or__)
|
|
101
|
+
__xor__ = _operator(operator.xor, IntFlag.__xor__)
|
|
102
|
+
|
|
103
|
+
__rand__ = __and__
|
|
104
|
+
__ror__ = __or__
|
|
105
|
+
__rxor__ = __xor__
|