dissect.util 3.23.dev6__tar.gz → 3.23.dev8__tar.gz

{dissect_util-3.23.dev6 → dissect_util-3.23.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.util
-Version: 3.23.dev6
+Version: 3.23.dev8
 Summary: A Dissect module implementing various utility functions for the other Dissect modules
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0

dissect_util-3.23.dev8/dissect/util/ldap.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+import operator
+import re
+from enum import Enum
+
+from dissect.util.exceptions import Error
+
+
+class InvalidQueryError(Error):
+    pass
+
+
+class LogicalOperator(Enum):
+    AND = "&"
+    OR = "|"
+    NOT = "!"
+
+
+_LOGICAL_OPERATORS = tuple(op.value for op in LogicalOperator)
+
+
+class ComparisonOperator(Enum):
+    GE = ">="
+    LE = "<="
+    GT = ">"
+    LT = "<"
+    EQ = "="
+    APPROX = "~="
+    BIT = ":="
+    EXTENDED = ":"
+
+
+_NORMAL_COMPARISON_OPERATORS = [op for op in ComparisonOperator if op != ComparisonOperator.EXTENDED]
+_SORTED_COMPARISON_OPERATORS = sorted(_NORMAL_COMPARISON_OPERATORS, key=lambda op: len(op.value), reverse=True)
+
+_RE_EXTENDED = re.compile(r"(.+?):(.+?):=(.+)?")
+
+
+class SearchFilter:
+    """Represents an LDAP search filter (simple or nested).
+
+    Args:
+        query: The LDAP search filter string.
+    """
+
+    def __init__(self, query: str) -> None:
+        self.query: str = query
+
+        self.children: list[SearchFilter] = []
+        self.operator: LogicalOperator | ComparisonOperator | None = None
+        self.attribute: str | None = None
+        self.value: str | None = None
+        self._extended_rule: str | None = None
+
+        _validate_syntax(query)
+
+        if query[1:-1].startswith(_LOGICAL_OPERATORS):
+            self._parse_nested()
+        else:
+            self._parse_simple()
+
+    def __repr__(self) -> str:
+        if self.is_nested():
+            return f"<SearchFilter nested operator={self.operator.value!r} children={self.children}>"
+        return f"<SearchFilter attribute={self.attribute!r} operator={self.operator.value!r} value={self.value}>"
+
+    @classmethod
+    def parse(cls, query: str, optimize: bool = True) -> SearchFilter:
+        """Parse an LDAP query into a filter object, with optional optimization."""
+        result = cls(query)
+        if optimize:
+            return optimize_ldap_query(result)[0]
+        return result
+
+    def is_nested(self) -> bool:
+        """Return whether the filter is nested (i.e., contains logical operators and child filters)."""
+        return isinstance(self.operator, LogicalOperator)
+
+    def format(self) -> str:
+        """Format the search filter back into an LDAP query string."""
+        if self.is_nested():
+            childs = "".join([child.format() for child in self.children])
+            return f"({self.operator.value}{childs})"
+
+        if self.operator == ComparisonOperator.EXTENDED:
+            return f"({self.attribute}:{self._extended_rule}:={self.value})"
+
+        return f"({self.attribute}{self.operator.value}{self.value})"
+
+    def _parse_simple(self) -> None:
+        """Parse simple filter."""
+        query = self.query[1:-1]
+
+        # Check for extended matching rules first
+        if ":" in query and (match := _RE_EXTENDED.match(query)):
+            self.operator = ComparisonOperator.EXTENDED
+            self.attribute, self._extended_rule, self.value = match.groups()
+            return
+
+        # Regular operator parsing
+        test = query
+        operators: list[ComparisonOperator] = []
+        for op in _SORTED_COMPARISON_OPERATORS:
+            if op.value not in test:
+                continue
+
+            if test.count(op.value) > 1:
+                raise InvalidQueryError(f"Comparison operator {op.value} found multiple times in query: {self.query}")
+
+            operators.append(op)
+            test = test.replace(op.value, "")
+
+        if len(operators) == 0:
+            raise InvalidQueryError(
+                f"No comparison operator found in query: {self.query}. "
+                f"Expected one of {[op.value for op in _NORMAL_COMPARISON_OPERATORS]}."
+            )
+
+        if len(operators) > 1:
+            raise InvalidQueryError(
+                f"Multiple comparison operators found in query: {self.query} -> {[o.value for o in operators]} "
+                f"Expected only one of {[op.value for op in _NORMAL_COMPARISON_OPERATORS]}."
+            )
+
+        self.operator = operators[0]
+        self.attribute, _, self.value = query.partition(self.operator.value)
+
+    def _parse_nested(self) -> None:
+        """Parse nested filter."""
+        query = self.query[1:-1]
+        self.operator = LogicalOperator(query[0])
+
+        start = 1
+        while start < len(query):
+            end = start + 1
+            depth = 1
+
+            while end < len(query) and depth > 0:
+                if query[end] == "(":
+                    depth += 1
+                elif query[end] == ")":
+                    depth -= 1
+                end += 1
+
+            self.children.append(SearchFilter(query[start:end]))
+            start = end
+
+
+_ATTRIBUTE_WEIGHTS = {
+    "objectGUID": 1,
+    "distinguishedName": 1,
+    "sAMAccountName": 2,
+    "userPrincipalName": 2,
+    "mail": 2,
+    "sAMAccountType": 3,
+    "servicePrincipalName": 3,
+    "userAccountControl": 4,
+    "memberOf": 5,
+    "member": 5,
+    "pwdLastSet": 5,
+    "primaryGroupID": 6,
+    "whenCreated": 6,
+    "ou": 6,
+    "lastLogonTimestamp": 6,
+    "cn": 7,
+    "givenName": 7,
+    "name": 7,
+    "telephoneNumber": 7,
+    "objectCategory": 8,
+    "description": 9,
+    "objectClass": 10,
+}
+
+
+def optimize_ldap_query(query: SearchFilter) -> tuple[SearchFilter, int]:
+    """Optimize an LDAP query in-place.
+
+    Removes redundant conditions and sorts filters and conditions based on how specific they are.
+
+    Args:
+        query: The LDAP query to optimize.
+
+    Returns:
+        A tuple containing the optimized LDAP query and its weight.
+    """
+    # Simplify single-child AND/OR
+    if query.is_nested() and len(query.children) == 1 and query.operator in (LogicalOperator.AND, LogicalOperator.OR):
+        return optimize_ldap_query(query.children[0])
+
+    # Sort nested children by weight
+    if query.is_nested() and len(query.children) > 1:
+        children = sorted((optimize_ldap_query(child) for child in query.children), key=operator.itemgetter(1))
+
+        query.children = [child for child, _ in children]
+        query.query = query.format()
+
+        return query, max(weight for _, weight in children)
+
+    # Handle NOT
+    if query.is_nested() and len(query.children) == 1 and query.operator == LogicalOperator.NOT:
+        child, weight = optimize_ldap_query(query.children[0])
+
+        query.children[0] = child
+        query.query = query.format()
+
+        return query, weight
+
+    # Base case: simple filter
+    if not query.is_nested():
+        return query, _ATTRIBUTE_WEIGHTS.get(query.attribute, max(_ATTRIBUTE_WEIGHTS.values()))
+
+    return query, max(_ATTRIBUTE_WEIGHTS.values())
+
+
+def _validate_syntax(query: str) -> None:
+    """Validate basic LDAP query syntax.
+
+    Args:
+        query: The LDAP query to validate.
+    """
+    if not query:
+        raise InvalidQueryError("Empty query")
+
+    if not query.startswith("(") or not query.endswith(")"):
+        raise InvalidQueryError(f"Query must be wrapped in parentheses: {query}")
+
+    if query.count("(") != query.count(")"):
+        raise InvalidQueryError(f"Unbalanced parentheses in query: {query}")
+
+    # Check for empty parentheses
+    if "()" in query:
+        raise InvalidQueryError(f"Empty parentheses found in query: {query}")
+
+    # Check for queries that start with double opening parentheses
+    if query.startswith("(("):
+        raise InvalidQueryError(f"Invalid query structure: {query}")

{dissect_util-3.23.dev6 → dissect_util-3.23.dev8}/dissect.util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.util
-Version: 3.23.dev6
+Version: 3.23.dev8
 Summary: A Dissect module implementing various utility functions for the other Dissect modules
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0

{dissect_util-3.23.dev6 → dissect_util-3.23.dev8}/dissect.util.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ dissect/util/_build.py
 dissect/util/cpio.py
 dissect/util/exceptions.py
 dissect/util/feature.py
+dissect/util/ldap.py
 dissect/util/plist.py
 dissect/util/sid.py
 dissect/util/stream.py
@@ -54,10 +55,10 @@ dissect/util/tools/__init__.py
 dissect/util/tools/dump_nskeyedarchiver.py
 tests/__init__.py
 tests/conftest.py
-tests/test_compression.py
 tests/test_cpio.py
 tests/test_feature.py
 tests/test_hash.py
+tests/test_ldap.py
 tests/test_plist.py
 tests/test_sid.py
 tests/test_stream.py
@@ -71,4 +72,15 @@ tests/_data/newc.cpio.gz
 tests/_data/odc.cpio.gz
 tests/_docs/Makefile
 tests/_docs/conf.py
-tests/_docs/index.rst
+tests/_docs/index.rst
+tests/compression/__init__.py
+tests/compression/test_lz4.py
+tests/compression/test_lzbitmap.py
+tests/compression/test_lzfse.py
+tests/compression/test_lznt1.py
+tests/compression/test_lzo.py
+tests/compression/test_lzvn.py
+tests/compression/test_lzxpress.py
+tests/compression/test_lzxpress_huffman.py
+tests/compression/test_sevenbit.py
+tests/compression/test_xz.py

dissect_util-3.23.dev8/tests/compression/test_lz4.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+
+@pytest.mark.parametrize(
+    ("data", "digest"),
+    [
+        pytest.param(
+            "ff0c4c5a3420636f6d7072657373696f6e207465737420737472696e671b00db507472696e67",
+            "5ec59b1b60247178b260e145ae888c318e1fa3ee4466cf0e145de8f24c4b1501",
+            id="basic",
+        ),
+        pytest.param(
+            "ffffa94c6f72656d20697073756d20646f6c6f722073697420616d657420636f"
+            "6e73656374657475722061646970697363696e6720656c69742e205175697371"
+            "75652066617563696275732065782073617069656e2076697461652070656c6c"
+            "656e7465737175652073656d20706c6163657261742e20496e20696420637572"
+            "737573206d69207072657469756d2074656c6c7573206475697320636f6e7661"
+            "6c6c69732e2054656d707573206c656f2065752061656e65616e207365642064"
+            "69616d2075726e612074656d706f722e2050756c76696e617220766976616d75"
+            "73206672696e67696c6c61206c61637573206e6563206d657475732062696265"
+            "6e64756d20656765737461732e20496163756c6973206d61737361206e69736c"
+            "206d616c657375616461206c6163696e696120696e7465676572206e756e6320"
+            "706f73756572652e2055742068656e6472657269742073656d7065722076656c"
+            "20636c61737320617074656e742074616369746920736f63696f7371752e2041"
+            "64206c69746f726120746f727175656e742070657220636f6e75626961206e6f"
+            "7374726120696e636570746f732068696d656e61656f732e0a0ab701ffffffff"
+            "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+            "ffffffffffffffffffffffffffffffffff4550656f732e0a",
+            "73d3dd96ca2e2f0144a117019256d770ee7c6febeaee09b24956c723ae22b529",
+            id="large",
+        ),
+    ],
+)
+def test_lz4_decompress(lz4: ModuleType, data: str, digest: str) -> None:
+    assert hashlib.sha256(lz4.decompress(bytes.fromhex(data))).hexdigest() == digest
+
+
+@pytest.mark.benchmark
+def test_benchmark_lz4_decompress(lz4: ModuleType, benchmark: BenchmarkFixture) -> None:
+    buf = bytes.fromhex("ff0c4c5a3420636f6d7072657373696f6e207465737420737472696e671b00db507472696e67")
+    assert benchmark(lz4.decompress, buf) == b"LZ4 compression test string" * 10
+
+
+@pytest.mark.benchmark
+def test_benchmark_large_lz4_decompress(lz4: ModuleType, benchmark: BenchmarkFixture) -> None:
+    buf = bytes.fromhex(
+        "ffffa94c6f72656d20697073756d20646f6c6f722073697420616d657420636f"
+        "6e73656374657475722061646970697363696e6720656c69742e205175697371"
+        "75652066617563696275732065782073617069656e2076697461652070656c6c"
+        "656e7465737175652073656d20706c6163657261742e20496e20696420637572"
+        "737573206d69207072657469756d2074656c6c7573206475697320636f6e7661"
+        "6c6c69732e2054656d707573206c656f2065752061656e65616e207365642064"
+        "69616d2075726e612074656d706f722e2050756c76696e617220766976616d75"
+        "73206672696e67696c6c61206c61637573206e6563206d657475732062696265"
+        "6e64756d20656765737461732e20496163756c6973206d61737361206e69736c"
+        "206d616c657375616461206c6163696e696120696e7465676572206e756e6320"
+        "706f73756572652e2055742068656e6472657269742073656d7065722076656c"
+        "20636c61737320617074656e742074616369746920736f63696f7371752e2041"
+        "64206c69746f726120746f727175656e742070657220636f6e75626961206e6f"
+        "7374726120696e636570746f732068696d656e61656f732e0a0ab701ffffffff"
+        "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+        "ffffffffffffffffffffffffffffffffff4550656f732e0a"
+    )
+    assert (
+        hashlib.sha256(benchmark(lz4.decompress, buf)).hexdigest()
+        == "73d3dd96ca2e2f0144a117019256d770ee7c6febeaee09b24956c723ae22b529"
+    )

dissect_util-3.23.dev8/tests/compression/test_lzbitmap.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING
+
+import pytest
+
+from dissect.util.compression import lzbitmap
+
+if TYPE_CHECKING:
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+
+@pytest.mark.parametrize(
+    ("data", "digest"),
+    [
+        pytest.param(
+            "5a424d093100002b0000536d616c6c2066696c657320646f6e2774206765"
+            "7420636f6d7072657373656420617420616c6c2e2e2e0a060000000000",
+            "8c929efb5fd28b5b82385b67b408f5e775a4756d7cc6373eebddb8668343ad40",
+            id="uncompressed",
+        ),
+        pytest.param(
+            "5a424d092d0000a0000018000018000018000061616161616161617835ef"
+            "340f0000f10f00000000000000000000000000060000000000",
+            "ef56118ff333a8bfeffc346c4987a1a178762570b3eb1d704a2c1e9b3a877561",
+            id="small",
+        ),
+        pytest.param(
+            "5a424d09d80100df36002301005001007601004c6f72656d20697073756d"
+            "20646f6c6f722073742061657420636e7365636574757220616469706973"
+            "696e67656c69742e517573716566617563627520657820736170656e7661"
+            "6570656c6c6e6573656d6c61637261496e20696475727573206d69707269"
+            "74656c757364206f76616c7354656d7075206c656f7561656e6561646469"
+            "6d206e2074656d706f72506c6e7276697675736672696c61206c636e636d"
+            "657473626962656e64756d67657461732e496c696d6173206e6c6d616c75"
+            "64616c696e616e746567726e6e63707365722e5574206864726572697473"
+            "6d7072766c637373207074656e74747469736f6f73712e41646c696f7261"
+            "6f7175657465727562696f74726370746f686d6f732e0a0a4c0d2c341b41"
+            "3e26113c5f6e805b6b7c65529d967ec4b310edb922ca7b1deca5faf4434a"
+            "fbfa52fb8272b2ffb7016f7fcbc1b9f1373af99e3eb4e94fa9b3bafe39d3"
+            "6959add6f36b55eecdb59d2ec3d1d029fc0055a9cbed016111718114103f"
+            "480147110116011040f125f3ffffffffffffffffffffffffffffffffffff"
+            "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+            "ffffffffffffffff1f002cf68f57d9c576d73bfd7cd3d756000006000000"
+            "0000",
+            "73d3dd96ca2e2f0144a117019256d770ee7c6febeaee09b24956c723ae22b529",
+            id="large",
+        ),
+    ],
+)
+def test_lzbitmap_decompress(data: str, digest: str) -> None:
+    assert hashlib.sha256(lzbitmap.decompress(bytes.fromhex(data))).hexdigest() == digest
+
+
+@pytest.mark.benchmark
+def test_benchmark_lzbitmap_decompress(benchmark: BenchmarkFixture) -> None:
+    buf = bytes.fromhex(
+        "5a424d092d0000a0000018000018000018000061616161616161617835ef"
+        "340f0000f10f00000000000000000000000000060000000000",
+    )
+    assert benchmark(lzbitmap.decompress, buf) == b"a" * 158 + b"xa"
+
+
+@pytest.mark.benchmark
+def test_benchmark_large_lzbitmap_decompress(benchmark: BenchmarkFixture) -> None:
+    buf = bytes.fromhex(
+        "5a424d09d80100df36002301005001007601004c6f72656d20697073756d"
+        "20646f6c6f722073742061657420636e7365636574757220616469706973"
+        "696e67656c69742e517573716566617563627520657820736170656e7661"
+        "6570656c6c6e6573656d6c61637261496e20696475727573206d69707269"
+        "74656c757364206f76616c7354656d7075206c656f7561656e6561646469"
+        "6d206e2074656d706f72506c6e7276697675736672696c61206c636e636d"
+        "657473626962656e64756d67657461732e496c696d6173206e6c6d616c75"
+        "64616c696e616e746567726e6e63707365722e5574206864726572697473"
+        "6d7072766c637373207074656e74747469736f6f73712e41646c696f7261"
+        "6f7175657465727562696f74726370746f686d6f732e0a0a4c0d2c341b41"
+        "3e26113c5f6e805b6b7c65529d967ec4b310edb922ca7b1deca5faf4434a"
+        "fbfa52fb8272b2ffb7016f7fcbc1b9f1373af99e3eb4e94fa9b3bafe39d3"
+        "6959add6f36b55eecdb59d2ec3d1d029fc0055a9cbed016111718114103f"
+        "480147110116011040f125f3ffffffffffffffffffffffffffffffffffff"
+        "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+        "ffffffffffffffff1f002cf68f57d9c576d73bfd7cd3d756000006000000"
+        "0000",
+    )
+    assert (
+        hashlib.sha256(benchmark(lzbitmap.decompress, buf)).hexdigest()
+        == "73d3dd96ca2e2f0144a117019256d770ee7c6febeaee09b24956c723ae22b529"
+    )

dissect_util-3.23.dev8/tests/compression/test_lzfse.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING
+
+import pytest
+
+from dissect.util.compression import lzfse
+
+if TYPE_CHECKING:
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+
+@pytest.mark.parametrize(
+    ("data", "digest"),
+    [
+        pytest.param(
+            "6276786e2c01000013000000c803616263f0fff005e163060000000000000062767824",
+            "d9f5aeb06abebb3be3f38adec9a2e3b94228d52193be923eb4e24c9b56ee0930",
+            id="basic",
+        ),
+        pytest.param(
+            "62767832df360000a401200e000d0030b92c8bef56220070a50000003984"
+            "70085f00b0000000383e59c0f1090000005fc027710c0000fc031c1f03c7"
+            "700cc70000000000000000006c00003600000000005f07000000e7000000"
+            "00100060418004120000e06338f9061e8067629dacdf013e836fe041f807"
+            "be80fd14fc0e3fc29f704700000000000000000000000000000000000000"
+            "000000000000000000000000000000000000101288d24318d9f446277ac6"
+            "885a4b5dea360854e4c4616262d667f9f1ff53187e598fe2f5ddf7b768f4"
+            "bcc1f6441b9e55e0d1be84b4b91544337f11c4d0d615068c79817f5d19f2"
+            "09c83975cf9669b7f3d1024d9cc795e8ac449090696a7660585fac1a891c"
+            "40557bb46c1b62a35ab2608d574e82ba9f3956d0f811370c78d69b24240f"
+            "fd80ec4eccb6dc1e7f1c6f2f276a71e9c73183844c3dce83088eeed6c77c"
+            "3e35316f414db430fcd2e22d0c07998d601addd5907f852df080386fe69e"
+            "b78675198704b4bf5361caaf482e9333c6de0d46fbf87b4387fc6ac57116"
+            "0300000000000000000066b7fffffff3fffa3ff7ff1fd2273e1f85c5f04f"
+            "0f4945ab8462767824",
+            "73d3dd96ca2e2f0144a117019256d770ee7c6febeaee09b24956c723ae22b529",
+            id="large",
+        ),
+    ],
+)
+def test_lzfse_decompress(data: str, digest: str) -> None:
+    assert hashlib.sha256(lzfse.decompress(bytes.fromhex(data))).hexdigest() == digest
+
+
+@pytest.mark.benchmark
+def test_benchmark_lzfse_decompress(benchmark: BenchmarkFixture) -> None:
+    buf = bytes.fromhex("6276786e2c01000013000000c803616263f0fff005e163060000000000000062767824")
+    assert benchmark(lzfse.decompress, buf) == b"abc" * 100
+
+
+@pytest.mark.benchmark
+def test_benchmark_large_lzfse_decompress(benchmark: BenchmarkFixture) -> None:
+    buf = bytes.fromhex(
+        "62767832df360000a401200e000d0030b92c8bef56220070a50000003984"
+        "70085f00b0000000383e59c0f1090000005fc027710c0000fc031c1f03c7"
+        "700cc70000000000000000006c00003600000000005f07000000e7000000"
+        "00100060418004120000e06338f9061e8067629dacdf013e836fe041f807"
+        "be80fd14fc0e3fc29f704700000000000000000000000000000000000000"
+        "000000000000000000000000000000000000101288d24318d9f446277ac6"
+        "885a4b5dea360854e4c4616262d667f9f1ff53187e598fe2f5ddf7b768f4"
+        "bcc1f6441b9e55e0d1be84b4b91544337f11c4d0d615068c79817f5d19f2"
+        "09c83975cf9669b7f3d1024d9cc795e8ac449090696a7660585fac1a891c"
+        "40557bb46c1b62a35ab2608d574e82ba9f3956d0f811370c78d69b24240f"
+        "fd80ec4eccb6dc1e7f1c6f2f276a71e9c73183844c3dce83088eeed6c77c"
+        "3e35316f414db430fcd2e22d0c07998d601addd5907f852df080386fe69e"
+        "b78675198704b4bf5361caaf482e9333c6de0d46fbf87b4387fc6ac57116"
+        "0300000000000000000066b7fffffff3fffa3ff7ff1fd2273e1f85c5f04f"
+        "0f4945ab8462767824"
+    )
+    assert (
+        hashlib.sha256(benchmark(lzfse.decompress, buf)).hexdigest()
+        == "73d3dd96ca2e2f0144a117019256d770ee7c6febeaee09b24956c723ae22b529"
+    )