dissect.util 3.24.dev2__cp314-cp314t-manylinux_2_28_aarch64.whl

dissect/util/__init__.py

@@ -0,0 +1,20 @@
+from dissect.util.compression import (
+    lz4,
+    lznt1,
+    lzo,
+    lzxpress,
+    lzxpress_huffman,
+    sevenbit,
+)
+from dissect.util.hash import crc32c, jenkins
+
+__all__ = [
+    "crc32c",
+    "jenkins",
+    "lz4",
+    "lznt1",
+    "lzo",
+    "lzxpress",
+    "lzxpress_huffman",
+    "sevenbit",
+]

dissect/util/_build.py

@@ -0,0 +1,17 @@
+# Reference: https://setuptools.pypa.io/en/latest/build_meta.html#dynamic-build-dependencies-and-other-build-meta-tweaks
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import setuptools.build_meta
+from setuptools.build_meta import *  # noqa: F403
+
+
+def get_requires_for_build_wheel(config_settings: dict[str, Any] | None = None) -> list[str]:
+    base = setuptools.build_meta.get_requires_for_build_wheel(config_settings)
+
+    if os.getenv("BUILD_RUST", "").lower() in {"1", "true"} or (config_settings and "--build-rust" in config_settings):
+        return [*base, "setuptools-rust"]
+
+    return base

dissect/util/_native/__init__.pyi

@@ -0,0 +1,3 @@
+from dissect.util._native import compression, hash
+
+__all__ = ["compression", "hash"]

dissect/util/_native/compression/__init__.pyi

@@ -0,0 +1,3 @@
+from dissect.util._native.compression import lz4, lzo
+
+__all__ = ["lz4", "lzo"]

dissect/util/_native/compression/lz4.pyi

@@ -0,0 +1,7 @@
+from typing import BinaryIO
+
+def decompress(
+    src: bytes | BinaryIO,
+    uncompressed_size: int = -1,
+    return_bytearray: bool = False,
+) -> bytes | tuple[bytes, int]: ...

dissect/util/_native/compression/lzo.pyi

@@ -0,0 +1,3 @@
+from typing import BinaryIO
+
+def decompress(src: bytes | BinaryIO, header: bool = True, buflen: int = -1) -> bytes: ...

dissect/util/_native/hash/__init__.py

@@ -0,0 +1,3 @@
+from dissect.util._native.hash import crc32c
+
+__all__ = ["crc32c"]

dissect/util/_native/hash/crc32c.py

@@ -0,0 +1,2 @@
+def update(crc: int, data: bytes) -> int: ...
+def crc32c(data: bytes, value: int = 0) -> int: ...

dissect/util/compression/__init__.py

@@ -0,0 +1,45 @@
+from dissect.util.compression import lz4, lzo
+
+lz4_python = lz4
+lzo_python = lzo
+
+# This selects between a native Rust version of lz4 and lzo (when available) and our own
+# pure-Python implementation.
+#
+# By doing a:
+#  from dissect.util.compression import lz4
+# or
+#  from dissect.util.compression import lzo
+#
+# in another project will automatically give you one or the other.
+#
+# The native Rust version is also available as dissect.util.compression.lz4_native
+# and dissect.util.compression.lzo_native (when available) and the pure Python
+# version is always available as dissect.util.compression.lz4_python and
+# dissect.util.compression.lzo_python.
+#
+# Note that the pure Python implementation and the Rust implementation are NOT a full replacement
+# for the "official" lz4 and lzo Python packages: only the decompress() function is implemented.
+try:
+    from dissect.util import _native
+
+    lz4 = lz4_native = _native.compression.lz4
+    lzo = lzo_native = _native.compression.lzo
+except (ImportError, AttributeError):
+    lz4_native = lzo_native = None
+
+__all__ = [
+    "lz4",
+    "lz4_native",
+    "lz4_python",
+    "lzbitmap",
+    "lzfse",
+    "lznt1",
+    "lzo",
+    "lzo_native",
+    "lzo_python",
+    "lzvn",
+    "lzxpress",
+    "lzxpress_huffman",
+    "sevenbit",
+]

dissect/util/compression/lz4.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import io
+import struct
+from typing import BinaryIO
+
+from dissect.util.exceptions import CorruptDataError
+
+
+def _get_length(src: BinaryIO, length: int) -> int:
+    if length != 0xF:
+        return length
+
+    while True:
+        read_buf = src.read(1)
+        if len(read_buf) != 1:
+            raise CorruptDataError("EOF at length read")
+        len_part = ord(read_buf)
+        length += len_part
+
+        if len_part != 0xFF:
+            break
+
+    return length
+
+
+def decompress(
+    src: bytes | BinaryIO,
+    uncompressed_size: int = -1,
+    return_bytearray: bool = False,
+) -> bytes | tuple[bytes, int]:
+    """LZ4 decompress from a file-like object or bytes up to a certain length. Assumes no header.
+
+    Args:
+        src: File-like object or bytes to decompress from.
+        uncompressed_size: Ignored, present for compatibility with native lz4.
+        return_bytearray: Whether to return ``bytearray`` or ``bytes``.
+
+    Returns:
+        The decompressed data.
+    """
+    if not hasattr(src, "read"):
+        src = io.BytesIO(src)
+
+    dst = bytearray()
+    min_match_len = 4
+
+    while True:
+        if len(read_buf := src.read(1)) == 0:
+            raise CorruptDataError("EOF at reading literal-len")
+
+        token = ord(read_buf)
+        literal_len = _get_length(src, (token >> 4) & 0xF)
+
+        if len(dst) + literal_len > uncompressed_size > 0:
+            raise CorruptDataError("Decompressed size exceeds uncompressed_size")
+
+        if len(read_buf := src.read(literal_len)) != literal_len:
+            raise CorruptDataError("Not literal data")
+
+        dst.extend(read_buf)
+        if len(dst) >= uncompressed_size > 0:
+            break
+
+        if len(read_buf := src.read(2)) == 0:
+            token_and = token & 0xF
+            if token_and != 0:
+                raise CorruptDataError(f"EOF, but match-len > 0: {token_and}")
+            break
+
+        if len(read_buf) != 2:
+            raise CorruptDataError("Premature EOF")
+
+        if (offset := struct.unpack("<H", read_buf)[0]) == 0:
+            raise CorruptDataError("Offset can't be 0")
+
+        match_len = _get_length(src, (token >> 0) & 0xF)
+        match_len += min_match_len
+
+        if len(dst) + match_len > uncompressed_size > 0:
+            raise CorruptDataError("Decompressed size exceeds uncompressed_size")
+
+        remaining = match_len
+        while remaining > 0:
+            match_size = min(remaining, offset)
+            dst += dst[-offset : (-offset + match_size) or None]
+            remaining -= match_size
+
+        if len(dst) >= uncompressed_size > 0:
+            break
+
+    if not return_bytearray:
+        dst = bytes(dst)
+
+    return dst[:uncompressed_size] if uncompressed_size > 0 else dst

dissect/util/compression/lzbitmap.py

@@ -0,0 +1,130 @@
+# References:
+# - https://github.com/eafer/libzbitmap
+# - https://github.com/sgan81/apfs-fuse
+from __future__ import annotations
+
+import io
+import itertools
+import struct
+from typing import BinaryIO
+
+_H = struct.Struct("<H")
+
+
+def decompress(src: bytes | BinaryIO) -> bytes:
+    """LZBITMAP decompress from a file-like object or bytes.
+
+    Decompresses until EOF or EOS of the input data.
+
+    Args:
+        src: File-like object or bytes to decompress.
+
+    Returns:
+        The decompressed data.
+    """
+    if not hasattr(src, "read"):
+        src = io.BytesIO(src)
+
+    if src.read(4) != b"ZBM\x09":
+        raise ValueError("Not a valid LZBITMAP stream")
+
+    dst = bytearray()
+
+    while True:
+        compressed_size = int.from_bytes(src.read(3), "little")
+        uncompressed_size = int.from_bytes(src.read(3), "little")
+
+        if compressed_size == uncompressed_size + 6:  # chunk header size
+            # Not compressed
+            dst += src.read(uncompressed_size)
+
+        elif uncompressed_size == 0:
+            # End of stream
+            break
+
+        else:
+            # Compressed
+            distance_offset = int.from_bytes(src.read(3), "little")
+            bitmap_offset = int.from_bytes(src.read(3), "little")
+            token_offset = int.from_bytes(src.read(3), "little")
+            literal_offset = 15
+
+            # Buffer the whole chunk
+            src.seek(-15, io.SEEK_CUR)
+            buf = memoryview(src.read(compressed_size))
+
+            # Build the bitmap/token map
+            token_map = []
+            bits = int.from_bytes(buf[-17:], "little")
+            for i in range(0xF):
+                if i < 3:
+                    token_map.append((None, i))
+                else:
+                    token_map.append((bits & 0xFF, (bits >> 8) & 3))
+                    bits >>= 10
+
+            # Tokens are stored as nibbles, so we need to split each byte into two
+            tokens = itertools.chain.from_iterable((b & 0xF, (b >> 4) & 0xF) for b in buf[token_offset:-17])
+
+            # Initial match distance is 8, and is not reset between tokens
+            distance = 8
+
+            prev_token = None
+            while uncompressed_size > 0:
+                if (idx := next(tokens, None) if prev_token is None else prev_token) is None:
+                    break
+
+                if idx == 0xF:
+                    # 0xF indicates a repeat count
+                    raise ValueError("Invalid token index in LZBITMAP stream")
+
+                if (repeat := next(tokens)) != 0xF:
+                    # No repeat count, store the token for next iteration
+                    prev_token = repeat
+                    repeat = 1
+                else:
+                    # Repeat count, read and sum
+                    prev_token = None
+                    total = 4
+                    while repeat == 0xF:
+                        repeat = next(tokens)
+                        total += repeat
+
+                        if total < repeat:
+                            raise ValueError("Invalid repeat count in LZBITMAP stream")
+
+                    repeat = total
+
+                for _ in range(repeat):
+                    bitmap, token = token_map[idx]
+                    if idx < 3:
+                        # Index 0, 1, 2 are special and indicate we need to read a bitmap from the bitmap region
+                        bitmap = buf[bitmap_offset]
+                        bitmap_offset += 1
+
+                    if token == 1:
+                        # 1-byte distance
+                        distance = buf[distance_offset]
+                        distance_offset += 1
+                    elif token == 2:
+                        # 2-byte distance
+                        (distance,) = _H.unpack_from(buf, distance_offset)
+                        distance_offset += 2
+
+                    for _ in range(8):
+                        if bitmap & 1:
+                            # Literal
+                            dst.append(buf[literal_offset])
+                            literal_offset += 1
+                        else:
+                            # Match
+                            if distance > len(dst):
+                                raise ValueError("Invalid match distance in LZBITMAP stream")
+                            dst.append(dst[-distance])
+
+                        bitmap >>= 1
+                        uncompressed_size -= 1
+                        if uncompressed_size == 0:
+                            break
+
+    return bytes(dst)