ntcompress 0.3.0__py3-none-any.whl

ntcompress/__init__.py

@@ -0,0 +1,20 @@
+"""Pure-Python codecs for Microsoft ESE record-compression and Windows ntdll compression formats.
+
+``ntcompress`` compresses and decompresses all ESE record schemes (7-bit ASCII/Unicode,
+XPRESS, SCRUB, XPRESS9, XPRESS10, LZ4) and all ntdll ``RtlCompressBuffer`` formats
+(LZNT1, XPRESS, XPRESS_HUFF, Compact XPRESS9, XP10, DEFLATE, ZLIB). Compress output is
+verified byte-identical to ``esent.dll`` and ``ntdll.dll`` across 16 Windows builds. No
+runtime dependencies beyond the standard library.
+
+ESE codecs live under :mod:`ntcompress.ese`; ntdll standalone codecs under
+:mod:`ntcompress.ntdll`.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("ntcompress")
+except PackageNotFoundError:  # pragma: no cover - source checkout without install metadata
+    __version__ = "0.0.0"

ntcompress/_types.py

@@ -0,0 +1,8 @@
+"""Shared type aliases used across both the ESE and ntdll subpackages."""
+
+from __future__ import annotations
+
+from typing import TypeAlias
+
+Buffer: TypeAlias = bytes | bytearray | memoryview
+"""Any read-only bytes-like input accepted by the codecs."""

ntcompress/ese/__init__.py

@@ -0,0 +1,216 @@
+"""ESE (Extensible Storage Engine) record-compression formats.
+
+Provides both Shape A (enum dispatch) and Shape B (direct module import) APIs for
+every ESE record-compression scheme. The ``Format`` enum values are the 5-bit scheme
+IDs extracted from the first byte of a compressed ESE cell, matching the
+``CDataCompressor::COMPRESSION_SCHEME`` constants in the MIT ESE source.
+
+Shape A (enum dispatch)::
+
+    import ntcompress.ese
+    compressed = ntcompress.ese.compress(data, ntcompress.ese.Format.XPRESS)
+    plain = ntcompress.ese.decompress(compressed)  # auto-detects format
+
+Shape B (direct module)::
+
+    from ntcompress.ese import xpress
+    compressed = xpress.compress(data)
+    plain = xpress.decompress(compressed)
+"""
+
+from __future__ import annotations
+
+from enum import IntEnum
+from typing import TYPE_CHECKING, Final
+
+from ntcompress.ese._registry import _get, _register
+from ntcompress.exceptions import (
+    CompressionError,
+    DecompressionError,
+    FormatUnavailableError,
+    ScrubDetectedError,
+)
+
+if TYPE_CHECKING:
+    from ntcompress._types import Buffer
+
+
+# --- Header bit layout ---
+
+_SCHEME_SHIFT: Final = 3
+"""Bit offset of the 5-bit format id within the header byte (compression.cxx:2003)."""
+
+_FLAG_MASK: Final = 0b0000_0111
+"""Mask selecting the low 3 format-specific flag bits of the header byte."""
+
+
+class Format(IntEnum):
+    """ESE record-compression format identifiers.
+
+    Values are the 5-bit scheme IDs extracted from the first byte of a compressed
+    ESE cell. See ``CDataCompressor::COMPRESSION_SCHEME`` in the MIT-licensed ESE
+    source (``compression.cxx:504-512``).
+    """
+
+    NONE = 0x00
+    """Sentinel -- uncompressed cell, no compression header."""
+
+    SEVEN_BIT_ASCII = 0x01
+    """7-bit ASCII packing (compression.cxx:1168-1387)."""
+
+    SEVEN_BIT_UNICODE = 0x02
+    """7-bit Unicode packing over UTF-16LE code units (compression.cxx:1390-1504)."""
+
+    XPRESS = 0x03
+    """Plain LZ77 ([MS-XCA] §2.1) with 3-byte ESE frame (compression.cxx:1507-1568)."""
+
+    SCRUB = 0x04
+    """Erase marker -- not a compression format. Use :mod:`ntcompress.ese.scrub`."""
+
+    XPRESS9 = 0x05
+    """LZ77+Huffman9 with ESE frame (compression.cxx:1686-1759)."""
+
+    XPRESS10 = 0x06
+    """LZ4 block + CRC-32C/CRC-64 integrity (compression.cxx:1935-2064)."""
+
+    LZ4 = 0x07
+    """Raw LZ4 block with 3-byte ESE frame (compression.cxx:2070-2109)."""
+
+    MAXIMUM = 0x1F
+    """Sentinel -- upper bound of the 5-bit scheme space, not a real format."""
+
+
+# --- First-byte helpers ---
+
+
+def format_id(first_byte: int) -> int:
+    """Extract the 5-bit format ID from an ESE cell's header byte.
+
+    Returns ``first_byte >> 3`` (0-31). Callers map this to a :class:`Format` member.
+    """
+    return first_byte >> _SCHEME_SHIFT
+
+
+def format_flags(first_byte: int) -> int:
+    """Extract the 3 format-specific flag bits from an ESE cell's header byte.
+
+    Returns ``first_byte & 0x7``. Meaning is format-specific.
+    """
+    return first_byte & _FLAG_MASK
+
+
+def header_byte(fmt: Format, flags: int = 0) -> int:
+    """Build a record header byte from a format id and optional flag bits.
+
+    Returns ``(fmt << 3) | (flags & 0x7)``.
+    """
+    return (fmt << _SCHEME_SHIFT) | (flags & _FLAG_MASK)
+
+
+# --- Shape A dispatch ---
+
+
+def _format_of(blob: Buffer) -> Format:
+    """Resolve a buffer's leading byte to a :class:`Format`, or raise."""
+    if len(blob) == 0:
+        msg = "cannot read a compression format from an empty buffer"
+        raise DecompressionError(msg)
+    raw = format_id(blob[0])
+    try:
+        fmt = Format(raw)
+    except ValueError:
+        msg = f"unknown ESE compression format id 0x{raw:x}"
+        raise DecompressionError(msg) from None
+    if fmt is Format.MAXIMUM:
+        msg = f"Format.MAXIMUM (0x{raw:x}) is a sentinel value, not a compression format"
+        raise DecompressionError(msg)
+    return fmt
+
+
+def compress(data: Buffer, fmt: Format) -> bytes:
+    """Encode plaintext into a framed ESE cell using the specified format.
+
+    Args:
+        data: The plaintext to compress.
+        fmt: The ESE compression format to use.
+
+    Raises:
+        CompressionError: The format is a sentinel (NONE, SCRUB, MAXIMUM), or the
+            input cannot be encoded by the requested format.
+        FormatUnavailableError: No codec is registered, or the codec does not
+            support compression (decode-only).
+    """
+    if fmt is Format.NONE:
+        msg = "Format.NONE is not a compression format"
+        raise CompressionError(msg)
+    if fmt is Format.SCRUB:
+        msg = "Format.SCRUB is not a compression format — use ntcompress.ese.scrub"
+        raise CompressionError(msg)
+    if fmt is Format.MAXIMUM:
+        msg = "Format.MAXIMUM is a sentinel value, not a compression format"
+        raise CompressionError(msg)
+    module = _get(fmt)
+    if not hasattr(module, "compress"):
+        msg = f"format {fmt.name} (0x{fmt.value:x}) is decode-only; no encoder is available"
+        raise FormatUnavailableError(msg)
+    return module.compress(data)
+
+
+def decompress(blob: Buffer, fmt: Format | None = None) -> bytes:
+    """Decode a framed ESE cell.
+
+    When ``fmt`` is None, the format is auto-detected from the header byte.
+
+    Args:
+        blob: The framed cell, including the header byte.
+        fmt: The format to use, or None for auto-detection.
+
+    Raises:
+        DecompressionError: Empty buffer, unknown format id, or decode failure.
+        ScrubDetectedError: The record is a SCRUB (0x4) erase marker.
+        FormatUnavailableError: The format is known but no codec is registered.
+    """
+    if fmt is None:
+        fmt = _format_of(blob)
+    if fmt is Format.NONE:
+        msg = "Format.NONE has no compression header; pass the raw cell body instead"
+        raise DecompressionError(msg)
+    if fmt is Format.SCRUB:
+        msg = "record is a SCRUB erase marker; no plaintext is recoverable — use ntcompress.ese.scrub"
+        raise ScrubDetectedError(msg)
+    if fmt is Format.MAXIMUM:
+        msg = "Format.MAXIMUM is a sentinel value, not a compression format"
+        raise DecompressionError(msg)
+    return _get(fmt).decompress(blob)
+
+
+def decompressed_size(blob: Buffer) -> int:
+    """Return a framed cell's recorded plaintext length without decoding.
+
+    Auto-detects the format from the header byte.
+
+    Raises:
+        DecompressionError: Empty buffer or unknown format id.
+        ScrubDetectedError: The record is a SCRUB erase marker.
+        FormatUnavailableError: The format is known but no codec is registered.
+    """
+    fmt = _format_of(blob)
+    if fmt is Format.NONE:
+        msg = "Format.NONE has no compression header; the size is the raw body length"
+        raise DecompressionError(msg)
+    if fmt is Format.SCRUB:
+        msg = "record is a SCRUB erase marker; it has no decompressed size — use ntcompress.ese.scrub"
+        raise ScrubDetectedError(msg)
+    return _get(fmt).decompressed_size(blob)
+
+
+# --- Auto-registration of codec modules ---
+
+from ntcompress.ese import lz4, sevenbit_ascii, sevenbit_unicode, xpress, xpress9, xpress10  # noqa: E402
+
+_register(Format.SEVEN_BIT_ASCII, sevenbit_ascii)
+_register(Format.SEVEN_BIT_UNICODE, sevenbit_unicode)
+_register(Format.XPRESS, xpress)
+_register(Format.XPRESS9, xpress9)
+_register(Format.XPRESS10, xpress10)
+_register(Format.LZ4, lz4)

ntcompress/ese/_registry.py

@@ -0,0 +1,32 @@
+"""Internal dispatch table for ESE compression formats.
+
+Maps :class:`Format` enum members to the modules that implement them. Each codec
+module is imported and registered when :mod:`ntcompress.ese` is first loaded.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ntcompress.exceptions import FormatUnavailableError
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+    from ntcompress.ese import Format
+
+_CODECS: dict[Format, ModuleType] = {}
+
+
+def _register(fmt: Format, module: ModuleType) -> None:
+    """Register a codec module for a given format."""
+    _CODECS[fmt] = module
+
+
+def _get(fmt: Format) -> ModuleType:
+    """Look up the codec module for a format, or raise."""
+    module = _CODECS.get(fmt)
+    if module is None:
+        msg = f"no codec registered for ESE format {fmt.name} (0x{fmt.value:x})"
+        raise FormatUnavailableError(msg)
+    return module

ntcompress/ese/_sevenbit.py

@@ -0,0 +1,167 @@
+"""Internal shared helpers for 7-bit ASCII and Unicode codecs.
+
+These two schemes share their framing and packer, so the common infrastructure lives
+here. Each packs 7 significant bits per source unit LSB-first into a continuous
+bitstream: 7BITASCII over bytes, 7BITUNICODE over UTF-16LE code units whose high byte
+is zero (so the wide scheme differs only in its 2-byte source stride and in re-emitting
+the ``0x00`` high byte on decode). The low 3 bits of the header byte hold the number of
+valid bits in the final packed byte, stored biased as 0-7 meaning 1-8.
+
+There is no public specification -- [MS-XCA] scopes only the Xpress family -- so the
+authority is the MIT-licensed ESE source: ``compression.cxx:985-1004`` (size math),
+``:1168-1387`` / ``:1390-1504`` (ASCII/Unicode writers), ``:2113-2185`` /
+``:2188-2272`` (ASCII/Unicode readers).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final
+
+from ntcompress.exceptions import DecompressionError
+
+if TYPE_CHECKING:
+    from ntcompress.ese import Format
+
+# --- Constants ---
+
+UNIT_BITS: Final = 7
+"""Packed bits per source unit; the writers' fixed ``cbits = 7`` (compression.cxx:1310)."""
+
+UNIT_MASK: Final = 0x7F
+"""Mask of one packed unit; both readers extract ``(x >> ibit) & 0x7F`` (compression.cxx:2166, 2243)."""
+
+_BYTE_BITS: Final = 8
+_WIDE_STRIDE: Final = 2  # bytes per UTF-16LE code unit (sizeof(WORD), compression.cxx:1002)
+
+MIN_CELL_SIZE: Final = 2
+"""Smallest well-formed cell: header byte + one packed byte, which the readers' ``Cb - 2`` size math presumes (compression.cxx:2135)."""
+
+
+# --- Shared framing ---
+
+
+@dataclass(frozen=True)
+class SevenBitHeader:
+    """A parsed 7-bit cell header, per ``ErrDecompress7Bit*_`` (compression.cxx:2127-2137).
+
+    Captures the two facts the readers derive before unpacking: how many bits of
+    the last packed byte are valid, and hence exactly how many 7-bit units the
+    stream holds.
+    """
+
+    fmt: Format
+    """The format id from the header's top 5 bits (SEVEN_BIT_ASCII or SEVEN_BIT_UNICODE)."""
+
+    final_byte_bits: int
+    """Valid bits in the last packed byte, 1-8: ``(bHeader & 0x7) + 1`` (compression.cxx:2131)."""
+
+    unit_count: int
+    """Number of 7-bit units in the stream: ``((Cb - 2) * 8 + final_byte_bits) // 7`` (compression.cxx:2135-2137)."""
+
+
+def _parse_header(blob: bytes, expected: Format) -> SevenBitHeader:
+    """Validate a framed 7-bit cell and derive its unit count.
+
+    Ports the size prologue shared by both readers (compression.cxx:2127-2137).
+    ESE only *asserts* ``cbitTotal % 7 == 0`` in DEBUG builds and floor-divides in
+    retail; real-world producers do emit non-canonical bit counts (the same
+    Exchange payload circulates with header ``0x10`` and ``0x0e``), so this port
+    keeps the retail floor-division behavior.
+
+    Args:
+        blob: The framed cell, header byte included.
+        expected: The format the calling codec owns.
+
+    Returns:
+        The parsed header.
+
+    Raises:
+        DecompressionError: The cell is shorter than header + one stream byte, or
+            its format id is not ``expected``.
+    """
+    from ntcompress.ese import format_flags, format_id
+
+    if len(blob) < MIN_CELL_SIZE:
+        msg = f"a 7-bit cell needs a header byte plus at least one packed byte, got {len(blob)} byte(s)"
+        raise DecompressionError(msg)
+    raw = format_id(blob[0])
+    if raw != expected:
+        msg = f"cell header carries format id 0x{raw:x}, not {expected.name} (0x{expected.value:x})"
+        raise DecompressionError(msg)
+    final_byte_bits = format_flags(blob[0]) + 1
+    total_bits = (len(blob) - 2) * _BYTE_BITS + final_byte_bits
+    return SevenBitHeader(fmt=expected, final_byte_bits=final_byte_bits, unit_count=total_bits // UNIT_BITS)
+
+
+def _check_padding(blob: bytes, header: SevenBitHeader) -> None:
+    """Reject a cell whose declared-invalid final-byte bits are non-zero.
+
+    ESE's writers flush from a zero-initialized accumulator, so the padding bits
+    above ``final_byte_bits`` are always zero in a well-formed cell. This checks only
+    those high bits of the final packed byte; libesedb performs a comparable (not
+    identical) padding-rejection check on corrupt streams (cf. ``value_16bit != 0``,
+    libesedb_compression.c:220). Gated on ``verify`` by the callers.
+    """
+    if blob[-1] >> header.final_byte_bits:
+        msg = f"non-zero padding above the {header.final_byte_bits} declared valid bit(s) of the final packed byte"
+        raise DecompressionError(msg)
+
+
+def _unpack_units(stream: bytes, count: int) -> bytearray:
+    """Unpack ``count`` LSB-first 7-bit units from a packed stream.
+
+    Accumulator port of the readers' inner loop (compression.cxx:2154-2180): the C
+    code re-reads a byte or little-endian WORD at each bit offset, which is exactly
+    equivalent to draining 7-bit units off a little-endian accumulator. ``count``
+    comes from :func:`_parse_header`, which guarantees the stream holds enough bits.
+    """
+    out = bytearray()
+    accumulator = 0
+    bits = 0
+    for byte in stream:
+        accumulator |= byte << bits
+        bits += _BYTE_BITS
+        while bits >= UNIT_BITS:
+            if len(out) == count:
+                return out
+            out.append(accumulator & UNIT_MASK)
+            accumulator >>= UNIT_BITS
+            bits -= UNIT_BITS
+    return out
+
+
+def _pack_units(units: bytes) -> bytearray:
+    """Pack 7-bit values LSB-first into a byte stream.
+
+    Accumulator port of the writers' slow path (compression.cxx:1298-1348 ASCII,
+    :1413-1464 Unicode): OR each unit's 7 bits at the current bit offset and flush
+    whole bytes. The final partial byte is always written, zero-padded high --
+    matching ESE's ``(ibitOutputCurr + 7) / 8`` trailing flush (:1354-1362).
+    """
+    out = bytearray()
+    accumulator = 0
+    bits = 0
+    for value in units:
+        accumulator |= value << bits
+        bits += UNIT_BITS
+        if bits >= _BYTE_BITS:
+            out.append(accumulator & 0xFF)
+            accumulator >>= _BYTE_BITS
+            bits -= _BYTE_BITS
+    if bits:
+        out.append(accumulator)
+    return out
+
+
+def _frame(fmt: Format, units: bytes) -> bytes:
+    """Build a framed cell: header byte + packed stream.
+
+    The stored bit count is ``(ibitOutputCurr - 1) % 8`` (compression.cxx:1379,
+    :1497), which for ``n`` units reduces to ``(7n - 1) % 8`` -- the writers' 0->32
+    fixup for an exactly-full final DWORD (:1374-1378) folds into the same formula.
+    """
+    from ntcompress.ese import header_byte
+
+    final_bits_biased = (len(units) * UNIT_BITS - 1) % _BYTE_BITS
+    return bytes([header_byte(fmt, final_bits_biased)]) + bytes(_pack_units(units))

ntcompress/ese/checksums.py

@@ -0,0 +1,97 @@
+"""Checksums used by ESE compression headers.
+
+Two non-stdlib CRCs are required, and both are implemented here from their literal
+parameters. Python's :func:`zlib.crc32` uses a different polynomial and must not be
+substituted for either.
+
+* :func:`crc32c_ese` -- CRC-32C / Castagnoli (reflected polynomial ``0x82F63B78``,
+  normal form ``0x1EDC6F41``; init/xorout ``0xFFFFFFFF``): the "SSE 4.2 compatible" CRC
+  computed over the XPRESS10 plaintext. Verified check value
+  ``crc32c_ese(b"123456789") == 0xE3069283``.
+* :func:`crc64_ese` -- reflected polynomial ``0x9A6C9329AC4BC9B5``, init/xorout
+  all-ones: the "Corsica compatible" CRC computed over the XPRESS10 compressed
+  payload. Verified check value ``crc64_ese(b"123456789") == 0xAE8B14860A799888``.
+  This parameterization is **CRC-64/NVME** (RevEng catalogue: normal polynomial
+  ``0xAD93D23594C93659``, reflected ``0x9A6C9329AC4BC9B5``, init/xorout all-ones,
+  refin/refout, check ``0xAE8B14860A799888``). It is NOT CRC-64/XZ (reflected poly
+  ``0xC96C5795D7870F42``); do not use a library preset named ``crc64('xz')``.
+
+Both are little-endian / reflected CRCs, so a right-shifting byte-wise table drives
+them. The polynomials are already given in reflected form.
+
+Authority: ``dev/ese/src/_xpress10/xpress10sw.cxx:34-59`` (the CRC-64 bit loop, poly
+``CORSICA_CRC64_POLY``) and ESE ``Crc32Checksum`` in ``dev/ese/src/os/encrypt.cxx``
+(CRC-32C).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Final
+
+if TYPE_CHECKING:
+    from ntcompress._types import Buffer
+
+# --- Polynomials (reflected / LSB-first form) ---
+
+CRC32C_POLY: Final = 0x82F63B78
+"""Reflected CRC-32C (Castagnoli) polynomial (normal form 0x1EDC6F41)."""
+
+CRC64_ESE_POLY: Final = 0x9A6C9329AC4BC9B5
+"""Reflected CRC-64/NVME polynomial (xpress10sw.cxx:35, ``CORSICA_CRC64_POLY``). Not CRC-64/XZ."""
+
+_MASK32: Final = 0xFFFFFFFF
+_MASK64: Final = 0xFFFFFFFFFFFFFFFF
+
+
+def _make_table(poly: int, mask: int) -> tuple[int, ...]:
+    """Build a 256-entry byte-wise lookup table for a reflected (right-shifting) CRC."""
+    table: list[int] = []
+    for index in range(256):
+        crc = index
+        for _ in range(8):
+            crc = (crc >> 1) ^ poly if crc & 1 else crc >> 1
+            crc &= mask
+        table.append(crc)
+    return tuple(table)
+
+
+_CRC32C_TABLE: Final = _make_table(CRC32C_POLY, _MASK32)
+_CRC64_TABLE: Final = _make_table(CRC64_ESE_POLY, _MASK64)
+
+
+def crc32c_ese(data: Buffer) -> int:
+    """Return the CRC-32C (Castagnoli) of ``data``, as ESE stores it in headers.
+
+    Reflected, init/xorout ``0xFFFFFFFF``. This is the "SSE 4.2 compatible" checksum
+    ESE computes over uncompressed data (``Crc32Checksum``); the XPRESS10 header
+    stores it over the plaintext.
+
+    Args:
+        data: The bytes to checksum.
+
+    Returns:
+        The 32-bit CRC-32C. ``crc32c_ese(b"123456789") == 0xE3069283``.
+    """
+    crc = _MASK32
+    for byte in data:
+        crc = (crc >> 8) ^ _CRC32C_TABLE[(crc ^ byte) & 0xFF]
+    return crc ^ _MASK32
+
+
+def crc64_ese(data: Buffer) -> int:
+    """Return the ESE/Corsica CRC-64 of ``data``.
+
+    Reflected poly ``0x9A6C9329AC4BC9B5``, init/xorout all-ones -- the CRC-64/NVME
+    parameterization. The XPRESS10 header stores this over the compressed payload
+    (``UtilGenCorsicaCrc64``, ``xpress10sw.cxx:34-59``).
+
+    Args:
+        data: The bytes to checksum.
+
+    Returns:
+        The 64-bit CRC. ``crc64_ese(b"123456789") == 0xAE8B14860A799888``.
+    """
+    crc = _MASK64
+    for byte in data:
+        crc = (crc >> 8) ^ _CRC64_TABLE[(crc ^ byte) & 0xFF]
+    return crc ^ _MASK64