mercurykit 0.1.0__py3-none-any.whl

mercurykit/__init__.py

@@ -0,0 +1,18 @@
+"""MercuryKit public package exports."""
+
+from mercurykit.archive import ArchiveContext, ArchiveEntry, ArchiveInfo, ArchiveMatch, UnsupportedOperation
+from mercurykit.bfpk import BfpkEngine
+from mercurykit.binary import BinaryReader, Endian
+from mercurykit.mirror_of_fate import MirrorOfFatePackEngine
+
+__all__ = [
+    "ArchiveContext",
+    "ArchiveEntry",
+    "ArchiveInfo",
+    "ArchiveMatch",
+    "BfpkEngine",
+    "BinaryReader",
+    "Endian",
+    "MirrorOfFatePackEngine",
+    "UnsupportedOperation",
+]

mercurykit/__main__.py

@@ -0,0 +1,4 @@
+from mercurykit.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mercurykit/_bfpk/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .engine import BfpkEngine
+
+__all__ = ["BfpkEngine"]

mercurykit/_bfpk/codecs.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from mercurykit.binary import EndOfStreamError
+from mercurykit.archive import UnsupportedOperation
+
+
+class BfpkCodecMixin:
+    def _aes_256_cbc_crypt(self, data: bytes, *, decrypt: bool) -> bytes:
+        if len(data) % 16 != 0:
+            raise ValueError("BFPK AES table data must be 16-byte aligned")
+        try:
+            from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+        except ImportError as exc:
+            raise UnsupportedOperation("BFPK AES table support requires the cryptography package") from exc
+
+        cipher = Cipher(algorithms.AES(self.lords_of_shadow_ultimate_aes_key), modes.CBC(bytes(16)))
+        transform = cipher.decryptor() if decrypt else cipher.encryptor()
+        return transform.update(data) + transform.finalize()
+
+    def _decrypt_lords_of_shadow_ultimate_table(self, data: bytes) -> bytes:
+        return self._aes_256_cbc_crypt(data, decrypt=True)
+
+    def _encrypt_lords_of_shadow_ultimate_table(self, data: bytes) -> bytes:
+        return self._aes_256_cbc_crypt(data, decrypt=False)
+
+    def _require_lz4_block(self):
+        try:
+            import lz4.block as lz4_block
+        except ImportError as exc:
+            raise UnsupportedOperation("BFPK LZ4 repack requires the optional lz4 package") from exc
+        return lz4_block
+
+    def _decompress_lz4_block(self, data: bytes, expected_size: int) -> bytes:
+        output = bytearray()
+        index = 0
+        while index < len(data):
+            token = data[index]
+            index += 1
+            literal_length = token >> 4
+            if literal_length == 15:
+                while True:
+                    if index >= len(data):
+                        raise EndOfStreamError("BFPK LZ4 literal length extends beyond chunk")
+                    value = data[index]
+                    index += 1
+                    literal_length += value
+                    if value != 255:
+                        break
+
+            if index + literal_length > len(data):
+                raise EndOfStreamError("BFPK LZ4 literal data extends beyond chunk")
+            output.extend(data[index:index + literal_length])
+            index += literal_length
+            if index >= len(data):
+                break
+
+            if index + 2 > len(data):
+                raise EndOfStreamError("BFPK LZ4 match offset extends beyond chunk")
+            offset = data[index] | (data[index + 1] << 8)
+            index += 2
+            if offset == 0 or offset > len(output):
+                raise ValueError("BFPK LZ4 match offset is invalid")
+
+            match_length = (token & 0x0F) + 4
+            if (token & 0x0F) == 15:
+                while True:
+                    if index >= len(data):
+                        raise EndOfStreamError("BFPK LZ4 match length extends beyond chunk")
+                    value = data[index]
+                    index += 1
+                    match_length += value
+                    if value != 255:
+                        break
+
+            start = len(output) - offset
+            for offset_index in range(match_length):
+                output.append(output[start + offset_index])
+
+        if len(output) != expected_size:
+            raise ValueError("BFPK LZ4 chunk decompressed to unexpected size")
+        return bytes(output)
+
+    def _xxh32(self, data: bytes, seed: int = 0) -> int:
+        prime1 = 0x9E3779B1
+        prime2 = 0x85EBCA77
+        prime3 = 0xC2B2AE3D
+        prime4 = 0x27D4EB2F
+        prime5 = 0x165667B1
+
+        def rotate_left(value: int, amount: int) -> int:
+            return ((value << amount) | (value >> (32 - amount))) & self.max_u32
+
+        index = 0
+        length = len(data)
+        if length >= 16:
+            v1 = (seed + prime1 + prime2) & self.max_u32
+            v2 = (seed + prime2) & self.max_u32
+            v3 = seed & self.max_u32
+            v4 = (seed - prime1) & self.max_u32
+            while index <= length - 16:
+                lane = int.from_bytes(data[index:index + 4], "little")
+                index += 4
+                v1 = (rotate_left((v1 + lane * prime2) & self.max_u32, 13) * prime1) & self.max_u32
+                lane = int.from_bytes(data[index:index + 4], "little")
+                index += 4
+                v2 = (rotate_left((v2 + lane * prime2) & self.max_u32, 13) * prime1) & self.max_u32
+                lane = int.from_bytes(data[index:index + 4], "little")
+                index += 4
+                v3 = (rotate_left((v3 + lane * prime2) & self.max_u32, 13) * prime1) & self.max_u32
+                lane = int.from_bytes(data[index:index + 4], "little")
+                index += 4
+                v4 = (rotate_left((v4 + lane * prime2) & self.max_u32, 13) * prime1) & self.max_u32
+            h32 = (rotate_left(v1, 1) + rotate_left(v2, 7) + rotate_left(v3, 12) + rotate_left(v4, 18)) & self.max_u32
+        else:
+            h32 = (seed + prime5) & self.max_u32
+
+        h32 = (h32 + length) & self.max_u32
+        while index <= length - 4:
+            lane = int.from_bytes(data[index:index + 4], "little")
+            index += 4
+            h32 = (rotate_left((h32 + lane * prime3) & self.max_u32, 17) * prime4) & self.max_u32
+        while index < length:
+            h32 = (rotate_left((h32 + data[index] * prime5) & self.max_u32, 11) * prime1) & self.max_u32
+            index += 1
+
+        h32 ^= h32 >> 15
+        h32 = (h32 * prime2) & self.max_u32
+        h32 ^= h32 >> 13
+        h32 = (h32 * prime3) & self.max_u32
+        h32 ^= h32 >> 16
+        return h32 & self.max_u32
+
+    def _check_u32(self, value: int, label: str) -> None:
+        if not 0 <= value <= self.max_u32:
+            raise ValueError(f"{label} must fit in u32")
+
+    def _check_u64(self, value: int, label: str) -> None:
+        if not 0 <= value <= self.max_u64:
+            raise ValueError(f"{label} must fit in u64")
+

mercurykit/_bfpk/engine.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from typing import Iterable
+
+from mercurykit.archive import ArchiveContext, ArchiveEntry, ArchiveInfo, ArchiveMatch, UnsupportedOperation
+from mercurykit.binary import BinaryReader, EndOfStreamError
+
+from .codecs import BfpkCodecMixin
+from .extraction import BfpkExtractionMixin
+from .manifest import BfpkManifestMixin
+from .models import BfpkState
+from .repack import BfpkRepackMixin
+
+
+class BfpkEngine(BfpkManifestMixin, BfpkExtractionMixin, BfpkRepackMixin, BfpkCodecMixin):
+    """Engine for MercurySteam BFPK game archives."""
+
+    format_name = "MercurySteam BFPK Archive"
+
+    archive_magic = b"BFPK"
+    legacy_layout = "legacy"
+    blades_of_fire_layout = "blades_of_fire"
+    spacelords_layout = "spacelords"
+    lords_of_shadow_ultimate_layout = "lords_of_shadow_ultimate"
+    scrapland_layout = "scrapland"
+    scrapland_archive_version = 0x0
+    scrapland_path_encoding = "cp1252"
+    legacy_archive_versions = frozenset({0x100, 0x101, 0x102})
+    blades_of_fire_archive_versions = frozenset({0x100, 0x102, 0x300})
+    spacelords_archive_versions = frozenset({0x500, 0x502})
+    lords_of_shadow_ultimate_archive_versions = frozenset({0x2, 0x3})
+    chunked_header_archive_versions = frozenset({0x102, 0x502})
+    default_file_chunk_size = 0x10000
+    blades_of_fire_default_file_chunk_size = 0x40000
+    spacelords_default_file_chunk_size = 0x40000
+    spacelords_default_trailing_padding = 0x10000
+    encrypted_picture_alignment = 0x10000
+    spacelords_d01_archive_version = 0xD01
+    blades_of_fire_pics_archive_version = 0x901
+    blades_of_fire_pics_default_record_flags = 0x09FF
+    blades_of_fire_pics_gif_record_flag_low = 0xC0
+    spacelords_d01_default_record_flags = 0x00F1
+    spacelords_d01_gif_record_flag_low = 0xDF
+    spacelords_d01_tga_record_flag_low = 0x44
+    default_trailing_padding = 0x8000
+    default_compression_level = -1
+    lords_of_shadow_ultimate_table_prefix = b"MercuryKitLoSUE!"
+    lords_of_shadow_ultimate_aes_key = bytes.fromhex(
+        "50 43 56 80 72 73 EE 6F F1 44 F3 6E EA DF 79 43 "
+        "6C 69 6D 61 78 53 74 75 64 69 6F 73 32 30 31 33"
+    )
+    max_u32 = 0xFFFFFFFF
+    max_u64 = 0xFFFFFFFFFFFFFFFF
+
+    def _spacelords_d01_xor_key(self, offset: int) -> int:
+        return (((offset * offset * 0x343FD) + 0x269EC3) >> 16) & 0xFF
+
+    def _crypt_spacelords_d01_table(self, data: bytes, start_offset: int) -> bytes:
+        return bytes(byte ^ self._spacelords_d01_xor_key(start_offset + index) for index, byte in enumerate(data))
+
+    def _has_encrypted_picture_table(self, archive_version: int) -> bool:
+        return archive_version in {self.spacelords_d01_archive_version, self.blades_of_fire_pics_archive_version}
+
+    def _has_chunked_header(self, archive_version: int) -> bool:
+        return archive_version in self.chunked_header_archive_versions
+
+    def _has_lords_of_shadow_ultimate_table(self, archive_version: int) -> bool:
+        return archive_version in self.lords_of_shadow_ultimate_archive_versions
+
+    def _supports_legacy_layout(self, archive_version: int) -> bool:
+        return archive_version in self.legacy_archive_versions
+
+    def _supports_scrapland_layout(self, archive_version: int) -> bool:
+        return archive_version == self.scrapland_archive_version
+
+    def _supports_lords_of_shadow_ultimate_layout(self, archive_version: int) -> bool:
+        return archive_version in self.lords_of_shadow_ultimate_archive_versions
+
+    def _supports_blades_of_fire_layout(self, archive_version: int) -> bool:
+        return archive_version in self.blades_of_fire_archive_versions or archive_version == self.blades_of_fire_pics_archive_version
+
+    def _supports_spacelords_layout(self, archive_version: int) -> bool:
+        return archive_version in self.spacelords_archive_versions or archive_version == self.spacelords_d01_archive_version
+
+    def _encrypted_picture_layout(self, archive_version: int) -> str:
+        if archive_version == self.blades_of_fire_pics_archive_version:
+            return self.blades_of_fire_layout
+        if archive_version == self.spacelords_d01_archive_version:
+            return self.spacelords_layout
+        raise ValueError(f"BFPK archive version 0x{archive_version:x} is not an encrypted picture archive")
+
+    def evaluate(self, path: Path, reader: BinaryReader) -> ArchiveMatch:
+        try:
+            header = self._read_header(reader)
+            file_size = self._reader_size(reader)
+            layout, records = self._select_table_layout(reader, header, file_size)
+            self._validate_sample_payloads(reader, header, layout, records, file_size)
+        except (EndOfStreamError, UnicodeDecodeError, ValueError, UnsupportedOperation) as exc:
+            return ArchiveMatch(0.0, self.format_name, reason=str(exc))
+
+        reason = f"BFPK {layout} layout matched"
+        return ArchiveMatch(0.99, self.format_name, reason=reason)
+
+    def read_manifest(self, path: Path, reader: BinaryReader) -> ArchiveInfo:
+        header = self._read_header(reader)
+        file_size = self._reader_size(reader)
+        layout, records = self._select_table_layout(reader, header, file_size)
+        entries = [
+            self._entry_for_version(reader, header.archive_version, header.file_chunk_size, layout, record)
+            for record in records
+        ]
+
+        return ArchiveInfo(
+            self.format_name,
+            len(entries),
+            {
+                "archive_version": header.archive_version,
+                "file_chunk_size": header.file_chunk_size,
+                "table_format": layout,
+                "encrypted_table_size": header.encrypted_table_size,
+                "table_prefix": header.table_prefix,
+                "entries": tuple(entries),
+            },
+        )
+
+    def open(self, path: Path) -> ArchiveContext:
+        with path.open("rb") as file:
+            info = self.read_manifest(path, BinaryReader(file))
+        return ArchiveContext(path, self, info, BfpkState(tuple(info.metadata["entries"])))
+
+    def iter_entries(self, context: ArchiveContext) -> Iterable[ArchiveEntry]:
+        state = self._state(context)
+        yield from state.entries
+
+    def _state(self, context: ArchiveContext) -> BfpkState:
+        if isinstance(context.state, BfpkState):
+            return context.state
+        return BfpkState(tuple(context.info.metadata["entries"]))
+
+

mercurykit/_bfpk/extraction.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import zlib
+from typing import BinaryIO
+
+from mercurykit.binary import BinaryReader, EndOfStreamError
+from mercurykit.codecs import compression_registry
+from mercurykit.archive import ArchiveContext, ArchiveEntry
+
+
+class BfpkExtractionMixin:
+    def extract_entry(self, context: ArchiveContext, entry: ArchiveEntry, output_stream: BinaryIO) -> None:
+        if entry.offset is None:
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} does not define an offset")
+        if entry.uncompressed_size is None:
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} does not define an uncompressed size")
+
+        if entry.metadata.get("chunked"):
+            self._extract_chunked_entry(context, entry, output_stream)
+            return
+
+        if entry.compression is not None:
+            if entry.compressed_size is None:
+                raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} does not define a stored size")
+            with context.archive_path.open("rb") as file:
+                file.seek(entry.offset)
+                payload = file.read(entry.compressed_size)
+            if len(payload) != entry.compressed_size:
+                raise EndOfStreamError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} extends beyond archive data")
+            payload = compression_registry.decompress(entry.compression, payload)
+            if len(payload) != entry.uncompressed_size:
+                raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} decompressed to unexpected size")
+            output_stream.write(payload)
+            return
+
+        with context.archive_path.open("rb") as file:
+            file.seek(entry.offset)
+            payload = file.read(entry.uncompressed_size)
+        if len(payload) != entry.uncompressed_size:
+            raise EndOfStreamError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} extends beyond archive data")
+        expected_crc = entry.metadata.get("table_hash")
+        if expected_crc is not None and (zlib.crc32(payload) & self.max_u32) != expected_crc:
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} CRC did not match")
+        output_stream.write(payload)
+
+    def _extract_chunked_entry(self, context: ArchiveContext, entry: ArchiveEntry, output_stream: BinaryIO) -> None:
+        chunk_offsets = tuple(entry.metadata.get("chunk_offsets") or ())
+        chunk_compressed_sizes = tuple(entry.metadata.get("chunk_compressed_sizes") or ())
+        chunk_uncompressed_sizes = tuple(entry.metadata.get("chunk_uncompressed_sizes") or ())
+        chunk_hashes = tuple(entry.metadata.get("chunk_hashes") or ())
+        if not (len(chunk_offsets) == len(chunk_compressed_sizes) == len(chunk_uncompressed_sizes)):
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} has inconsistent chunk metadata")
+        if chunk_hashes and len(chunk_hashes) != len(chunk_offsets):
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} has inconsistent chunk hash metadata")
+
+        total_uncompressed_size = 0
+        crc = 0
+        with context.archive_path.open("rb") as file:
+            for index, (chunk_offset, chunk_compressed_size, chunk_uncompressed_size) in enumerate(
+                zip(chunk_offsets, chunk_compressed_sizes, chunk_uncompressed_sizes)
+            ):
+                file.seek(chunk_offset)
+                payload = file.read(chunk_compressed_size)
+                if len(payload) != chunk_compressed_size:
+                    raise EndOfStreamError(
+                        f"Entry {entry.path or entry.entry_id or '<unnamed>'} chunk extends beyond archive data"
+                    )
+                if chunk_hashes and self._xxh32(payload) != chunk_hashes[index]:
+                    raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} chunk hash did not match")
+
+                if entry.compression == "lz4-block":
+                    payload = self._decompress_lz4_block(payload, chunk_uncompressed_size)
+                else:
+                    payload = compression_registry.decompress("zlib", payload)
+                if len(payload) != chunk_uncompressed_size:
+                    raise ValueError(
+                        f"Entry {entry.path or entry.entry_id or '<unnamed>'} chunk decompressed to unexpected size"
+                    )
+                output_stream.write(payload)
+                crc = zlib.crc32(payload, crc)
+                total_uncompressed_size += len(payload)
+
+        if total_uncompressed_size != entry.uncompressed_size:
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} decompressed to unexpected size")
+        expected_crc = entry.metadata.get("table_hash")
+        if expected_crc is not None and (crc & self.max_u32) != expected_crc:
+            raise ValueError(f"Entry {entry.path or entry.entry_id or '<unnamed>'} CRC did not match")
+
+    def _crc32_lz4_chunks(self, reader: BinaryReader, entry: ArchiveEntry) -> int:
+        chunk_offsets = tuple(entry.metadata.get("chunk_offsets") or ())
+        chunk_compressed_sizes = tuple(entry.metadata.get("chunk_compressed_sizes") or ())
+        chunk_uncompressed_sizes = tuple(entry.metadata.get("chunk_uncompressed_sizes") or ())
+        chunk_hashes = tuple(entry.metadata.get("chunk_hashes") or ())
+        layout = entry.metadata.get("table_format")
+        crc = 0
+        for index, (chunk_offset, chunk_compressed_size, chunk_uncompressed_size) in enumerate(
+            zip(chunk_offsets, chunk_compressed_sizes, chunk_uncompressed_sizes)
+        ):
+            reader.seek(chunk_offset)
+            payload = reader.read_exact(chunk_compressed_size)
+            if chunk_hashes and self._xxh32(payload) != chunk_hashes[index]:
+                if layout == self.spacelords_layout:
+                    raise ValueError("BFPK Spacelords chunk hash did not match")
+                raise ValueError("BFPK Blades of Fire chunk hash did not match")
+            decompressed = self._decompress_lz4_block(payload, chunk_uncompressed_size)
+            crc = zlib.crc32(decompressed, crc)
+        return crc & self.max_u32
+