zipwire 0.0.1__py3-none-any.whl

zipwire/_parser.py

@@ -0,0 +1,273 @@
+"""Pure ZIP parsing: EOCD, central directory, and local file header.
+
+All functions operate on `bytes` - no IO. This module is shared by both
+the sync and async code paths.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from zipwire._constants import (
+    CENTRAL_DIR_SIGNATURE,
+    CENTRAL_DIR_SIZE,
+    CENTRAL_DIR_STRUCT,
+    EOCD_SIGNATURE,
+    EOCD_SIZE,
+    EOCD_STRUCT,
+    LOCAL_FILE_HEADER_SIGNATURE,
+    LOCAL_FILE_HEADER_SIZE,
+    LOCAL_FILE_HEADER_STRUCT,
+    MAX_EOCD_SEARCH,
+    ZIP64_EOCD_LOCATOR_SIGNATURE,
+    ZIP64_EOCD_LOCATOR_SIZE,
+    ZIP64_EOCD_LOCATOR_STRUCT,
+    ZIP64_EOCD_SIGNATURE,
+    ZIP64_EOCD_SIZE,
+    ZIP64_EOCD_STRUCT,
+)
+from zipwire._errors import BadZipFile
+
+
+@dataclass(frozen=True, slots=True)
+class EOCDInfo:
+    """Parsed End of Central Directory information."""
+
+    cd_offset: int
+    cd_size: int
+    cd_entry_count: int
+
+
+@dataclass(frozen=True, slots=True)
+class CentralDirEntry:
+    """Raw parsed central directory entry."""
+
+    version_made_by: int
+    version_needed: int
+    flags: int
+    compression_method: int
+    mod_time: int
+    mod_date: int
+    crc32: int
+    compressed_size: int
+    uncompressed_size: int
+    disk_start: int
+    internal_attr: int
+    external_attr: int
+    header_offset: int
+    filename: bytes
+    extra: bytes
+    comment: bytes
+
+
+@dataclass(frozen=True, slots=True)
+class LocalHeaderInfo:
+    """Parsed local file header - just enough to find the data offset."""
+
+    filename_length: int
+    extra_length: int
+
+    @property
+    def data_offset_past_header(self) -> int:
+        """Number of bytes after the fixed header to skip to reach file data."""
+        return self.filename_length + self.extra_length
+
+
+def find_eocd(tail: bytes, file_size: int) -> EOCDInfo:
+    """Find and parse the End of Central Directory record.
+
+    Args:
+        tail: The last `min(MAX_EOCD_SEARCH, file_size)` bytes of the file.
+        file_size: Total size of the ZIP file.
+
+    Returns:
+        EOCDInfo with central directory location.
+
+    Raises:
+        BadZipFile: If no valid EOCD record is found.
+    """
+    # Search backwards for EOCD signature
+    pos = tail.rfind(EOCD_SIGNATURE)
+    if pos == -1:
+        raise BadZipFile("Could not find End of Central Directory record")
+
+    if len(tail) - pos < EOCD_SIZE:
+        raise BadZipFile("EOCD record is truncated")
+
+    fields = EOCD_STRUCT.unpack_from(tail, pos)
+    # fields: signature, disk_num, disk_cd_start, cd_entries_this_disk,
+    #         cd_entries_total, cd_size, cd_offset, comment_length
+    cd_entry_count = fields[4]
+    cd_size = fields[5]
+    cd_offset = fields[6]
+
+    # Check for ZIP64 - indicated by 0xFFFF or 0xFFFFFFFF sentinel values
+    is_zip64 = cd_entry_count == 0xFFFF or cd_size == 0xFFFFFFFF or cd_offset == 0xFFFFFFFF
+
+    if is_zip64:
+        return _parse_zip64_eocd(tail, pos, file_size)
+
+    return EOCDInfo(
+        cd_offset=cd_offset,
+        cd_size=cd_size,
+        cd_entry_count=cd_entry_count,
+    )
+
+
+def _parse_zip64_eocd(tail: bytes, eocd_pos: int, file_size: int) -> EOCDInfo:
+    """Parse ZIP64 EOCD locator and record.
+
+    Args:
+        tail: The tail bytes of the file.
+        eocd_pos: Position of the regular EOCD in `tail`.
+        file_size: Total file size.
+
+    Returns:
+        EOCDInfo with 64-bit values.
+
+    Raises:
+        BadZipFile: If ZIP64 structures are missing or invalid.
+    """
+    # ZIP64 EOCD Locator is right before the regular EOCD
+    locator_pos = eocd_pos - ZIP64_EOCD_LOCATOR_SIZE
+    if locator_pos < 0:
+        raise BadZipFile("ZIP64 EOCD locator not found (not enough data before EOCD)")
+
+    locator_sig = tail[locator_pos : locator_pos + 4]
+    if locator_sig != ZIP64_EOCD_LOCATOR_SIGNATURE:
+        raise BadZipFile("ZIP64 EOCD locator signature mismatch")
+
+    locator_fields = ZIP64_EOCD_LOCATOR_STRUCT.unpack_from(tail, locator_pos)
+    zip64_eocd_abs_offset = locator_fields[2]
+
+    # The ZIP64 EOCD record offset is absolute in the file. Convert to tail-relative.
+    tail_start = file_size - len(tail)
+    zip64_eocd_rel = zip64_eocd_abs_offset - tail_start
+
+    if zip64_eocd_rel < 0 or zip64_eocd_rel + ZIP64_EOCD_SIZE > len(tail):
+        raise BadZipFile("ZIP64 EOCD record is outside the fetched tail data")
+
+    z64_sig = tail[zip64_eocd_rel : zip64_eocd_rel + 4]
+    if z64_sig != ZIP64_EOCD_SIGNATURE:
+        raise BadZipFile("ZIP64 EOCD record signature mismatch")
+
+    z64_fields = ZIP64_EOCD_STRUCT.unpack_from(tail, zip64_eocd_rel)
+    # fields: signature, size_of_record, version_made, version_needed,
+    #         disk_num, disk_cd_start, cd_entries_this_disk,
+    #         cd_entries_total, cd_size, cd_offset
+    cd_entry_count = z64_fields[7]
+    cd_size = z64_fields[8]
+    cd_offset = z64_fields[9]
+
+    return EOCDInfo(
+        cd_offset=cd_offset,
+        cd_size=cd_size,
+        cd_entry_count=cd_entry_count,
+    )
+
+
+def parse_central_directory(data: bytes, entry_count: int) -> list[CentralDirEntry]:
+    """Parse all central directory entries from raw bytes.
+
+    Args:
+        data: The raw central directory data.
+        entry_count: Expected number of entries.
+
+    Returns:
+        List of CentralDirEntry objects.
+
+    Raises:
+        BadZipFile: If the data is malformed.
+    """
+    entries: list[CentralDirEntry] = []
+    offset = 0
+
+    for _ in range(entry_count):
+        if offset + CENTRAL_DIR_SIZE > len(data):
+            raise BadZipFile("Central directory is truncated")
+
+        if data[offset : offset + 4] != CENTRAL_DIR_SIGNATURE:
+            raise BadZipFile(f"Expected central directory signature at offset {offset}")
+
+        fields = CENTRAL_DIR_STRUCT.unpack_from(data, offset)
+        # fields: signature(0), version_made(1), version_needed(2), flags(3),
+        #         compression(4), mod_time(5), mod_date(6), crc32(7),
+        #         compressed_size(8), uncompressed_size(9), filename_length(10),
+        #         extra_length(11), comment_length(12), disk_start(13),
+        #         internal_attr(14), external_attr(15), header_offset(16)
+
+        filename_length = fields[10]
+        extra_length = fields[11]
+        comment_length = fields[12]
+
+        var_start = offset + CENTRAL_DIR_SIZE
+        var_end = var_start + filename_length + extra_length + comment_length
+
+        if var_end > len(data):
+            raise BadZipFile("Central directory entry extends past end of data")
+
+        filename = data[var_start : var_start + filename_length]
+        extra = data[var_start + filename_length : var_start + filename_length + extra_length]
+        comment = data[
+            var_start + filename_length + extra_length : var_start
+            + filename_length
+            + extra_length
+            + comment_length
+        ]
+
+        entry = CentralDirEntry(
+            version_made_by=fields[1],
+            version_needed=fields[2],
+            flags=fields[3],
+            compression_method=fields[4],
+            mod_time=fields[5],
+            mod_date=fields[6],
+            crc32=fields[7],
+            compressed_size=fields[8],
+            uncompressed_size=fields[9],
+            disk_start=fields[13],
+            internal_attr=fields[14],
+            external_attr=fields[15],
+            header_offset=fields[16],
+            filename=filename,
+            extra=extra,
+            comment=comment,
+        )
+        entries.append(entry)
+        offset = var_end
+
+    return entries
+
+
+def parse_local_file_header(data: bytes) -> LocalHeaderInfo:
+    """Parse a local file header to determine the data offset.
+
+    Args:
+        data: At least LOCAL_FILE_HEADER_SIZE bytes of the local header.
+
+    Returns:
+        LocalHeaderInfo with filename and extra field lengths.
+
+    Raises:
+        BadZipFile: If the header is invalid.
+    """
+    if len(data) < LOCAL_FILE_HEADER_SIZE:
+        raise BadZipFile("Local file header is truncated")
+
+    fields = LOCAL_FILE_HEADER_STRUCT.unpack_from(data, 0)
+    # fields: signature(0), version_needed(1), flags(2), compression(3),
+    #         mod_time(4), mod_date(5), crc32(6), compressed_size(7),
+    #         uncompressed_size(8), filename_length(9), extra_length(10)
+
+    if fields[0] != LOCAL_FILE_HEADER_SIGNATURE:
+        raise BadZipFile("Invalid local file header signature")
+
+    return LocalHeaderInfo(
+        filename_length=fields[9],
+        extra_length=fields[10],
+    )
+
+
+def eocd_search_length(file_size: int) -> int:
+    """How many bytes to fetch from the end of the file for EOCD search."""
+    return min(MAX_EOCD_SEARCH, file_size)

zipwire/_sync.py

@@ -0,0 +1,163 @@
+"""Synchronous remote ZIP reader."""
+
+from __future__ import annotations
+
+import typing
+
+from zipwire._constants import LOCAL_FILE_HEADER_SIZE
+from zipwire._decompress import StreamingDecompressor, decompress
+from zipwire._errors import FileNotFoundInZip
+from zipwire._parser import (
+    eocd_search_length,
+    find_eocd,
+    parse_central_directory,
+    parse_local_file_header,
+)
+from zipwire._zipinfo import RemoteZipInfo
+
+if typing.TYPE_CHECKING:
+    from zipwire._types import SyncReader, Writable
+
+
+class SyncRemoteZip:
+    """Read files from a remote ZIP archive using synchronous HTTP range requests.
+
+    Usage::
+
+        with SyncRemoteZip(reader) as rz:
+            for info in rz.infolist():
+                print(info.filename)
+            data = rz.read("path/to/file.txt")
+    """
+
+    def __init__(self, reader: SyncReader) -> None:
+        self._reader = reader
+        self._entries: list[RemoteZipInfo] | None = None
+        self._name_index: dict[str, RemoteZipInfo] | None = None
+        self._file_size: int | None = None
+
+    def __enter__(self) -> SyncRemoteZip:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Close the underlying reader."""
+        self._reader.close()
+
+    def _ensure_loaded(self) -> None:
+        """Fetch and parse the central directory if not already done."""
+        if self._entries is not None:
+            return
+
+        # Step 1: Get file size
+        self._file_size = self._reader.get_content_length()
+
+        # Step 2: Fetch the tail for EOCD search
+        search_len = eocd_search_length(self._file_size)
+        tail_offset = self._file_size - search_len
+        tail = self._reader.read_range(tail_offset, search_len)
+
+        # Step 3: Parse EOCD
+        eocd = find_eocd(tail, self._file_size)
+
+        # Step 4: Fetch and parse central directory
+        cd_data = self._reader.read_range(eocd.cd_offset, eocd.cd_size)
+        raw_entries = parse_central_directory(cd_data, eocd.cd_entry_count)
+
+        # Step 5: Convert to RemoteZipInfo objects
+        self._entries = [RemoteZipInfo._from_central_dir_entry(e) for e in raw_entries]
+        self._name_index = {info.filename: info for info in self._entries}
+
+    def infolist(self) -> list[RemoteZipInfo]:
+        """Return a list of RemoteZipInfo objects for all files in the archive."""
+        self._ensure_loaded()
+        assert self._entries is not None
+        return list(self._entries)
+
+    def namelist(self) -> list[str]:
+        """Return a list of filenames in the archive."""
+        self._ensure_loaded()
+        assert self._entries is not None
+        return [info.filename for info in self._entries]
+
+    def getinfo(self, name: str) -> RemoteZipInfo:
+        """Return the RemoteZipInfo for the given filename.
+
+        Raises:
+            FileNotFoundInZip: If the name is not in the archive.
+        """
+        self._ensure_loaded()
+        assert self._name_index is not None
+        try:
+            return self._name_index[name]
+        except KeyError:
+            raise FileNotFoundInZip(name) from None
+
+    def read(self, name: str | RemoteZipInfo) -> bytes:
+        """Read and decompress a file from the archive.
+
+        Args:
+            name: Filename string or RemoteZipInfo object.
+
+        Returns:
+            The decompressed file contents.
+        """
+        info = name if isinstance(name, RemoteZipInfo) else self.getinfo(name)
+
+        # Directories have no data
+        if info.is_dir():
+            return b""
+
+        # Step 1: Read the local file header to get the actual data offset
+        local_header_data = self._reader.read_range(info.header_offset, LOCAL_FILE_HEADER_SIZE)
+        local_header = parse_local_file_header(local_header_data)
+
+        # Step 2: Calculate the data offset
+        data_offset = (
+            info.header_offset + LOCAL_FILE_HEADER_SIZE + local_header.data_offset_past_header
+        )
+
+        # Step 3: Fetch the compressed data
+        compressed_data = self._reader.read_range(data_offset, info.compress_size)
+
+        # Step 4: Decompress and verify CRC
+        return decompress(
+            compressed_data,
+            info.compress_type,
+            info.file_size,
+            info.CRC,
+        )
+
+    def read_into(self, name: str | RemoteZipInfo, dest: Writable) -> None:
+        """Decompress a file from the archive into a writable destination.
+
+        Unlike :meth:`read`, this truly streams: the compressed payload is
+        fetched in chunks via :meth:`stream_range` and each chunk is
+        decompressed and written immediately, keeping peak memory low.
+
+        Args:
+            name: Filename string or RemoteZipInfo object.
+            dest: A writable file-like object (e.g. ``io.BytesIO``, open file).
+        """
+        info = name if isinstance(name, RemoteZipInfo) else self.getinfo(name)
+
+        # Directories have no data
+        if info.is_dir():
+            return
+
+        # Step 1: Read the local file header to get the actual data offset
+        local_header_data = self._reader.read_range(info.header_offset, LOCAL_FILE_HEADER_SIZE)
+        local_header = parse_local_file_header(local_header_data)
+
+        # Step 2: Calculate the data offset
+        data_offset = (
+            info.header_offset + LOCAL_FILE_HEADER_SIZE + local_header.data_offset_past_header
+        )
+
+        # Step 3: Stream compressed data and decompress into dest
+        sd = StreamingDecompressor(info.compress_type, info.CRC, dest)
+        for chunk in self._reader.stream_range(data_offset, info.compress_size):
+            sd.feed(chunk)
+        sd.finish()

zipwire/_types.py

@@ -0,0 +1,57 @@
+"""Reader protocols for sync and async IO."""
+
+from __future__ import annotations
+
+import typing
+
+if typing.TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+
+
+@typing.runtime_checkable
+class SyncReader(typing.Protocol):
+    """Protocol for synchronous HTTP range-request readers."""
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        """Read *length* bytes starting at *offset*."""
+        ...
+
+    def stream_range(self, offset: int, length: int) -> Iterator[bytes]:
+        """Stream *length* bytes starting at *offset* as chunks."""
+        ...
+
+    def get_content_length(self) -> int:
+        """Return the total size of the remote resource in bytes."""
+        ...
+
+    def close(self) -> None:
+        """Release any held resources."""
+        ...
+
+
+@typing.runtime_checkable
+class AsyncReader(typing.Protocol):
+    """Protocol for asynchronous HTTP range-request readers."""
+
+    async def read_range(self, offset: int, length: int) -> bytes:
+        """Read *length* bytes starting at *offset*."""
+        ...
+
+    def stream_range(self, offset: int, length: int) -> AsyncIterator[bytes]:
+        """Stream *length* bytes starting at *offset* as chunks."""
+        ...
+
+    async def get_content_length(self) -> int:
+        """Return the total size of the remote resource in bytes."""
+        ...
+
+    async def close(self) -> None:
+        """Release any held resources."""
+        ...
+
+
+@typing.runtime_checkable
+class Writable(typing.Protocol):
+    """Protocol for writable file-like objects (io.BytesIO, io.BufferedWriter, etc.)."""
+
+    def write(self, data: bytes, /) -> object: ...

zipwire/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)
+
+__commit_id__ = commit_id = None

zipwire/_zipinfo.py

@@ -0,0 +1,93 @@
+"""ZipInfo subclass that is fully compatible with stdlib zipfile.ZipInfo."""
+
+from __future__ import annotations
+
+import struct
+import typing
+import zipfile
+
+from zipwire._constants import ZIP64_EXTRA_FIELD_ID
+
+if typing.TYPE_CHECKING:
+    from zipwire._parser import CentralDirEntry
+
+
+class RemoteZipInfo(zipfile.ZipInfo):
+    """A ZipInfo subclass for entries from a remote ZIP archive.
+
+    ``isinstance(info, zipfile.ZipInfo)`` returns True. All standard
+    attributes and methods (is_dir, FileHeader, etc.) are available.
+    """
+
+    @classmethod
+    def _from_central_dir_entry(cls, entry: CentralDirEntry) -> RemoteZipInfo:
+        """Construct a ZipInfo from a parsed central directory entry."""
+        # Decode filename
+        # Bit 11 of flags indicates UTF-8 encoding
+        if entry.flags & 0x800:
+            filename = entry.filename.decode("utf-8")
+        else:
+            filename = entry.filename.decode("cp437")
+
+        info = cls(filename)
+
+        # Convert DOS date/time to tuple
+        info.date_time = (
+            ((entry.mod_date >> 9) & 0x7F) + 1980,  # year
+            (entry.mod_date >> 5) & 0x0F,  # month
+            entry.mod_date & 0x1F,  # day
+            (entry.mod_time >> 11) & 0x1F,  # hour
+            (entry.mod_time >> 5) & 0x3F,  # minute
+            (entry.mod_time & 0x1F) * 2,  # second
+        )
+
+        info.compress_type = entry.compression_method
+        info.CRC = entry.crc32
+        info.compress_size = entry.compressed_size
+        info.file_size = entry.uncompressed_size
+        info.header_offset = entry.header_offset
+        info.internal_attr = entry.internal_attr
+        info.external_attr = entry.external_attr
+        info.create_system = (entry.version_made_by >> 8) & 0xFF
+        info.create_version = entry.version_made_by & 0xFF
+        info.extract_version = entry.version_needed
+        info.flag_bits = entry.flags
+        info.volume = entry.disk_start
+        info.extra = entry.extra
+        info.comment = entry.comment
+
+        # Handle ZIP64 extra field
+        _apply_zip64_extra(info)
+
+        return info
+
+
+def _apply_zip64_extra(info: RemoteZipInfo) -> None:
+    """Parse ZIP64 extra field and update sizes/offset if present."""
+    extra = info.extra
+    if not extra:
+        return
+
+    offset = 0
+    while offset + 4 <= len(extra):
+        header_id, data_size = struct.unpack_from("<HH", extra, offset)
+        offset += 4
+
+        if header_id == ZIP64_EXTRA_FIELD_ID:
+            # ZIP64 extended information extra field
+            # Fields appear in order only if the corresponding regular field is 0xFFFFFFFF
+            idx = 0
+            if info.file_size == 0xFFFFFFFF and idx + 8 <= data_size:
+                info.file_size = struct.unpack_from("<Q", extra, offset + idx)[0]
+                idx += 8
+            if info.compress_size == 0xFFFFFFFF and idx + 8 <= data_size:
+                info.compress_size = struct.unpack_from("<Q", extra, offset + idx)[0]
+                idx += 8
+            if info.header_offset == 0xFFFFFFFF and idx + 8 <= data_size:
+                info.header_offset = struct.unpack_from("<Q", extra, offset + idx)[0]
+                idx += 8
+            if info.volume == 0xFFFF and idx + 4 <= data_size:
+                info.volume = struct.unpack_from("<I", extra, offset + idx)[0]
+            return
+
+        offset += data_size

zipwire/backends/__init__.py

@@ -0,0 +1,42 @@
+"""Backend readers with lazy imports.
+
+No backend library is imported until its reader class is accessed.
+
+Usage::
+
+    from zipwire.backends import Httpx2SyncReader
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from zipwire.backends._aiohttp import AiohttpReader as AiohttpReader
+    from zipwire.backends._httpx2_async import Httpx2AsyncReader as Httpx2AsyncReader
+    from zipwire.backends._httpx2_sync import Httpx2SyncReader as Httpx2SyncReader
+    from zipwire.backends._requests import RequestsReader as RequestsReader
+    from zipwire.backends._urllib3 import Urllib3Reader as Urllib3Reader
+
+_LAZY_IMPORTS: dict[str, tuple[str, str]] = {
+    "Httpx2SyncReader": ("zipwire.backends._httpx2_sync", "Httpx2SyncReader"),
+    "Httpx2AsyncReader": ("zipwire.backends._httpx2_async", "Httpx2AsyncReader"),
+    "AiohttpReader": ("zipwire.backends._aiohttp", "AiohttpReader"),
+    "Urllib3Reader": ("zipwire.backends._urllib3", "Urllib3Reader"),
+    "RequestsReader": ("zipwire.backends._requests", "RequestsReader"),
+}
+
+__all__ = list(_LAZY_IMPORTS)
+
+
+def __getattr__(name: str) -> object:
+    if name in _LAZY_IMPORTS:
+        module_path, attr_name = _LAZY_IMPORTS[name]
+        module = importlib.import_module(module_path)
+        return getattr(module, attr_name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    return __all__