zipwire 0.0.1__py3-none-any.whl

zipwire/__init__.py

@@ -0,0 +1,33 @@
+"""zipwire - Read and extract files from remote ZIP archives over HTTP range requests."""
+
+from __future__ import annotations
+
+from zipwire._async import AsyncRemoteZip
+from zipwire._constants import CompressionMethod
+from zipwire._errors import (
+    BadZipFile,
+    CRCMismatch,
+    FileNotFoundInZip,
+    RangeRequestUnsupported,
+    UnsupportedCompression,
+    ZipwireError,
+)
+from zipwire._sync import SyncRemoteZip
+from zipwire._types import AsyncReader, SyncReader, Writable
+from zipwire._zipinfo import RemoteZipInfo
+
+__all__ = [
+    "AsyncReader",
+    "AsyncRemoteZip",
+    "BadZipFile",
+    "CRCMismatch",
+    "CompressionMethod",
+    "FileNotFoundInZip",
+    "RangeRequestUnsupported",
+    "RemoteZipInfo",
+    "SyncReader",
+    "SyncRemoteZip",
+    "UnsupportedCompression",
+    "Writable",
+    "ZipwireError",
+]

zipwire/__main__.py

@@ -0,0 +1,74 @@
+"""CLI entry-point: ``python -m zipwire <url>``."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+
+from zipwire import AsyncRemoteZip, SyncRemoteZip, ZipwireError
+
+
+def _print_table(infos: list) -> None:
+    print(f"{'Size':>10}  {'Date':>10}  {'Time':>8}  Name")
+    print(f"{'----':>10}  {'----':>10}  {'----':>8}  ----")
+    total_size = 0
+    for info in infos:
+        y, mo, d, h, mi, s = info.date_time
+        date_str = f"{y:04d}-{mo:02d}-{d:02d}"
+        time_str = f"{h:02d}:{mi:02d}:{s:02d}"
+        print(f"{info.file_size:>10}  {date_str:>10}  {time_str:>8}  {info.filename}")
+        total_size += info.file_size
+    print(f"{'----':>10}  {' ':>10}  {' ':>8}  ----")
+    print(f"{total_size:>10}  {' ':>10}  {' ':>8}  {len(infos)} entries")
+
+
+def _run_sync(url: str, backend: str) -> None:
+    from zipwire import backends
+
+    reader_cls = {
+        "urllib3": backends.Urllib3Reader,
+        "requests": backends.RequestsReader,
+    }[backend]
+    with SyncRemoteZip(reader_cls(url)) as rz:
+        _print_table(rz.infolist())
+
+
+async def _run_async(url: str, backend: str) -> None:
+    from zipwire import backends
+
+    reader_cls = {
+        "httpx2": backends.Httpx2AsyncReader,
+        "aiohttp": backends.AiohttpReader,
+    }[backend]
+    async with AsyncRemoteZip(reader_cls(url)) as rz:
+        _print_table(await rz.infolist())
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="python -m zipwire",
+        description="List files in a remote ZIP archive.",
+    )
+    parser.add_argument("url", help="URL of the remote ZIP archive")
+    parser.add_argument(
+        "-b",
+        "--backend",
+        choices=["urllib3", "requests", "httpx2", "aiohttp"],
+        default="urllib3",
+        help="HTTP backend to use (default: urllib3)",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        if args.backend in ("urllib3", "requests"):
+            _run_sync(args.url, args.backend)
+        else:
+            asyncio.run(_run_async(args.url, args.backend))
+    except ZipwireError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

zipwire/_async.py

@@ -0,0 +1,173 @@
+"""Asynchronous remote ZIP reader."""
+
+from __future__ import annotations
+
+import typing
+
+from zipwire._constants import LOCAL_FILE_HEADER_SIZE
+from zipwire._decompress import StreamingDecompressor, decompress
+from zipwire._errors import FileNotFoundInZip
+from zipwire._parser import (
+    eocd_search_length,
+    find_eocd,
+    parse_central_directory,
+    parse_local_file_header,
+)
+from zipwire._zipinfo import RemoteZipInfo
+
+if typing.TYPE_CHECKING:
+    from zipwire._types import AsyncReader, Writable
+
+
+class AsyncRemoteZip:
+    """Read files from a remote ZIP archive using async HTTP range requests.
+
+    Usage::
+
+        async with AsyncRemoteZip(reader) as rz:
+            for info in await rz.infolist():
+                print(info.filename)
+            data = await rz.read("path/to/file.txt")
+    """
+
+    def __init__(self, reader: AsyncReader) -> None:
+        self._reader = reader
+        self._entries: list[RemoteZipInfo] | None = None
+        self._name_index: dict[str, RemoteZipInfo] | None = None
+        self._file_size: int | None = None
+
+    async def __aenter__(self) -> AsyncRemoteZip:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.close()
+
+    async def close(self) -> None:
+        """Close the underlying reader."""
+        await self._reader.close()
+
+    async def _ensure_loaded(self) -> None:
+        """Fetch and parse the central directory if not already done."""
+        if self._entries is not None:
+            return
+
+        # Step 1: Get file size
+        self._file_size = await self._reader.get_content_length()
+
+        # Step 2: Fetch the tail for EOCD search
+        search_len = eocd_search_length(self._file_size)
+        tail_offset = self._file_size - search_len
+        tail = await self._reader.read_range(tail_offset, search_len)
+
+        # Step 3: Parse EOCD
+        eocd = find_eocd(tail, self._file_size)
+
+        # Step 4: Fetch and parse central directory
+        cd_data = await self._reader.read_range(eocd.cd_offset, eocd.cd_size)
+        raw_entries = parse_central_directory(cd_data, eocd.cd_entry_count)
+
+        # Step 5: Convert to RemoteZipInfo objects
+        self._entries = [RemoteZipInfo._from_central_dir_entry(e) for e in raw_entries]
+        self._name_index = {info.filename: info for info in self._entries}
+
+    async def infolist(self) -> list[RemoteZipInfo]:
+        """Return a list of RemoteZipInfo objects for all files in the archive."""
+        await self._ensure_loaded()
+        assert self._entries is not None
+        return list(self._entries)
+
+    async def namelist(self) -> list[str]:
+        """Return a list of filenames in the archive."""
+        await self._ensure_loaded()
+        assert self._entries is not None
+        return [info.filename for info in self._entries]
+
+    async def getinfo(self, name: str) -> RemoteZipInfo:
+        """Return the RemoteZipInfo for the given filename.
+
+        Raises:
+            FileNotFoundInZip: If the name is not in the archive.
+        """
+        await self._ensure_loaded()
+        assert self._name_index is not None
+        try:
+            return self._name_index[name]
+        except KeyError:
+            raise FileNotFoundInZip(name) from None
+
+    async def read(self, name: str | RemoteZipInfo) -> bytes:
+        """Read and decompress a file from the archive.
+
+        Args:
+            name: Filename string or RemoteZipInfo object.
+
+        Returns:
+            The decompressed file contents.
+        """
+        if isinstance(name, RemoteZipInfo):
+            info = name
+        else:
+            info = await self.getinfo(name)
+
+        # Directories have no data
+        if info.is_dir():
+            return b""
+
+        # Step 1: Read the local file header to get the actual data offset
+        local_header_data = await self._reader.read_range(
+            info.header_offset, LOCAL_FILE_HEADER_SIZE
+        )
+        local_header = parse_local_file_header(local_header_data)
+
+        # Step 2: Calculate the data offset
+        data_offset = (
+            info.header_offset + LOCAL_FILE_HEADER_SIZE + local_header.data_offset_past_header
+        )
+
+        # Step 3: Fetch the compressed data
+        compressed_data = await self._reader.read_range(data_offset, info.compress_size)
+
+        # Step 4: Decompress and verify CRC
+        return decompress(
+            compressed_data,
+            info.compress_type,
+            info.file_size,
+            info.CRC,
+        )
+
+    async def read_into(self, name: str | RemoteZipInfo, dest: Writable) -> None:
+        """Decompress a file from the archive into a writable destination.
+
+        Unlike :meth:`read`, this truly streams: the compressed payload is
+        fetched in chunks via :meth:`stream_range` and each chunk is
+        decompressed and written immediately, keeping peak memory low.
+
+        Args:
+            name: Filename string or RemoteZipInfo object.
+            dest: A writable file-like object (e.g. ``io.BytesIO``, open file).
+        """
+        if isinstance(name, RemoteZipInfo):
+            info = name
+        else:
+            info = await self.getinfo(name)
+
+        # Directories have no data
+        if info.is_dir():
+            return
+
+        # Step 1: Read the local file header to get the actual data offset
+        local_header_data = await self._reader.read_range(
+            info.header_offset, LOCAL_FILE_HEADER_SIZE
+        )
+        local_header = parse_local_file_header(local_header_data)
+
+        # Step 2: Calculate the data offset
+        data_offset = (
+            info.header_offset + LOCAL_FILE_HEADER_SIZE + local_header.data_offset_past_header
+        )
+
+        # Step 3: Stream compressed data and decompress into dest
+        sd = StreamingDecompressor(info.compress_type, info.CRC, dest)
+        async for chunk in self._reader.stream_range(data_offset, info.compress_size):
+            sd.feed(chunk)
+        sd.finish()

zipwire/_constants.py

@@ -0,0 +1,126 @@
+"""ZIP format constants: signatures, struct formats, and compression methods."""
+
+from __future__ import annotations
+
+import struct
+from enum import IntEnum
+
+# ---------------------------------------------------------------------------
+# Signatures
+# ---------------------------------------------------------------------------
+EOCD_SIGNATURE = b"PK\x05\x06"  # zipfile.stringEndArchive
+ZIP64_EOCD_LOCATOR_SIGNATURE = b"PK\x06\x07"  # zipfile.stringEndArchive64Locator
+ZIP64_EOCD_SIGNATURE = b"PK\x06\x06"  # zipfile.stringEndArchive64
+CENTRAL_DIR_SIGNATURE = b"PK\x01\x02"  # zipfile.stringCentralDir
+LOCAL_FILE_HEADER_SIGNATURE = b"PK\x03\x04"  # zipfile.stringFileHeader
+
+# ---------------------------------------------------------------------------
+# Fixed-part sizes
+# ---------------------------------------------------------------------------
+EOCD_SIZE = 22  # zipfile.sizeEndCentDir
+ZIP64_EOCD_LOCATOR_SIZE = 20  # zipfile.sizeEndCentDir64Locator
+ZIP64_EOCD_SIZE = 56  # zipfile.sizeEndCentDir64
+CENTRAL_DIR_SIZE = 46  # zipfile.sizeCentralDir
+LOCAL_FILE_HEADER_SIZE = 30  # zipfile.sizeFileHeader
+
+# ---------------------------------------------------------------------------
+# Compression methods
+# ---------------------------------------------------------------------------
+
+
+class CompressionMethod(IntEnum):
+    """ZIP compression method identifiers."""
+
+    STORED = 0  # zipfile.ZIP_STORED
+    DEFLATED = 8  # zipfile.ZIP_DEFLATED
+    BZIP2 = 12  # zipfile.ZIP_BZIP2
+    LZMA = 14  # zipfile.ZIP_LZMA
+    ZSTANDARD = 93  # zipfile.ZIP_ZSTANDARD (3.14+)
+
+
+# ---------------------------------------------------------------------------
+# Struct formats
+#
+# The stdlib defines struct formats too (e.g. zipfile.structCentralDir), but
+# they use different field groupings that shift unpack indices.  We define
+# our own for cleaner parser code.
+# ---------------------------------------------------------------------------
+
+# End of Central Directory Record (22 bytes)
+EOCD_STRUCT = struct.Struct(
+    "<4s"  # [0] signature
+    "H"  # [1] disk_num
+    "H"  # [2] disk_cd_start
+    "H"  # [3] entries_this_disk
+    "H"  # [4] entries_total
+    "I"  # [5] cd_size
+    "I"  # [6] cd_offset
+    "H"  # [7] comment_length
+)
+
+# ZIP64 End of Central Directory Locator (20 bytes)
+ZIP64_EOCD_LOCATOR_STRUCT = struct.Struct(
+    "<4s"  # [0] signature
+    "I"  # [1] disk_with_zip64_eocd
+    "Q"  # [2] zip64_eocd_offset
+    "I"  # [3] total_disks
+)
+
+# ZIP64 End of Central Directory Record (56 bytes)
+ZIP64_EOCD_STRUCT = struct.Struct(
+    "<4s"  # [0] signature
+    "Q"  # [1] record_size
+    "H"  # [2] version_made
+    "H"  # [3] version_needed
+    "I"  # [4] disk_num
+    "I"  # [5] disk_cd_start
+    "Q"  # [6] entries_this_disk
+    "Q"  # [7] entries_total
+    "Q"  # [8] cd_size
+    "Q"  # [9] cd_offset
+)
+
+# Central Directory File Header (46 bytes)
+CENTRAL_DIR_STRUCT = struct.Struct(
+    "<4s"  # [0] signature
+    "H"  # [1] version_made
+    "H"  # [2] version_needed
+    "H"  # [3] flags
+    "H"  # [4] compression
+    "H"  # [5] mod_time
+    "H"  # [6] mod_date
+    "I"  # [7] crc32
+    "I"  # [8] compressed_size
+    "I"  # [9] uncompressed_size
+    "H"  # [10] filename_len
+    "H"  # [11] extra_len
+    "H"  # [12] comment_len
+    "H"  # [13] disk_start
+    "H"  # [14] internal_attr
+    "I"  # [15] external_attr
+    "I"  # [16] header_offset
+)
+
+# Local File Header (30 bytes)
+LOCAL_FILE_HEADER_STRUCT = struct.Struct(
+    "<4s"  # [0] signature
+    "H"  # [1] version_needed
+    "H"  # [2] flags
+    "H"  # [3] compression
+    "H"  # [4] mod_time
+    "H"  # [5] mod_date
+    "I"  # [6] crc32
+    "I"  # [7] compressed_size
+    "I"  # [8] uncompressed_size
+    "H"  # [9] filename_len
+    "H"  # [10] extra_len
+)
+
+# ---------------------------------------------------------------------------
+# Derived / misc
+# ---------------------------------------------------------------------------
+MAX_EOCD_SEARCH = EOCD_SIZE + 65535  # max ZIP comment is 65 535 bytes
+
+ZIP64_EXTRA_FIELD_ID = 0x0001
+
+STREAM_CHUNK_SIZE = 2 * 1024 * 1024  # 2 MiB - default for stream_range backends

zipwire/_decompress.py

@@ -0,0 +1,146 @@
+"""Decompression and CRC verification for ZIP entries."""
+
+from __future__ import annotations
+
+import typing
+import zlib
+from enum import Enum
+
+from zipwire._constants import CompressionMethod
+from zipwire._errors import CRCMismatch, UnsupportedCompression
+
+if typing.TYPE_CHECKING:
+    from zipwire._types import Writable
+
+
+class _DecompressMode(Enum):
+    """Internal mode tags for StreamingDecompressor."""
+
+    STORED = "stored"
+    DEFLATED = "deflated"
+    INCREMENTAL = "incremental"
+
+
+def decompress(
+    data: bytes,
+    method: int,
+    expected_size: int,
+    expected_crc: int,
+) -> bytes:
+    """Decompress data and verify CRC-32.
+
+    Args:
+        data: The raw (possibly compressed) file data.
+        method: ZIP compression method (0=stored, 8=deflated, 12=bzip2, 14=lzma, 93=zstandard).
+        expected_size: Expected uncompressed size.
+        expected_crc: Expected CRC-32 of the uncompressed data.
+
+    Returns:
+        The decompressed bytes.
+
+    Raises:
+        UnsupportedCompression: If the compression method is not supported.
+        CRCMismatch: If the CRC-32 check fails.
+    """
+    match method:
+        case CompressionMethod.STORED:
+            result = data
+        case CompressionMethod.DEFLATED:
+            # -15 = raw deflate (no zlib/gzip header)
+            result = zlib.decompress(data, -15)
+        case CompressionMethod.BZIP2:
+            try:
+                import bz2
+            except ImportError:
+                raise UnsupportedCompression(method) from None
+
+            result = bz2.decompress(data)
+        case CompressionMethod.LZMA:
+            try:
+                import lzma
+            except ImportError:
+                raise UnsupportedCompression(method) from None
+
+            result = lzma.decompress(data)
+        case CompressionMethod.ZSTANDARD:
+            try:
+                import zstandard  # ty: ignore[unresolved-import]
+            except ImportError:
+                raise UnsupportedCompression(method) from None
+
+            result = zstandard.decompress(data)
+        case _:
+            raise UnsupportedCompression(method)
+
+    actual_crc = zlib.crc32(result) & 0xFFFFFFFF
+    if actual_crc != expected_crc:
+        raise CRCMismatch(expected_crc, actual_crc)
+
+    return result
+
+
+class StreamingDecompressor:
+    """Feed compressed chunks via :meth:`feed`, then call :meth:`finish` to verify CRC-32."""
+
+    def __init__(self, method: int, expected_crc: int, dest: Writable) -> None:
+        self._expected_crc = expected_crc
+        self._dest = dest
+        self._crc: int = 0
+        self._dobj: typing.Any = None
+
+        match method:
+            case CompressionMethod.STORED:
+                self._mode = _DecompressMode.STORED
+            case CompressionMethod.DEFLATED:
+                self._mode = _DecompressMode.DEFLATED
+                self._dobj = zlib.decompressobj(-15)
+            case CompressionMethod.BZIP2:
+                try:
+                    import bz2
+                except ImportError:
+                    raise UnsupportedCompression(method) from None
+
+                self._mode = _DecompressMode.INCREMENTAL
+                self._dobj = bz2.BZ2Decompressor()
+            case CompressionMethod.LZMA:
+                try:
+                    import lzma
+                except ImportError:
+                    raise UnsupportedCompression(method) from None
+
+                self._mode = _DecompressMode.INCREMENTAL
+                self._dobj = lzma.LZMADecompressor()
+            case CompressionMethod.ZSTANDARD:
+                try:
+                    import zstandard  # ty: ignore[unresolved-import]
+                except ImportError:
+                    raise UnsupportedCompression(method) from None
+
+                self._mode = _DecompressMode.INCREMENTAL
+                self._dobj = zstandard.ZstdDecompressor().decompressobj()
+            case _:
+                raise UnsupportedCompression(method)
+
+    def feed(self, data: bytes) -> None:
+        """Decompress *data*, write output to dest, update CRC."""
+        if self._mode is _DecompressMode.STORED:
+            self._crc = zlib.crc32(data, self._crc)
+            self._dest.write(data)
+        else:
+            # deflate, bz2, lzma, zstandard - all expose .decompress()
+            chunk = self._dobj.decompress(data)
+            if chunk:
+                self._crc = zlib.crc32(chunk, self._crc)
+                self._dest.write(chunk)
+
+    def finish(self) -> None:
+        """Flush remaining data (deflate only) and verify CRC-32."""
+        if self._mode is _DecompressMode.DEFLATED:
+            chunk = self._dobj.flush()
+            if chunk:
+                self._crc = zlib.crc32(chunk, self._crc)
+                self._dest.write(chunk)
+
+        actual_crc = self._crc & 0xFFFFFFFF
+        if actual_crc != self._expected_crc:
+            raise CRCMismatch(self._expected_crc, actual_crc)

zipwire/_errors.py

@@ -0,0 +1,40 @@
+"""Exception hierarchy for zipwire."""
+
+from __future__ import annotations
+
+
+class ZipwireError(Exception):
+    """Base exception for all zipwire errors."""
+
+
+class BadZipFile(ZipwireError):
+    """The data does not appear to be a valid ZIP archive."""
+
+
+class UnsupportedCompression(ZipwireError):
+    """The file uses an unsupported compression method."""
+
+    def __init__(self, method: int) -> None:
+        self.method = method
+        super().__init__(f"Unsupported compression method: {method}")
+
+
+class CRCMismatch(ZipwireError):
+    """CRC-32 check failed after decompression."""
+
+    def __init__(self, expected: int, actual: int) -> None:
+        self.expected = expected
+        self.actual = actual
+        super().__init__(f"CRC-32 mismatch: expected {expected:#010x}, got {actual:#010x}")
+
+
+class RangeRequestUnsupported(ZipwireError):
+    """The HTTP server does not support range requests."""
+
+
+class FileNotFoundInZip(ZipwireError, KeyError):
+    """The requested file was not found in the ZIP archive."""
+
+    def __init__(self, filename: str) -> None:
+        self.filename = filename
+        super().__init__(f"File not found in archive: {filename!r}")