rtzip 0.1.0__tar.gz

rtzip-0.1.0/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: rtzip
+Version: 0.1.0
+Requires-Python: >=3.12
+Requires-Dist: obstruct==0.5.2

rtzip-0.1.0/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "rtzip"
+version = "0.1.0"
+requires-python = ">=3.12"
+dependencies = [
+    "obstruct==0.5.2",
+]

rtzip-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

rtzip-0.1.0/src/rtzip/__init__.py

@@ -0,0 +1,3 @@
+from .models import EOCD, Zip64EOCD, CDEntry
+from .errors import *
+from .remotezip import RemoteZip

rtzip-0.1.0/src/rtzip/errors.py

@@ -0,0 +1,10 @@
+class UnsupportedAlgorithmError(Exception):
+    ...
+
+
+class UnsupportedCryptoError(Exception):
+    ...
+
+
+class UnsupportedDataDescriptorError(Exception):
+    ...

rtzip-0.1.0/src/rtzip/models.py

@@ -0,0 +1,244 @@
+import dataclasses
+import struct
+from collections.abc import Buffer
+from typing import Annotated, Self
+
+import obstruct as obs
+
+EXTRA_FIELD_HEADER = struct.Struct("<HH")
+
+
+def parse_extra_fields(buffer):
+    res = {}
+
+    buffer = memoryview(buffer)
+    length = buffer.nbytes
+    offset = 0
+
+    while offset + 4 <= length:
+        header_id, n_data = EXTRA_FIELD_HEADER.unpack_from(buffer, offset)
+
+        offset += 4
+        res[header_id] = buffer[offset: offset + n_data].tobytes()
+        offset += n_data
+
+    return res
+
+
+@dataclasses.dataclass(slots=True)
+class EOCD(obs.Struct):
+    disk_num: obs.u16
+    cd_disk: obs.u16
+    disk_entries: obs.u16
+    total_entries: obs.u16
+    cd_size: obs.u32
+    cd_offset: obs.u32
+    comment_length: obs.u16
+
+    @classmethod
+    def from_buffer(cls, buffer: Buffer, offset=4):
+        return cls.unpack_from(buffer, offset)
+
+
+@dataclasses.dataclass(slots=True)
+class Zip64EOCD(obs.Struct):
+    # 签名部分已忽略
+    record_size: obs.u64
+    version: obs.u16
+    unzip_min_version: obs.u16
+    disk_id: obs.u32
+    cd_disk: obs.u32
+    disk_entries: obs.u64
+    total_entries: obs.u64
+    cd_size: obs.u64
+    cd_offset: obs.u64
+    extra_data: Annotated[bytes, obs.Marks.NOT_A_CFIELD]
+
+    @property
+    def extra_offset(self):
+        return self.__cstruct__.size
+
+    @property
+    def extra_length(self):
+        return self.record_size - self.__cstruct__.size
+
+    @classmethod
+    def from_buffer(cls, buffer, offset=4, header_only: bool = False):
+        self = cls.unpack_from(buffer, offset)
+
+        extra_offset = self.extra_offset + offset
+        extra_length = self.extra_length
+
+        if not header_only:
+            self.extra_data = buffer[extra_offset: extra_offset + extra_length]
+        return self
+
+
+@dataclasses.dataclass(slots=True)
+class CDEntry(obs.Struct):
+    version: obs.u16
+    unzip_min_version: obs.u16
+    flags: obs.u16
+    algorithm: obs.u16
+    last_modified: obs.u16
+    last_modified_date: obs.u16
+    crc32_raw: obs.u32
+    compressed_size: obs.u32
+    original_size: obs.u32
+    filename_len: obs.u16
+    extra_len: obs.u16
+    comment_len: obs.u16
+    disk_id: obs.u16
+    inner_file_attrs: obs.u16
+    outer_file_attrs: obs.u32
+    local_header_offset: obs.u32
+
+    filename: Annotated[bytes, obs.Marks.NOT_A_CFIELD]
+    raw_extra_fields: Annotated[bytes, obs.Marks.NOT_A_CFIELD]
+    comment: Annotated[bytes, obs.Marks.NOT_A_CFIELD]
+
+    _extra_fields: Annotated[dict | None, obs.Marks.NOT_A_CFIELD] = dataclasses.field(default=None, repr=False)
+
+    @property
+    def is_encrypted(self):
+        return bool(self.flags & 0x0001)
+
+    @property
+    def is_zip64(self):
+        return self.compressed_size == 0xFFFFFFFF or self.original_size == 0xFFFFFFFF
+
+    @property
+    def has_data_descriptor(self):
+        return bool(self.flags & 0b100)
+
+    @property
+    def extra_fields(self) -> dict[int, bytes]:
+        if getattr(self, '_extra_fields', None) is None:
+            self._extra_fields = parse_extra_fields(self.raw_extra_fields)
+
+        return self._extra_fields
+
+    def fix_by_zip64(self):
+        zip64_data = (self.extra_fields.get(1))
+        offset = 0
+        if zip64_data:
+            zip64_data = memoryview(zip64_data)
+            length = len(zip64_data)
+            if self.original_size == 0xFFFFFFFF:
+                assert offset + 8 <= length
+                self.original_size = int.from_bytes(zip64_data[offset:offset + 8], 'little')
+                offset += 8
+            if self.compressed_size == 0xFFFFFFFF:
+                assert offset + 8 <= length
+                self.compressed_size = int.from_bytes(zip64_data[offset:offset + 8], 'little')
+                offset += 8
+            if self.local_header_offset == 0xFFFFFFFF:
+                assert offset + 8 <= length
+                self.local_header_offset = int.from_bytes(zip64_data[offset: offset + 8], 'little')
+                offset += 8
+            if self.disk_id == 0xFFFF:
+                assert offset + 4 <= length
+                self.disk_id = int.from_bytes(zip64_data[offset: offset + 4], 'little')
+
+    @classmethod
+    def from_buffer(cls, buffer: Buffer, offset=4) -> tuple[Self, int]:
+        buffer = memoryview(buffer)
+        # print(buffer.tobytes())
+        header_size = cls.__cstruct__.size
+
+        self = cls.unpack_from(buffer, offset)
+        filename_len, extra_len, comment_len = self.filename_len, self.extra_len, self.comment_len
+
+        pos = offset + header_size
+        filename = bytes(buffer[pos: pos + filename_len])
+        pos += filename_len
+
+        extra_attrs = buffer[pos: pos + extra_len].tobytes()
+        pos += extra_len
+
+        comment = buffer[pos: pos + comment_len].tobytes()
+        pos += comment_len
+
+        self.filename = filename
+        self.raw_extra_fields = extra_attrs
+        self.comment = comment
+
+        # print(self)
+
+        return (
+            self,
+            pos - offset
+        )
+
+
+@dataclasses.dataclass()
+class LocalFileHeader(obs.Struct):
+    unzip_min_version: obs.u16
+    flags: obs.u16
+    algorithm: obs.u16
+    last_modified_time: obs.u16
+    last_modified_date: obs.u16
+    crc32: obs.u32
+    compressed_size: obs.u32
+    uncompressed_size: obs.u32
+    filename_len: obs.u16
+    extra_len: obs.u16
+
+    filename: Annotated[bytes, obs.Marks.NOT_A_CFIELD]
+    raw_extra_fields: Annotated[bytes, obs.Marks.NOT_A_CFIELD]
+
+    _extra_fields: Annotated[dict[int, bytes] | None, obs.Marks.NOT_A_CFIELD] = None
+
+    @property
+    def extra_fields(self):
+        if self._extra_fields is None:
+            self._extra_fields = parse_extra_fields(self.raw_extra_fields)
+
+        return self._extra_fields
+
+    def fix_by_zip64(self):
+        zip64_data = self.extra_fields.get(0x0001)
+        if zip64_data:
+            original_size, compressed_size = struct.unpack('<QQ', zip64_data)
+            self.uncompressed_size = original_size
+            self.compressed_size = compressed_size
+
+    @classmethod
+    def from_buffer(cls, buffer: Buffer, offset: int = 4, header_only=False) -> tuple[Self, int]:
+        """从缓冲区解析 Local File Header，返回 (实例, 消耗字节数)"""
+        buf = memoryview(buffer)
+        header_size = cls.__cstruct__.size
+
+        # 解析固定头部
+        self = cls.unpack_from(buf, offset)
+
+        if header_only:
+            return self, header_size
+
+        # 提取长度字段
+        filename_len = self.filename_len
+        extra_len = self.extra_len
+
+        pos = offset + header_size
+
+        # 读取文件名
+        filename = bytes(buf[pos:pos + filename_len])
+        pos += filename_len
+
+        # 读取额外字段
+        extra_fields = bytes(buf[pos:pos + extra_len])
+        pos += extra_len
+
+        self.filename = filename
+        self.raw_extra_fields = extra_fields
+
+        return (
+            self,
+            pos - offset
+        )
+
+    @property
+    def data_offset(self) -> int:
+        """返回实际文件数据的起始偏移量（相对于 Local File Header 起始位置）"""
+        # 固定头 30 字节 + 文件名长度 + 额外字段长度
+        return self.__cstruct__.size + self.filename_len + self.extra_len + 4

rtzip-0.1.0/src/rtzip/remotezip.py

@@ -0,0 +1,238 @@
+import struct
+from typing import AsyncGenerator
+
+from .errors import UnsupportedAlgorithmError, UnsupportedCryptoError, UnsupportedDataDescriptorError
+from .models import EOCD, Zip64EOCD, CDEntry, LocalFileHeader
+from .utils import deflate_wrapper, single_chunk_wrapper
+
+
+class DataSourceCallbacks:
+    async def read_range(self, offset, length) -> bytes | bytearray | memoryview:
+        ...
+
+    def read_range_stream(self, offset, length) -> AsyncGenerator[bytes | bytearray | memoryview]:
+        ...
+
+    async def get_total_size(self) -> int:
+        ...
+
+
+class RemoteZip:
+    EOCD_SIGNATURE = b'PK\x05\x06'
+    CDENTITY_SIGNATURE = b'PK\x01\x02'
+    LFH_SIGNATURE = b'PK\x03\x04'
+    ZIP64_EOCD_LOCATOR_SIGNATURE = b'PK\x06\x07'
+    ZIP64_EOCD_SIGNATURE = b'PK\x06\x06'
+
+    async def _read_range(self, offset, length):
+        return await self.source.read_range(offset, length)
+
+    def _read_range_stream(self, offset, length):
+        return self.source.read_range_stream(offset, length)
+
+    async def _get_total_size(self):
+        return await self.source.get_total_size()
+
+    def __init__(self, source: DataSourceCallbacks):
+        self.source = source
+
+        self.cd_info: EOCD | Zip64EOCD | None = None
+        self.files: list[CDEntry] = []
+        self.file_mapping: dict[str, CDEntry] | None = None
+
+        self.is_zip64 = False
+
+    async def fetch_eocd(self, initial_chunk=4096, max_cnt=-1, loss_factor: int | float = 2):
+        chunk_size = initial_chunk
+
+        last_start = await self._get_total_size()
+        cnt = 0
+
+        buffer = b''
+        while last_start - chunk_size > 0:
+            cnt += 1
+            content = await self._read_range(last_start - chunk_size, chunk_size)
+
+            buffer = content + buffer[:32]
+
+            signature_pos = buffer.rfind(RemoteZip.EOCD_SIGNATURE)
+            if signature_pos != -1:
+                self.cd_info = eocd = EOCD.from_buffer(buffer[signature_pos:])
+                # return cnt
+                if eocd.total_entries == 0xFFFFFFFF or eocd.cd_offset == 0xFFFFFFFF or eocd.cd_size == 0xFFFFFFFF:
+                    self.is_zip64 = True
+
+                    # 继续获取 zip64 的 eocd
+                    # zip64 locator 在 eocd 前的 20 字节处
+                    # 字段包括 4 字节的签名，两个 8 字节数据
+
+                    zip64_locator_offset = signature_pos - 20
+                    # 检查需要的数据是否已经存在
+                    if zip64_locator_offset < 0:
+                        # print("--------------------Zero Padding")
+                        buffer = (
+                                await self._read_range(
+                                    last_start - chunk_size - abs(zip64_locator_offset),
+                                    abs(zip64_locator_offset)
+                                )
+                                + buffer
+                        )
+
+                        zip64_locator_offset = 0
+
+                    # print(buffer)
+
+                    assert buffer[zip64_locator_offset:zip64_locator_offset + 4] == self.ZIP64_EOCD_LOCATOR_SIGNATURE
+                    zip64_eocd_disk_id, zip64_eocd_offset, zip64_total_disks = \
+                        struct.unpack_from('<4xIQI', buffer, zip64_locator_offset)
+
+                    # print(zip64_eocd_disk_id, zip64_eocd_offset, zip64_total_disks)
+
+                    zip64_eocd_data = await self._read_range(zip64_eocd_offset, 96)
+                    # print(zip64_eocd_data[:64])
+                    assert zip64_eocd_data[:4] == self.ZIP64_EOCD_SIGNATURE
+
+                    zip64_eocd = Zip64EOCD.from_buffer(zip64_eocd_data, header_only=True)
+                    # print(zip64_eocd)
+
+                    extra_offset = zip64_eocd.extra_offset + 4
+                    extra_length = zip64_eocd.extra_length
+                    extra_end = extra_offset + extra_length
+
+                    # print(extra_offset, extra_end, buffer[extra_offset:extra_end])
+
+                    if extra_end > len(zip64_eocd_data):
+                        zip64_eocd_data += await self._read_range(
+                            zip64_eocd_offset + 64,
+                            extra_end - len(zip64_eocd_data)
+                        )
+
+                    extra_data = zip64_eocd_data[extra_offset:extra_offset + extra_length]
+
+                    zip64_eocd.extra_data = extra_data
+
+                    # print(zip64_eocd)
+
+                    self.cd_info = zip64_eocd
+                    return cnt
+                else:
+                    return cnt
+
+            if max_cnt != -1 and cnt >= max_cnt:
+                raise RuntimeError
+
+            last_start -= chunk_size
+            chunk_size = int(chunk_size * loss_factor)
+        else:
+            raise RuntimeError
+
+    async def fetch_cd(self, ignore_extra: bool = False):
+        assert self.cd_info
+        eocd = self.cd_info
+        buffer = bytearray()
+        files = []
+
+        def consume_entry():
+            nonlocal buffer
+            entry, nbytes = CDEntry.from_buffer(buffer)
+
+            if self.is_zip64:
+                entry.fix_by_zip64()
+
+            files.append(entry)
+            buffer = buffer[nbytes + 4:]
+
+        async for chunk in self._read_range_stream(eocd.cd_offset, eocd.cd_size):
+            # print(chunk)
+            buffer.extend(chunk)
+
+            while buffer and buffer[:4] == self.CDENTITY_SIGNATURE:
+                try:
+                    consume_entry()
+                except struct.error:
+                    continue
+
+        if buffer and not ignore_extra:
+            # print("Read Entries: ", len(files))
+            # print("Buffer:", buffer)
+            raise ValueError()
+
+        self.files = files
+
+    async def fetch_single_file_header(self, filename=None, entry: CDEntry | None = None):
+        filename = str(filename)
+        entry = entry or self.find_file_entry(filename)
+        buffer = await self._read_range(entry.local_header_offset, entry.__cstruct__.size + 64)  # 预留64字节的变长字段
+
+        # 仅头部模式解析 LocalFileHeader
+        header, _ = LocalFileHeader.from_buffer(buffer, 4, header_only=True)
+        if header.extra_len + header.filename_len > 64:
+            _ = await self._read_range(entry.local_header_offset + 64, header.extra_len + header.filename_len - 64)
+            buffer = bytearray(buffer) + _
+
+        pos = header.__cstruct__.size + 4  # 4 字节的签名在计算 nbytes 时会被忽略，实际需要加上
+        filename = buffer[pos: pos + header.filename_len]
+        pos += header.filename_len
+
+        extra_fields = buffer[pos:pos + header.extra_len]
+        pos += header.extra_len
+
+        header.filename = filename
+        header.raw_extra_fields = extra_fields
+
+        header.fix_by_zip64()
+        return header, buffer[pos:]  # 返回剩余的数据部分
+
+    def build_mapping(self):
+        m = self.file_mapping = {}
+        for entry in self.files:
+            m[entry.filename] = entry
+
+    def find_file_entry(self, filename):
+        if self.file_mapping:
+            return self.file_mapping[filename]
+        else:
+            for i in self.files:
+                if i.filename == filename:
+                    return i
+            else:
+                raise FileNotFoundError(filename)
+
+    async def _stream_single_file(self, filename):
+        entry = self.find_file_entry(filename)
+
+        if entry.is_encrypted:
+            raise UnsupportedCryptoError()
+
+        if entry.has_data_descriptor:
+            raise UnsupportedDataDescriptorError()
+
+        if entry.algorithm not in {0x0000, 0x0008}:
+            raise UnsupportedAlgorithmError(entry.algorithm)
+
+        header, buffer = await self.fetch_single_file_header(entry=entry)
+        # print(header)
+        real_data_offset = entry.local_header_offset + header.data_offset
+
+        if entry.compressed_size < len(buffer):
+            raw_generator = single_chunk_wrapper(buffer[:entry.compressed_size])
+        else:
+            raw_generator = self._read_range_stream(real_data_offset, entry.compressed_size)
+
+        match entry.algorithm:
+            case 0x0000:
+                return raw_generator
+            case 0x0008:
+                return deflate_wrapper(raw_generator)
+
+    async def stream_single_file(self, filename):
+        gen = await self._stream_single_file(filename)
+        async for i in gen:
+            yield i
+
+    async def load_single_file(self, filename) -> bytearray:
+        buffer = bytearray()
+        async for i in self.stream_single_file(filename):
+            buffer.extend(i)
+
+        return buffer

rtzip-0.1.0/src/rtzip/utils.py

@@ -0,0 +1,15 @@
+import zlib
+from collections.abc import Buffer
+
+
+async def deflate_wrapper(gen):
+    decompresser = zlib.decompressobj(-15)
+    async for chunk in gen:
+        yield decompresser.decompress(chunk)
+    remaining = decompresser.flush()
+    if remaining:
+        yield remaining
+
+
+async def single_chunk_wrapper(data: Buffer):
+    yield data

rtzip-0.1.0/src/rtzip.egg-info/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: rtzip
+Version: 0.1.0
+Requires-Python: >=3.12
+Requires-Dist: obstruct==0.5.2

rtzip-0.1.0/src/rtzip.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+pyproject.toml
+src/rtzip/__init__.py
+src/rtzip/errors.py
+src/rtzip/models.py
+src/rtzip/remotezip.py
+src/rtzip/utils.py
+src/rtzip.egg-info/PKG-INFO
+src/rtzip.egg-info/SOURCES.txt
+src/rtzip.egg-info/dependency_links.txt
+src/rtzip.egg-info/requires.txt
+src/rtzip.egg-info/top_level.txt

rtzip-0.1.0/src/rtzip.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

rtzip-0.1.0/src/rtzip.egg-info/requires.txt

@@ -0,0 +1 @@
+obstruct==0.5.2

rtzip-0.1.0/src/rtzip.egg-info/top_level.txt

@@ -0,0 +1 @@
+rtzip