rtzip 0.2.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rtzip
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Requires-Python: >=3.12
5
5
  License-File: LICENSE
6
6
  Requires-Dist: obstruct==0.5.2
rtzip-0.3.0/README.md ADDED
@@ -0,0 +1,119 @@
1
+ # Rtzip
2
+
3
+ _一个以最小的数据量提取远程 zip 文件内容的库_
4
+
5
+ ## 特点
6
+
7
+ - 不与任何 HTTP 库耦合
8
+ - 依赖简单(只有用于解析二进制结构的 `obstruct`)
9
+ - 兼容标准 zip 格式、ZIP64 扩展
10
+
11
+ ## 安装
12
+
13
+ ```commandline
14
+ pip install rtzip
15
+ ```
16
+
17
+ ## 用法
18
+
19
+ ### 直接使用
20
+
21
+ ```python
22
+ from rtzip import RemoteZip, DataSource
23
+
24
+ import niquests # 如果你使用 niquests 作为 http 依赖的话
25
+
26
+
27
+ class HTTPSource(DataSource):
28
+ def __init__(self, url, session=None):
29
+ self.url = url
30
+ self.session = session or niquests.AsyncSession
31
+
32
+ async def _request_range(self, offset, length):
33
+ resp = await self.session.get(self.url, headers={'Range': f"bytes={offset}-{offset + length - 1}"}, stream=True)
34
+ resp.raise_for_status()
35
+ if resp.status_code != 206:
36
+ await resp.close()
37
+ raise niquests.HTTPError(response=resp)
38
+ return resp
39
+
40
+ async def read_range(self, offset, length):
41
+ return await (await self._request_range(offset, length)).content
42
+
43
+ async def read_range_stream(self, offset, length):
44
+ async for i in await (await self._request_range(offset, length)).iter_content():
45
+ yield i
46
+
47
+ async def get_total_size(self):
48
+ resp = await self.session.head(self.url)
49
+ resp.raise_for_status()
50
+ return int(resp['content-length'])
51
+
52
+
53
+ async def main():
54
+ url = ...
55
+ rz = RemoteZip(HTTPSource(url))
56
+ rz.fetch_zip_info() # 加载文件的 EOCD,返回一个 EOCD 获取 Zip64EOCD 对象
57
+ rz.fetch_file_list() # 加载文件的中央目录,返回一个包含 CDEntry 对象的列表
58
+ # 这些数据如果没有即时接收,也会被缓存在 rz.eocd 和 rz.files 中
59
+
60
+ rz.build_mapping() # 构建文件名 -> Entry 对象的映射表,方便查表
61
+ # 返回映射表本身,同时缓存在 rz.file_mapping 中
62
+
63
+ # 一次性读取一个文件的全量数据
64
+ data = await rz.load_single_file(b'filename')
65
+
66
+ # 流式读取一个文件的数据
67
+ buffer = bytearray()
68
+ async for i in rz.stream_single_file(b'filename'):
69
+ buffer.extend(i)
70
+
71
+ # 获取一个文件的元数据信息
72
+ entry = rz.find_file_entry(b'filename')
73
+ # 可能抛出 FileNotFoundError
74
+
75
+ # 从文件名拉取一个文件对应的本地文件头
76
+ header = await rz.fetch_file_header(filename=b'filename')
77
+
78
+ # 也可以从已有的 entry 对象拉取
79
+ # 但这时你要自己保证这个 entry 对象是有效的
80
+ # 将返回一个 LocalFileHeader 对象
81
+ header = await rz.fetch_file_header(entry=entry)
82
+
83
+
84
+ if __name__ == '__main__':
85
+ import asyncio
86
+
87
+ asyncio.run(main())
88
+
89
+ ```
90
+
91
+ ### 自定义压缩算法处理
92
+
93
+ ```python
94
+ # 示例:扩展 PPMd 算法支持(需要 pip install ppmd)
95
+ import ppmd
96
+ from rtzip import algorithm_handler
97
+
98
+
99
+ @algorithm_handler(0x000A)
100
+ async def ppmd_wrapper(raw_generator):
101
+ # PPMd 流式解压(API 取决于具体库)
102
+ decomp = ppmd.Ppmd7Decompressor()
103
+ async for chunk in raw_generator:
104
+ yield decomp.decompress(chunk)
105
+
106
+
107
+ # 这之后使用 RemoteZip 时遇到 ppmd 算法会自动应用你的包装器
108
+
109
+ # 如果想要重写某个内置的包装器
110
+ # 你需要指定 `override=True`,否则程序将会阻止覆盖行为
111
+ @algorithm_handler(0x0008, override=True)
112
+ async def your_deflate_wrapper(raw_generator):
113
+ ...
114
+
115
+ ```
116
+
117
+ > [!TIP]
118
+ > 本库无计划新功能,若是让自行添加功能,可克隆仓库并重写 `rtzip/remotezip.py` 下的代码
119
+ > 大多数库的功能已完成并可以直接投入使用
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "rtzip"
3
- version = "0.2.2"
3
+ version = "0.3.0"
4
4
  requires-python = ">=3.12"
5
5
  dependencies = [
6
6
  "obstruct==0.5.2",
@@ -0,0 +1,4 @@
1
+ from .algorithm_register import algorithm_handler, get_algorithm_handler
2
+ from .errors import *
3
+ from .models import EOCD, Zip64EOCD, CDEntry
4
+ from .remotezip import RemoteZip, DataSource
@@ -0,0 +1,45 @@
1
+ from collections.abc import Buffer
2
+ from typing import Callable, AsyncGenerator
3
+
4
+ AlgorithmHandler = Callable[[AsyncGenerator[Buffer, None]], AsyncGenerator[bytes | bytearray | memoryview, None]]
5
+
6
+ algorithm_mapping: dict[int, AlgorithmHandler] = {}
7
+
8
+
9
+ def algorithm_handler(algorithm: int, override=False):
10
+ """
11
+ 压缩流包装器注册装饰器
12
+
13
+ :param algorithm: 压缩算法 ID
14
+ :param override: 如果你希望覆盖一个已注册的包装器,请设置为 True,否则将会阻止可能的覆盖行为
15
+ :param decorator.func: 被包装的函数,接受一个异步生成器,返回一个异步生成器
16
+ :return:
17
+ """
18
+ if not override and algorithm in algorithm_mapping:
19
+ raise ValueError(f"Algorithm {algorithm} is already registered")
20
+
21
+ def decorator(func: AlgorithmHandler):
22
+ algorithm_mapping[algorithm] = func
23
+ return func
24
+
25
+ return decorator
26
+
27
+
28
+ def get_algorithm_handler(algorithm: int) -> AlgorithmHandler:
29
+ """
30
+ 获取包装器函数
31
+
32
+ :param algorithm: 压缩算法 ID
33
+ :return: AlgorithmHandler 可调用对象,接受一个异步生成器,返回一个异步生成器
34
+ """
35
+ return algorithm_mapping[algorithm]
36
+
37
+
38
+ def is_algorithm_available(algorithm: int) -> bool:
39
+ """
40
+ 判断某个压缩算法是否已有注册的包装器
41
+
42
+ :param algorithm: 压缩算法 ID
43
+ :return:
44
+ """
45
+ return algorithm in algorithm_mapping
@@ -0,0 +1,90 @@
1
+ import sys
2
+
3
+ from .algorithm_register import algorithm_handler
4
+
5
+
6
+ @algorithm_handler(0x0008)
7
+ async def deflate_wrapper(gen):
8
+ import zlib
9
+ decompresser = zlib.decompressobj(-15) # 禁用 zlib 头部
10
+ async for chunk in gen:
11
+ yield decompresser.decompress(chunk)
12
+ remaining = decompresser.flush()
13
+ if remaining:
14
+ yield remaining
15
+
16
+
17
+ @algorithm_handler(0x0000)
18
+ async def raw_wrapper(gen):
19
+ async for chunk in gen:
20
+ yield chunk
21
+
22
+
23
+ @algorithm_handler(0x000E)
24
+ async def lzma_wrapper(gen):
25
+ import lzma
26
+ decompressor = lzma.LZMADecompressor()
27
+
28
+ async for chunk in gen:
29
+ data = decompressor.decompress(chunk)
30
+ if data:
31
+ yield data
32
+
33
+ if not decompressor.eof:
34
+ raise lzma.LZMAError(
35
+ 'LZMA compressed data stream ended unexpectedly (EOF not reached) – data may be truncated or corrupted')
36
+
37
+ if decompressor.unused_data:
38
+ raise lzma.LZMAError(
39
+ f'LZMA stream contains {len(decompressor.unused_data)} bytes of trailing data after the end marker'
40
+ f' – possibly concatenated streams or extra metadata'
41
+ )
42
+
43
+ @algorithm_handler(0x000C)
44
+ async def bz2_wrapper(gen):
45
+ import bz2
46
+ decompressor = bz2.BZ2Decompressor()
47
+
48
+ async for chunk in gen:
49
+ if chunk:
50
+ data = decompressor.decompress(chunk)
51
+ if data:
52
+ yield data
53
+
54
+ if not decompressor.eof:
55
+ raise ValueError(
56
+ "BZIP2 compressed data stream ended unexpectedly (EOF not reached) – "
57
+ "data may be truncated or corrupted"
58
+ )
59
+
60
+ if decompressor.unused_data:
61
+ raise ValueError(
62
+ f"BZIP2 stream contains {len(decompressor.unused_data)} bytes of trailing data after the end marker – "
63
+ f"possibly concatenated streams or extra metadata"
64
+ )
65
+
66
+
67
+ if sys.version_info >= (3, 14):
68
+ @algorithm_handler(0x000D)
69
+ async def zstd_wrapper(gen): # Not Tested
70
+ import warnings
71
+ warnings.warn("ZStandard support is experimental and has not been tested")
72
+
73
+ import compression.zstd as zstd
74
+ decompressor = zstd.ZstdDecompressor()
75
+ async for chunk in gen:
76
+ data = decompressor.decompress(chunk)
77
+ if data:
78
+ yield data
79
+
80
+ if not decompressor.eof:
81
+ raise ValueError(
82
+ "ZStandard compressed data stream ended unexpectedly (EOF not reached) – "
83
+ "data may be truncated or corrupted"
84
+ )
85
+
86
+ if decompressor.unused_data:
87
+ raise ValueError(
88
+ f"ZStandard stream contains {len(decompressor.unused_data)} bytes of trailing data after the end marker – "
89
+ f"possibly concatenated streams or extra metadata"
90
+ )
@@ -1,14 +1,14 @@
1
1
  import struct
2
+ from collections.abc import Buffer
2
3
  from typing import AsyncGenerator
3
4
 
5
+ from .algorithm_register import is_algorithm_available, get_algorithm_handler
4
6
  from .const import *
5
7
  from .errors import UnsupportedAlgorithmError, UnsupportedCryptoError, UnsupportedDataDescriptorError
6
8
  from .models import EOCD, Zip64EOCD, CDEntry, LocalFileHeader
7
- from .utils import deflate_wrapper, single_chunk_wrapper
8
9
 
9
10
 
10
-
11
- class DataSourceCallbacks:
11
+ class DataSource:
12
12
  async def read_range(self, offset, length) -> bytes | bytearray | memoryview:
13
13
  """
14
14
  读取指定的一段数据
@@ -36,6 +36,16 @@ class DataSourceCallbacks:
36
36
  """
37
37
 
38
38
 
39
+ def exactly_get_slice(__obj, __offset, __length):
40
+ result = __obj[__offset:__offset + __length]
41
+ assert len(result) == __length
42
+ return result
43
+
44
+
45
+ async def single_chunk_wrapper(__data: Buffer, /):
46
+ yield __data
47
+
48
+
39
49
  class RemoteZip:
40
50
  """
41
51
  远程 Zip 文件访问器
@@ -58,7 +68,7 @@ class RemoteZip:
58
68
  async def _get_total_size(self):
59
69
  return await self.source.get_total_size()
60
70
 
61
- def __init__(self, source: DataSourceCallbacks):
71
+ def __init__(self, source: DataSource):
62
72
  self.source = source
63
73
 
64
74
  self.cd_info: EOCD | Zip64EOCD | None = None
@@ -74,7 +84,7 @@ class RemoteZip:
74
84
  :param initial_chunk: 初始扫描的大小
75
85
  :param max_cnt: 最大重试次数
76
86
  :param loss_factor: 重试后更新扫描大小的系数
77
- :return: 一个 EOCD 或者 Zip64EOCD 对象(如果这个 zip 文件启动了 zip64 扩展)
87
+ :return: 一个 EOCD 或者 Zip64EOCD 对象(如果这个 zip 文件启用了 zip64 扩展)
78
88
  """
79
89
  chunk_size = initial_chunk
80
90
 
@@ -279,7 +289,7 @@ class RemoteZip:
279
289
  if entry.has_data_descriptor:
280
290
  raise UnsupportedDataDescriptorError()
281
291
 
282
- if entry.algorithm not in {0x0000, 0x0008}:
292
+ if not is_algorithm_available(entry.algorithm):
283
293
  raise UnsupportedAlgorithmError(entry.algorithm)
284
294
 
285
295
  header, buffer = await self.fetch_file_header(entry=entry)
@@ -291,11 +301,7 @@ class RemoteZip:
291
301
  else:
292
302
  raw_generator = self._read_range_stream(real_data_offset, entry.compressed_size)
293
303
 
294
- match entry.algorithm:
295
- case 0x0000:
296
- return raw_generator
297
- case 0x0008:
298
- return deflate_wrapper(raw_generator)
304
+ return get_algorithm_handler(entry.algorithm)(raw_generator)
299
305
 
300
306
  async def stream_single_file(self, filename: bytes):
301
307
  """
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rtzip
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Requires-Python: >=3.12
5
5
  License-File: LICENSE
6
6
  Requires-Dist: obstruct==0.5.2
@@ -2,11 +2,12 @@ LICENSE
2
2
  README.md
3
3
  pyproject.toml
4
4
  src/rtzip/__init__.py
5
+ src/rtzip/algorithm_register.py
6
+ src/rtzip/codec_wrappers.py
5
7
  src/rtzip/const.py
6
8
  src/rtzip/errors.py
7
9
  src/rtzip/models.py
8
10
  src/rtzip/remotezip.py
9
- src/rtzip/utils.py
10
11
  src/rtzip.egg-info/PKG-INFO
11
12
  src/rtzip.egg-info/SOURCES.txt
12
13
  src/rtzip.egg-info/dependency_links.txt
rtzip-0.2.2/README.md DELETED
@@ -1,18 +0,0 @@
1
- # Rtzip
2
- _一个以最小的数据量提取远程 zip 文件内容的库_
3
-
4
-
5
- ## 特点
6
- - 不与任何 HTTP 库耦合
7
- - 依赖简单(只有用于解析二进制结构的 `obstruct`)
8
- - 兼容标准 zip 格式、ZIP64 扩展
9
-
10
-
11
- ## 安装
12
- ```commandline
13
- pip install rtzip
14
- ```
15
-
16
- > [!TIP]
17
- > 本库无计划新功能,若是让自行添加功能,可克隆仓库并重写 `rtzip/remotezip.py` 下的代码
18
- > 大多数库的功能已完成并可以直接投入使用
@@ -1,3 +0,0 @@
1
- from .models import EOCD, Zip64EOCD, CDEntry
2
- from .errors import *
3
- from .remotezip import RemoteZip, DataSourceCallbacks
@@ -1,21 +0,0 @@
1
- import zlib
2
- from collections.abc import Buffer
3
-
4
-
5
- async def deflate_wrapper(gen):
6
- decompresser = zlib.decompressobj(-15)
7
- async for chunk in gen:
8
- yield decompresser.decompress(chunk)
9
- remaining = decompresser.flush()
10
- if remaining:
11
- yield remaining
12
-
13
-
14
- async def single_chunk_wrapper(data: Buffer):
15
- yield data
16
-
17
-
18
- def exactly_get_slice(__obj, __offset, __length):
19
- result = __obj[__offset:__offset + __length]
20
- assert len(result) == __length
21
- return result
File without changes
File without changes
File without changes
File without changes
File without changes