rtzip 0.2.2__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rtzip-0.2.2/src/rtzip.egg-info → rtzip-0.3.1}/PKG-INFO +1 -1
- rtzip-0.3.1/README.md +119 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/pyproject.toml +1 -1
- rtzip-0.3.1/src/rtzip/__init__.py +4 -0
- rtzip-0.3.1/src/rtzip/algorithm_register.py +45 -0
- rtzip-0.3.1/src/rtzip/codec_wrappers.py +90 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip/models.py +6 -2
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip/remotezip.py +11 -11
- {rtzip-0.2.2 → rtzip-0.3.1/src/rtzip.egg-info}/PKG-INFO +1 -1
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip.egg-info/SOURCES.txt +2 -1
- rtzip-0.2.2/README.md +0 -18
- rtzip-0.2.2/src/rtzip/__init__.py +0 -3
- rtzip-0.2.2/src/rtzip/utils.py +0 -21
- {rtzip-0.2.2 → rtzip-0.3.1}/LICENSE +0 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/setup.cfg +0 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip/const.py +0 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip/errors.py +0 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip.egg-info/dependency_links.txt +0 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip.egg-info/requires.txt +0 -0
- {rtzip-0.2.2 → rtzip-0.3.1}/src/rtzip.egg-info/top_level.txt +0 -0
rtzip-0.3.1/README.md
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Rtzip
|
|
2
|
+
|
|
3
|
+
_一个以最小的数据量提取远程 zip 文件内容的库_
|
|
4
|
+
|
|
5
|
+
## 特点
|
|
6
|
+
|
|
7
|
+
- 不与任何 HTTP 库耦合
|
|
8
|
+
- 依赖简单(只有用于解析二进制结构的 `obstruct`)
|
|
9
|
+
- 兼容标准 zip 格式、ZIP64 扩展
|
|
10
|
+
|
|
11
|
+
## 安装
|
|
12
|
+
|
|
13
|
+
```commandline
|
|
14
|
+
pip install rtzip
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## 用法
|
|
18
|
+
|
|
19
|
+
### 直接使用
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from rtzip import RemoteZip, DataSource
|
|
23
|
+
|
|
24
|
+
import niquests # 如果你使用 niquests 作为 http 依赖的话
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class HTTPSource(DataSource):
|
|
28
|
+
def __init__(self, url, session=None):
|
|
29
|
+
self.url = url
|
|
30
|
+
self.session = session or niquests.AsyncSession
|
|
31
|
+
|
|
32
|
+
async def _request_range(self, offset, length):
|
|
33
|
+
resp = await self.session.get(self.url, headers={'Range': f"bytes={offset}-{offset + length - 1}"}, stream=True)
|
|
34
|
+
resp.raise_for_status()
|
|
35
|
+
if resp.status_code != 206:
|
|
36
|
+
await resp.close()
|
|
37
|
+
raise niquests.HTTPError(response=resp)
|
|
38
|
+
return resp
|
|
39
|
+
|
|
40
|
+
async def read_range(self, offset, length):
|
|
41
|
+
return await (await self._request_range(offset, length)).content
|
|
42
|
+
|
|
43
|
+
async def read_range_stream(self, offset, length):
|
|
44
|
+
async for i in await (await self._request_range(offset, length)).iter_content():
|
|
45
|
+
yield i
|
|
46
|
+
|
|
47
|
+
async def get_total_size(self):
|
|
48
|
+
resp = await self.session.head(self.url)
|
|
49
|
+
resp.raise_for_status()
|
|
50
|
+
return int(resp['content-length'])
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def main():
|
|
54
|
+
url = ...
|
|
55
|
+
rz = RemoteZip(HTTPSource(url))
|
|
56
|
+
rz.fetch_zip_info() # 加载文件的 EOCD,返回一个 EOCD 获取 Zip64EOCD 对象
|
|
57
|
+
rz.fetch_file_list() # 加载文件的中央目录,返回一个包含 CDEntry 对象的列表
|
|
58
|
+
# 这些数据如果没有即时接收,也会被缓存在 rz.eocd 和 rz.files 中
|
|
59
|
+
|
|
60
|
+
rz.build_mapping() # 构建文件名 -> Entry 对象的映射表,方便查表
|
|
61
|
+
# 返回映射表本身,同时缓存在 rz.file_mapping 中
|
|
62
|
+
|
|
63
|
+
# 一次性读取一个文件的全量数据
|
|
64
|
+
data = await rz.load_single_file(b'filename')
|
|
65
|
+
|
|
66
|
+
# 流式读取一个文件的数据
|
|
67
|
+
buffer = bytearray()
|
|
68
|
+
async for i in rz.stream_single_file(b'filename'):
|
|
69
|
+
buffer.extend(i)
|
|
70
|
+
|
|
71
|
+
# 获取一个文件的元数据信息
|
|
72
|
+
entry = rz.find_file_entry(b'filename')
|
|
73
|
+
# 可能抛出 FileNotFoundError
|
|
74
|
+
|
|
75
|
+
# 从文件名拉取一个文件对应的本地文件头
|
|
76
|
+
header = await rz.fetch_file_header(filename=b'filename')
|
|
77
|
+
|
|
78
|
+
# 也可以从已有的 entry 对象拉取
|
|
79
|
+
# 但这时你要自己保证这个 entry 对象是有效的
|
|
80
|
+
# 将返回一个 LocalFileHeader 对象
|
|
81
|
+
header = await rz.fetch_file_header(entry=entry)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
if __name__ == '__main__':
|
|
85
|
+
import asyncio
|
|
86
|
+
|
|
87
|
+
asyncio.run(main())
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### 自定义压缩算法处理
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
# 示例:扩展 PPMd 算法支持(需要 pip install ppmd)
|
|
95
|
+
import ppmd
|
|
96
|
+
from rtzip import algorithm_handler
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@algorithm_handler(0x000A)
|
|
100
|
+
async def ppmd_wrapper(raw_generator):
|
|
101
|
+
# PPMd 流式解压(API 取决于具体库)
|
|
102
|
+
decomp = ppmd.Ppmd7Decompressor()
|
|
103
|
+
async for chunk in raw_generator:
|
|
104
|
+
yield decomp.decompress(chunk)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# 这之后使用 RemoteZip 时遇到 ppmd 算法会自动应用你的包装器
|
|
108
|
+
|
|
109
|
+
# 如果想要重写某个内置的包装器
|
|
110
|
+
# 你需要指定 `override=True`,否则程序将会阻止覆盖行为
|
|
111
|
+
@algorithm_handler(0x0008, override=True)
|
|
112
|
+
async def your_deflate_wrapper(raw_generator):
|
|
113
|
+
...
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
> [!TIP]
|
|
118
|
+
> 本库无计划新功能,若是让自行添加功能,可克隆仓库并重写 `rtzip/remotezip.py` 下的代码
|
|
119
|
+
> 大多数库的功能已完成并可以直接投入使用
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from collections.abc import Buffer
|
|
2
|
+
from typing import Callable, AsyncGenerator
|
|
3
|
+
|
|
4
|
+
AlgorithmHandler = Callable[[AsyncGenerator[Buffer, None]], AsyncGenerator[bytes | bytearray | memoryview, None]]
|
|
5
|
+
|
|
6
|
+
algorithm_mapping: dict[int, AlgorithmHandler] = {}
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def algorithm_handler(algorithm: int, override=False):
|
|
10
|
+
"""
|
|
11
|
+
压缩流包装器注册装饰器
|
|
12
|
+
|
|
13
|
+
:param algorithm: 压缩算法 ID
|
|
14
|
+
:param override: 如果你希望覆盖一个已注册的包装器,请设置为 True,否则将会阻止可能的覆盖行为
|
|
15
|
+
:param decorator.func: 被包装的函数,接受一个异步生成器,返回一个异步生成器
|
|
16
|
+
:return:
|
|
17
|
+
"""
|
|
18
|
+
if not override and algorithm in algorithm_mapping:
|
|
19
|
+
raise ValueError(f"Algorithm {algorithm} is already registered")
|
|
20
|
+
|
|
21
|
+
def decorator(func: AlgorithmHandler):
|
|
22
|
+
algorithm_mapping[algorithm] = func
|
|
23
|
+
return func
|
|
24
|
+
|
|
25
|
+
return decorator
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def get_algorithm_handler(algorithm: int) -> AlgorithmHandler:
|
|
29
|
+
"""
|
|
30
|
+
获取包装器函数
|
|
31
|
+
|
|
32
|
+
:param algorithm: 压缩算法 ID
|
|
33
|
+
:return: AlgorithmHandler 可调用对象,接受一个异步生成器,返回一个异步生成器
|
|
34
|
+
"""
|
|
35
|
+
return algorithm_mapping[algorithm]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def is_algorithm_available(algorithm: int) -> bool:
|
|
39
|
+
"""
|
|
40
|
+
判断某个压缩算法是否已有注册的包装器
|
|
41
|
+
|
|
42
|
+
:param algorithm: 压缩算法 ID
|
|
43
|
+
:return:
|
|
44
|
+
"""
|
|
45
|
+
return algorithm in algorithm_mapping
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from .algorithm_register import algorithm_handler
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@algorithm_handler(0x0008)
|
|
7
|
+
async def deflate_wrapper(gen):
|
|
8
|
+
import zlib
|
|
9
|
+
decompresser = zlib.decompressobj(-15) # 禁用 zlib 头部
|
|
10
|
+
async for chunk in gen:
|
|
11
|
+
yield decompresser.decompress(chunk)
|
|
12
|
+
remaining = decompresser.flush()
|
|
13
|
+
if remaining:
|
|
14
|
+
yield remaining
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@algorithm_handler(0x0000)
|
|
18
|
+
async def raw_wrapper(gen):
|
|
19
|
+
async for chunk in gen:
|
|
20
|
+
yield chunk
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@algorithm_handler(0x000E)
|
|
24
|
+
async def lzma_wrapper(gen):
|
|
25
|
+
import lzma
|
|
26
|
+
decompressor = lzma.LZMADecompressor()
|
|
27
|
+
|
|
28
|
+
async for chunk in gen:
|
|
29
|
+
data = decompressor.decompress(chunk)
|
|
30
|
+
if data:
|
|
31
|
+
yield data
|
|
32
|
+
|
|
33
|
+
if not decompressor.eof:
|
|
34
|
+
raise lzma.LZMAError(
|
|
35
|
+
'LZMA compressed data stream ended unexpectedly (EOF not reached) – data may be truncated or corrupted')
|
|
36
|
+
|
|
37
|
+
if decompressor.unused_data:
|
|
38
|
+
raise lzma.LZMAError(
|
|
39
|
+
f'LZMA stream contains {len(decompressor.unused_data)} bytes of trailing data after the end marker'
|
|
40
|
+
f' – possibly concatenated streams or extra metadata'
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
@algorithm_handler(0x000C)
|
|
44
|
+
async def bz2_wrapper(gen):
|
|
45
|
+
import bz2
|
|
46
|
+
decompressor = bz2.BZ2Decompressor()
|
|
47
|
+
|
|
48
|
+
async for chunk in gen:
|
|
49
|
+
if chunk:
|
|
50
|
+
data = decompressor.decompress(chunk)
|
|
51
|
+
if data:
|
|
52
|
+
yield data
|
|
53
|
+
|
|
54
|
+
if not decompressor.eof:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"BZIP2 compressed data stream ended unexpectedly (EOF not reached) – "
|
|
57
|
+
"data may be truncated or corrupted"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if decompressor.unused_data:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"BZIP2 stream contains {len(decompressor.unused_data)} bytes of trailing data after the end marker – "
|
|
63
|
+
f"possibly concatenated streams or extra metadata"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if sys.version_info >= (3, 14):
|
|
68
|
+
@algorithm_handler(0x000D)
|
|
69
|
+
async def zstd_wrapper(gen): # Not Tested
|
|
70
|
+
import warnings
|
|
71
|
+
warnings.warn("ZStandard support is experimental and has not been tested")
|
|
72
|
+
|
|
73
|
+
import compression.zstd as zstd
|
|
74
|
+
decompressor = zstd.ZstdDecompressor()
|
|
75
|
+
async for chunk in gen:
|
|
76
|
+
data = decompressor.decompress(chunk)
|
|
77
|
+
if data:
|
|
78
|
+
yield data
|
|
79
|
+
|
|
80
|
+
if not decompressor.eof:
|
|
81
|
+
raise ValueError(
|
|
82
|
+
"ZStandard compressed data stream ended unexpectedly (EOF not reached) – "
|
|
83
|
+
"data may be truncated or corrupted"
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
if decompressor.unused_data:
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"ZStandard stream contains {len(decompressor.unused_data)} bytes of trailing data after the end marker – "
|
|
89
|
+
f"possibly concatenated streams or extra metadata"
|
|
90
|
+
)
|
|
@@ -5,11 +5,15 @@ from typing import Annotated, Self
|
|
|
5
5
|
|
|
6
6
|
import obstruct as obs
|
|
7
7
|
|
|
8
|
-
from rtzip.utils import exactly_get_slice
|
|
9
|
-
|
|
10
8
|
EXTRA_FIELD_HEADER = struct.Struct("<HH")
|
|
11
9
|
|
|
12
10
|
|
|
11
|
+
def exactly_get_slice(__obj, __offset, __length):
|
|
12
|
+
result = __obj[__offset:__offset + __length]
|
|
13
|
+
assert len(result) == __length
|
|
14
|
+
return result
|
|
15
|
+
|
|
16
|
+
|
|
13
17
|
def parse_extra_fields(buffer):
|
|
14
18
|
res = {}
|
|
15
19
|
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import struct
|
|
2
|
+
from collections.abc import Buffer
|
|
2
3
|
from typing import AsyncGenerator
|
|
3
4
|
|
|
5
|
+
from .algorithm_register import is_algorithm_available, get_algorithm_handler
|
|
4
6
|
from .const import *
|
|
5
7
|
from .errors import UnsupportedAlgorithmError, UnsupportedCryptoError, UnsupportedDataDescriptorError
|
|
6
8
|
from .models import EOCD, Zip64EOCD, CDEntry, LocalFileHeader
|
|
7
|
-
from .utils import deflate_wrapper, single_chunk_wrapper
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
|
|
11
|
-
class DataSourceCallbacks:
|
|
11
|
+
class DataSource:
|
|
12
12
|
async def read_range(self, offset, length) -> bytes | bytearray | memoryview:
|
|
13
13
|
"""
|
|
14
14
|
读取指定的一段数据
|
|
@@ -36,6 +36,10 @@ class DataSourceCallbacks:
|
|
|
36
36
|
"""
|
|
37
37
|
|
|
38
38
|
|
|
39
|
+
async def single_chunk_wrapper(__data: Buffer, /):
|
|
40
|
+
yield __data
|
|
41
|
+
|
|
42
|
+
|
|
39
43
|
class RemoteZip:
|
|
40
44
|
"""
|
|
41
45
|
远程 Zip 文件访问器
|
|
@@ -58,7 +62,7 @@ class RemoteZip:
|
|
|
58
62
|
async def _get_total_size(self):
|
|
59
63
|
return await self.source.get_total_size()
|
|
60
64
|
|
|
61
|
-
def __init__(self, source:
|
|
65
|
+
def __init__(self, source: DataSource):
|
|
62
66
|
self.source = source
|
|
63
67
|
|
|
64
68
|
self.cd_info: EOCD | Zip64EOCD | None = None
|
|
@@ -74,7 +78,7 @@ class RemoteZip:
|
|
|
74
78
|
:param initial_chunk: 初始扫描的大小
|
|
75
79
|
:param max_cnt: 最大重试次数
|
|
76
80
|
:param loss_factor: 重试后更新扫描大小的系数
|
|
77
|
-
:return: 一个 EOCD 或者 Zip64EOCD 对象(如果这个 zip
|
|
81
|
+
:return: 一个 EOCD 或者 Zip64EOCD 对象(如果这个 zip 文件启用了 zip64 扩展)
|
|
78
82
|
"""
|
|
79
83
|
chunk_size = initial_chunk
|
|
80
84
|
|
|
@@ -279,7 +283,7 @@ class RemoteZip:
|
|
|
279
283
|
if entry.has_data_descriptor:
|
|
280
284
|
raise UnsupportedDataDescriptorError()
|
|
281
285
|
|
|
282
|
-
if entry.algorithm
|
|
286
|
+
if not is_algorithm_available(entry.algorithm):
|
|
283
287
|
raise UnsupportedAlgorithmError(entry.algorithm)
|
|
284
288
|
|
|
285
289
|
header, buffer = await self.fetch_file_header(entry=entry)
|
|
@@ -291,11 +295,7 @@ class RemoteZip:
|
|
|
291
295
|
else:
|
|
292
296
|
raw_generator = self._read_range_stream(real_data_offset, entry.compressed_size)
|
|
293
297
|
|
|
294
|
-
|
|
295
|
-
case 0x0000:
|
|
296
|
-
return raw_generator
|
|
297
|
-
case 0x0008:
|
|
298
|
-
return deflate_wrapper(raw_generator)
|
|
298
|
+
return get_algorithm_handler(entry.algorithm)(raw_generator)
|
|
299
299
|
|
|
300
300
|
async def stream_single_file(self, filename: bytes):
|
|
301
301
|
"""
|
|
@@ -2,11 +2,12 @@ LICENSE
|
|
|
2
2
|
README.md
|
|
3
3
|
pyproject.toml
|
|
4
4
|
src/rtzip/__init__.py
|
|
5
|
+
src/rtzip/algorithm_register.py
|
|
6
|
+
src/rtzip/codec_wrappers.py
|
|
5
7
|
src/rtzip/const.py
|
|
6
8
|
src/rtzip/errors.py
|
|
7
9
|
src/rtzip/models.py
|
|
8
10
|
src/rtzip/remotezip.py
|
|
9
|
-
src/rtzip/utils.py
|
|
10
11
|
src/rtzip.egg-info/PKG-INFO
|
|
11
12
|
src/rtzip.egg-info/SOURCES.txt
|
|
12
13
|
src/rtzip.egg-info/dependency_links.txt
|
rtzip-0.2.2/README.md
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
# Rtzip
|
|
2
|
-
_一个以最小的数据量提取远程 zip 文件内容的库_
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
## 特点
|
|
6
|
-
- 不与任何 HTTP 库耦合
|
|
7
|
-
- 依赖简单(只有用于解析二进制结构的 `obstruct`)
|
|
8
|
-
- 兼容标准 zip 格式、ZIP64 扩展
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
## 安装
|
|
12
|
-
```commandline
|
|
13
|
-
pip install rtzip
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
> [!TIP]
|
|
17
|
-
> 本库无计划新功能,若是让自行添加功能,可克隆仓库并重写 `rtzip/remotezip.py` 下的代码
|
|
18
|
-
> 大多数库的功能已完成并可以直接投入使用
|
rtzip-0.2.2/src/rtzip/utils.py
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import zlib
|
|
2
|
-
from collections.abc import Buffer
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
async def deflate_wrapper(gen):
|
|
6
|
-
decompresser = zlib.decompressobj(-15)
|
|
7
|
-
async for chunk in gen:
|
|
8
|
-
yield decompresser.decompress(chunk)
|
|
9
|
-
remaining = decompresser.flush()
|
|
10
|
-
if remaining:
|
|
11
|
-
yield remaining
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
async def single_chunk_wrapper(data: Buffer):
|
|
15
|
-
yield data
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def exactly_get_slice(__obj, __offset, __length):
|
|
19
|
-
result = __obj[__offset:__offset + __length]
|
|
20
|
-
assert len(result) == __length
|
|
21
|
-
return result
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|