httpx2 2.11.0__tar.gz → 2.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpx2-2.11.0 → httpx2-2.12.0}/CHANGELOG.md +12 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/PKG-INFO +9 -17
- {httpx2-2.11.0 → httpx2-2.12.0}/README.md +1 -1
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_client.py +7 -2
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_content.py +10 -4
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_decoders.py +103 -62
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_models.py +41 -33
- {httpx2-2.11.0 → httpx2-2.12.0}/pyproject.toml +1 -1
- {httpx2-2.11.0 → httpx2-2.12.0}/.gitignore +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/LICENSE.md +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/__init__.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/__version__.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_alias.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_api.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_auth.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_config.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_exceptions.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_main.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_multipart.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_sse.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_status_codes.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_transports/__init__.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_transports/asgi.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_transports/base.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_transports/default.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_transports/mock.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_transports/wsgi.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_types.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_urlparse.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_urls.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/_utils.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/py.typed +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/websockets/__init__.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/websockets/_api.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/websockets/_exceptions.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/websockets/_ping.py +0 -0
- {httpx2-2.11.0 → httpx2-2.12.0}/httpx2/websockets/_transport.py +0 -0
|
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
6
6
|
|
|
7
|
+
## 2.12.0 (August 18th, 2026)
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
* Use `backports.zstd` for Zstandard decoding on Python 3.13 and earlier.
|
|
12
|
+
([#1146](https://github.com/pydantic/httpx2/pull/1146))
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
* Bound peak memory while streaming compressed responses and close response streams when decoding fails.
|
|
17
|
+
([#1126](https://github.com/pydantic/httpx2/pull/1126))
|
|
18
|
+
|
|
7
19
|
## 2.11.0 (August 18th, 2026)
|
|
8
20
|
|
|
9
21
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: httpx2
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.12.0
|
|
4
4
|
Summary: The next generation HTTP client.
|
|
5
5
|
Project-URL: Changelog, https://github.com/pydantic/httpx2/blob/main/src/httpx2/CHANGELOG.md
|
|
6
6
|
Project-URL: Homepage, https://github.com/pydantic/httpx2
|
|
@@ -27,7 +27,7 @@ Classifier: Programming Language :: Python :: 3.15
|
|
|
27
27
|
Classifier: Topic :: Internet :: WWW/HTTP
|
|
28
28
|
Requires-Python: >=3.10
|
|
29
29
|
Requires-Dist: anyio>=4.10; sys_platform != 'emscripten'
|
|
30
|
-
Requires-Dist: httpcore2==2.
|
|
30
|
+
Requires-Dist: httpcore2==2.12.0; sys_platform != 'emscripten'
|
|
31
31
|
Requires-Dist: httpx2-jsfetch; sys_platform == 'emscripten' and python_version >= '3.12'
|
|
32
32
|
Requires-Dist: idna>=3.18
|
|
33
33
|
Requires-Dist: truststore>=0.10; sys_platform != 'emscripten'
|
|
@@ -46,7 +46,7 @@ Requires-Dist: socksio==1.*; extra == 'socks'
|
|
|
46
46
|
Provides-Extra: ws
|
|
47
47
|
Requires-Dist: wsproto>=1.2; extra == 'ws'
|
|
48
48
|
Provides-Extra: zstd
|
|
49
|
-
Requires-Dist:
|
|
49
|
+
Requires-Dist: backports-zstd>=1.0.0; (python_version <= '3.13') and extra == 'zstd'
|
|
50
50
|
Description-Content-Type: text/markdown
|
|
51
51
|
|
|
52
52
|
<h1 align="center">HTTPX2</h1>
|
|
@@ -200,7 +200,7 @@ As well as these optional installs:
|
|
|
200
200
|
* `rich` - Rich terminal support. *(Optional, with `httpx2[cli]`)*
|
|
201
201
|
* `click` - Command line client support. *(Optional, with `httpx2[cli]`)*
|
|
202
202
|
* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx2[brotli]`)*
|
|
203
|
-
* `
|
|
203
|
+
* `backports.zstd` - Decoding for "zstd" compressed responses on Python 3.13 and below. *(Optional, with `httpx2[zstd]`. On Python 3.14+, `zstd` is supported via the stdlib [`compression.zstd`](https://docs.python.org/3/library/compression.zstd.html) module when it is available, and the `httpx2[zstd]` extra installs nothing.)*
|
|
204
204
|
|
|
205
205
|
A huge amount of credit is due to `requests` for the API layout that
|
|
206
206
|
much of this work follows, as well as to `urllib3` for plenty of design
|
|
@@ -212,25 +212,17 @@ inspiration around the lower-level networking details.
|
|
|
212
212
|
|
|
213
213
|
## Release Information
|
|
214
214
|
|
|
215
|
-
## 2.
|
|
216
|
-
|
|
217
|
-
### Added
|
|
218
|
-
|
|
219
|
-
* Add the public `Origin` value object and `URL.origin` property for normalized,
|
|
220
|
-
hashable origin comparisons. ([#1134](https://github.com/pydantic/httpx2/pull/1134))
|
|
215
|
+
## 2.12.0 (August 18th, 2026)
|
|
221
216
|
|
|
222
217
|
### Changed
|
|
223
218
|
|
|
224
|
-
*
|
|
219
|
+
* Use `backports.zstd` for Zstandard decoding on Python 3.13 and earlier.
|
|
220
|
+
([#1146](https://github.com/pydantic/httpx2/pull/1146))
|
|
225
221
|
|
|
226
222
|
### Fixed
|
|
227
223
|
|
|
228
|
-
*
|
|
229
|
-
|
|
230
|
-
* Respect explicit `Transfer-Encoding` headers and expose buffered request body lengths to WSGI applications.
|
|
231
|
-
([#1137](https://github.com/pydantic/httpx2/pull/1137))
|
|
232
|
-
* Validate multipart part header names and values before serialization.
|
|
233
|
-
([#1142](https://github.com/pydantic/httpx2/pull/1142))
|
|
224
|
+
* Bound peak memory while streaming compressed responses and close response streams when decoding fails.
|
|
225
|
+
([#1126](https://github.com/pydantic/httpx2/pull/1126))
|
|
234
226
|
|
|
235
227
|
---
|
|
236
228
|
|
|
@@ -149,7 +149,7 @@ As well as these optional installs:
|
|
|
149
149
|
* `rich` - Rich terminal support. *(Optional, with `httpx2[cli]`)*
|
|
150
150
|
* `click` - Command line client support. *(Optional, with `httpx2[cli]`)*
|
|
151
151
|
* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx2[brotli]`)*
|
|
152
|
-
* `
|
|
152
|
+
* `backports.zstd` - Decoding for "zstd" compressed responses on Python 3.13 and below. *(Optional, with `httpx2[zstd]`. On Python 3.14+, `zstd` is supported via the stdlib [`compression.zstd`](https://docs.python.org/3/library/compression.zstd.html) module when it is available, and the `httpx2[zstd]` extra installs nothing.)*
|
|
153
153
|
|
|
154
154
|
A huge amount of credit is due to `requests` for the API layout that
|
|
155
155
|
much of this work follows, as well as to `urllib3` for plenty of design
|
|
@@ -160,8 +160,13 @@ class BoundAsyncStream(AsyncByteStream):
|
|
|
160
160
|
self.elapsed: datetime.timedelta | None = None
|
|
161
161
|
|
|
162
162
|
async def __aiter__(self) -> typing.AsyncIterator[bytes]:
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
stream = self._stream.__aiter__()
|
|
164
|
+
try:
|
|
165
|
+
async for chunk in stream:
|
|
166
|
+
yield chunk
|
|
167
|
+
finally:
|
|
168
|
+
if isinstance(stream, AsyncGenerator):
|
|
169
|
+
await stream.aclose()
|
|
165
170
|
|
|
166
171
|
async def aclose(self) -> None:
|
|
167
172
|
self.elapsed = datetime.timedelta(seconds=time.perf_counter() - self._start)
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
4
|
import warnings
|
|
5
|
-
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Mapping
|
|
5
|
+
from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Iterable, Iterator, Mapping
|
|
6
6
|
from json import dumps as json_dumps
|
|
7
7
|
from typing import (
|
|
8
8
|
Any,
|
|
@@ -79,9 +79,15 @@ class AsyncIteratorByteStream(AsyncByteStream):
|
|
|
79
79
|
yield chunk
|
|
80
80
|
chunk = await self._stream.aread(self.CHUNK_SIZE)
|
|
81
81
|
else:
|
|
82
|
-
# Otherwise iterate
|
|
83
|
-
|
|
84
|
-
|
|
82
|
+
# Otherwise iterate, making sure the wrapped stream is closed even if the
|
|
83
|
+
# consumer stops early (e.g. an exception is raised part-way through decoding).
|
|
84
|
+
stream = self._stream.__aiter__()
|
|
85
|
+
try:
|
|
86
|
+
async for part in stream:
|
|
87
|
+
yield part
|
|
88
|
+
finally:
|
|
89
|
+
if isinstance(stream, AsyncGenerator):
|
|
90
|
+
await stream.aclose()
|
|
85
91
|
|
|
86
92
|
|
|
87
93
|
class UnattachedStream(AsyncByteStream, SyncByteStream):
|
|
@@ -7,8 +7,8 @@ See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import codecs
|
|
10
|
-
import functools
|
|
11
10
|
import io
|
|
11
|
+
import itertools
|
|
12
12
|
import sys
|
|
13
13
|
import typing
|
|
14
14
|
import zlib
|
|
@@ -31,37 +31,60 @@ except ImportError: # pragma: no cover
|
|
|
31
31
|
# Zstandard support is optional on Python <= 3.13.
|
|
32
32
|
# On Python 3.14+, the stdlib includes an optional built-in zstd implementation.
|
|
33
33
|
if typing.TYPE_CHECKING:
|
|
34
|
-
# We keep checking Python version in the type checker path because try..except doesn't help type checkers.
|
|
35
34
|
if sys.version_info >= (3, 14):
|
|
36
35
|
from compression.zstd import ZstdDecompressor, ZstdError
|
|
37
36
|
else:
|
|
38
|
-
from
|
|
37
|
+
from backports.zstd import ZstdDecompressor, ZstdError
|
|
39
38
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
_zstandard_installed: bool = False
|
|
39
|
+
_zstandard_installed: bool
|
|
43
40
|
else: # pragma: no cover
|
|
44
|
-
_zstandard_installed = False
|
|
45
41
|
try:
|
|
46
|
-
|
|
42
|
+
if sys.version_info >= (3, 14):
|
|
43
|
+
from compression.zstd import ZstdDecompressor, ZstdError
|
|
44
|
+
else:
|
|
45
|
+
from backports.zstd import ZstdDecompressor, ZstdError
|
|
47
46
|
|
|
48
47
|
_zstandard_installed = True
|
|
49
|
-
# Either Python <3.14 or the distro doesn't have `compression.zstd`.
|
|
50
48
|
except ImportError:
|
|
51
|
-
|
|
52
|
-
|
|
49
|
+
_zstandard_installed = False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
MAX_DECODE_CHUNK_SIZE = 2**20 # 1 MiB
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class Decompressor(typing.Protocol):
|
|
56
|
+
@property
|
|
57
|
+
def unconsumed_tail(self) -> bytes: ...
|
|
58
|
+
|
|
59
|
+
def decompress(self, data: bytes, max_length: int) -> bytes: ...
|
|
60
|
+
|
|
61
|
+
def flush(self) -> bytes: ...
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class ZlibDecompressor:
|
|
65
|
+
"""
|
|
66
|
+
Drain a `zlib`/`gzip` decompressor in bounded pieces so a small compressed
|
|
67
|
+
input cannot inflate to an unbounded buffer in a single call.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(self, decompressor: Decompressor) -> None:
|
|
71
|
+
self.decompressor = decompressor
|
|
53
72
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
73
|
+
def decompress(self, data: bytes) -> typing.Iterator[bytes]:
|
|
74
|
+
decompressed = self.decompressor.decompress(data, MAX_DECODE_CHUNK_SIZE)
|
|
75
|
+
while decompressed:
|
|
76
|
+
yield decompressed
|
|
77
|
+
decompressed = self.decompressor.decompress(self.decompressor.unconsumed_tail, MAX_DECODE_CHUNK_SIZE)
|
|
78
|
+
|
|
79
|
+
def flush(self) -> bytes:
|
|
80
|
+
return self.decompressor.flush()
|
|
58
81
|
|
|
59
82
|
|
|
60
83
|
class ContentDecoder:
|
|
61
|
-
def decode(self, data: bytes) -> bytes:
|
|
84
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
62
85
|
raise NotImplementedError() # pragma: no cover
|
|
63
86
|
|
|
64
|
-
def flush(self) -> bytes:
|
|
87
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
65
88
|
raise NotImplementedError() # pragma: no cover
|
|
66
89
|
|
|
67
90
|
|
|
@@ -70,11 +93,11 @@ class IdentityDecoder(ContentDecoder):
|
|
|
70
93
|
Handle unencoded data.
|
|
71
94
|
"""
|
|
72
95
|
|
|
73
|
-
def decode(self, data: bytes) -> bytes:
|
|
74
|
-
|
|
96
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
97
|
+
yield data
|
|
75
98
|
|
|
76
|
-
def flush(self) -> bytes:
|
|
77
|
-
|
|
99
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
100
|
+
yield from ()
|
|
78
101
|
|
|
79
102
|
|
|
80
103
|
class DeflateDecoder(ContentDecoder):
|
|
@@ -86,22 +109,23 @@ class DeflateDecoder(ContentDecoder):
|
|
|
86
109
|
|
|
87
110
|
def __init__(self) -> None:
|
|
88
111
|
self.first_attempt = True
|
|
89
|
-
self.decompressor = zlib.decompressobj()
|
|
112
|
+
self.decompressor = ZlibDecompressor(zlib.decompressobj())
|
|
90
113
|
|
|
91
|
-
def decode(self, data: bytes) -> bytes:
|
|
114
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
92
115
|
was_first_attempt = self.first_attempt
|
|
93
116
|
self.first_attempt = False
|
|
94
117
|
try:
|
|
95
|
-
|
|
118
|
+
yield from self.decompressor.decompress(data)
|
|
96
119
|
except zlib.error as exc:
|
|
97
120
|
if was_first_attempt:
|
|
98
|
-
self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
|
|
99
|
-
|
|
100
|
-
|
|
121
|
+
self.decompressor = ZlibDecompressor(zlib.decompressobj(-zlib.MAX_WBITS))
|
|
122
|
+
yield from self.decode(data)
|
|
123
|
+
else:
|
|
124
|
+
raise DecodingError(str(exc)) from exc
|
|
101
125
|
|
|
102
|
-
def flush(self) -> bytes:
|
|
126
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
103
127
|
try:
|
|
104
|
-
|
|
128
|
+
yield self.decompressor.flush()
|
|
105
129
|
except zlib.error as exc: # pragma: no cover
|
|
106
130
|
raise DecodingError(str(exc)) from exc
|
|
107
131
|
|
|
@@ -114,17 +138,17 @@ class GZipDecoder(ContentDecoder):
|
|
|
114
138
|
"""
|
|
115
139
|
|
|
116
140
|
def __init__(self) -> None:
|
|
117
|
-
self.decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
|
|
141
|
+
self.decompressor = ZlibDecompressor(zlib.decompressobj(zlib.MAX_WBITS | 16))
|
|
118
142
|
|
|
119
|
-
def decode(self, data: bytes) -> bytes:
|
|
143
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
120
144
|
try:
|
|
121
|
-
|
|
145
|
+
yield from self.decompressor.decompress(data)
|
|
122
146
|
except zlib.error as exc:
|
|
123
147
|
raise DecodingError(str(exc)) from exc
|
|
124
148
|
|
|
125
|
-
def flush(self) -> bytes:
|
|
149
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
126
150
|
try:
|
|
127
|
-
|
|
151
|
+
yield self.decompressor.flush()
|
|
128
152
|
except zlib.error as exc: # pragma: no cover
|
|
129
153
|
raise DecodingError(str(exc)) from exc
|
|
130
154
|
|
|
@@ -149,26 +173,31 @@ class BrotliDecoder(ContentDecoder):
|
|
|
149
173
|
|
|
150
174
|
self.decompressor = brotli.Decompressor()
|
|
151
175
|
self.seen_data = False
|
|
152
|
-
self._decompress: typing.Callable[
|
|
176
|
+
self._decompress: typing.Callable[..., bytes]
|
|
153
177
|
if hasattr(self.decompressor, "decompress"):
|
|
154
178
|
# The 'brotlicffi' package.
|
|
155
179
|
self._decompress = self.decompressor.decompress # pragma: no cover
|
|
156
180
|
else:
|
|
157
181
|
# The 'brotli' package.
|
|
158
|
-
self._decompress = self.decompressor.process
|
|
182
|
+
self._decompress = self.decompressor.process
|
|
159
183
|
|
|
160
|
-
def decode(self, data: bytes) -> bytes:
|
|
184
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
161
185
|
if not data:
|
|
162
|
-
return
|
|
186
|
+
return
|
|
163
187
|
self.seen_data = True
|
|
164
188
|
try:
|
|
165
|
-
|
|
189
|
+
# The C backend may allocate nearly twice the requested threshold.
|
|
190
|
+
output_buffer_limit = MAX_DECODE_CHUNK_SIZE // 2
|
|
191
|
+
decompressed = self._decompress(data, output_buffer_limit=output_buffer_limit)
|
|
192
|
+
while decompressed:
|
|
193
|
+
yield decompressed
|
|
194
|
+
decompressed = self._decompress(b"", output_buffer_limit=output_buffer_limit)
|
|
166
195
|
except brotli.error as exc:
|
|
167
196
|
raise DecodingError(str(exc)) from exc
|
|
168
197
|
|
|
169
|
-
def flush(self) -> bytes:
|
|
198
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
170
199
|
if not self.seen_data:
|
|
171
|
-
return
|
|
200
|
+
return
|
|
172
201
|
try:
|
|
173
202
|
if hasattr(self.decompressor, "finish"):
|
|
174
203
|
# Only available in the 'brotlicffi' package.
|
|
@@ -177,16 +206,15 @@ class BrotliDecoder(ContentDecoder):
|
|
|
177
206
|
# will never actually emit any data. However, it will potentially throw
|
|
178
207
|
# errors if a truncated or damaged data stream has been used.
|
|
179
208
|
self.decompressor.finish() # pragma: no cover
|
|
180
|
-
return b""
|
|
181
209
|
except brotli.error as exc: # pragma: no cover
|
|
182
210
|
raise DecodingError(str(exc)) from exc
|
|
211
|
+
yield from ()
|
|
183
212
|
|
|
184
213
|
|
|
185
214
|
class ZStandardDecoder(ContentDecoder):
|
|
186
215
|
"""Handle 'zstd' RFC 8878 decoding.
|
|
187
216
|
|
|
188
|
-
|
|
189
|
-
`pip install zstandard` or `pip install httpx2[zstd]`.
|
|
217
|
+
On Python 3.13 and below, this requires `pip install httpx2[zstd]`.
|
|
190
218
|
"""
|
|
191
219
|
|
|
192
220
|
# inspired by the ZstdDecoder implementation in urllib3
|
|
@@ -199,30 +227,34 @@ class ZStandardDecoder(ContentDecoder):
|
|
|
199
227
|
self.decompressor = ZstdDecompressor()
|
|
200
228
|
self.seen_data = False
|
|
201
229
|
|
|
202
|
-
def decode(self, data: bytes) -> bytes:
|
|
230
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
203
231
|
if not data:
|
|
204
|
-
return
|
|
232
|
+
return
|
|
205
233
|
self.seen_data = True
|
|
206
|
-
output = io.BytesIO()
|
|
207
234
|
try:
|
|
208
235
|
if self.decompressor.eof:
|
|
209
236
|
data = self.decompressor.unused_data + data
|
|
210
237
|
self.decompressor = ZstdDecompressor()
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
238
|
+
while True:
|
|
239
|
+
decompressed = self.decompressor.decompress(data, MAX_DECODE_CHUNK_SIZE)
|
|
240
|
+
while decompressed:
|
|
241
|
+
yield decompressed
|
|
242
|
+
if self.decompressor.needs_input or self.decompressor.eof:
|
|
243
|
+
break
|
|
244
|
+
decompressed = self.decompressor.decompress(b"", MAX_DECODE_CHUNK_SIZE)
|
|
245
|
+
if not (self.decompressor.eof and self.decompressor.unused_data):
|
|
246
|
+
break
|
|
247
|
+
data = self.decompressor.unused_data
|
|
214
248
|
self.decompressor = ZstdDecompressor()
|
|
215
|
-
output.write(self.decompressor.decompress(unused_data))
|
|
216
249
|
except ZstdError as exc:
|
|
217
250
|
raise DecodingError(str(exc)) from exc
|
|
218
|
-
return output.getvalue()
|
|
219
251
|
|
|
220
|
-
def flush(self) -> bytes:
|
|
252
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
221
253
|
if not self.seen_data:
|
|
222
|
-
return
|
|
254
|
+
return
|
|
223
255
|
if not self.decompressor.eof:
|
|
224
256
|
raise DecodingError("Zstandard data is incomplete") # pragma: no cover
|
|
225
|
-
|
|
257
|
+
yield from ()
|
|
226
258
|
|
|
227
259
|
|
|
228
260
|
class MultiDecoder(ContentDecoder):
|
|
@@ -243,16 +275,25 @@ class MultiDecoder(ContentDecoder):
|
|
|
243
275
|
# Note that we reverse the order for decoding.
|
|
244
276
|
self.children: list[ContentDecoder] = [SUPPORTED_DECODERS[coding]() for coding in reversed(codings)]
|
|
245
277
|
|
|
246
|
-
def decode(self, data: bytes) -> bytes:
|
|
278
|
+
def decode(self, data: bytes) -> typing.Iterator[bytes]:
|
|
279
|
+
streams: typing.Iterator[bytes] = iter((data,))
|
|
247
280
|
for child in self.children:
|
|
248
|
-
|
|
249
|
-
|
|
281
|
+
streams = self._pipe(child.decode, streams)
|
|
282
|
+
yield from streams
|
|
250
283
|
|
|
251
|
-
def flush(self) -> bytes:
|
|
252
|
-
|
|
284
|
+
def flush(self) -> typing.Iterator[bytes]:
|
|
285
|
+
streams: typing.Iterator[bytes] = iter(())
|
|
253
286
|
for child in self.children:
|
|
254
|
-
|
|
255
|
-
|
|
287
|
+
streams = itertools.chain(self._pipe(child.decode, streams), child.flush())
|
|
288
|
+
yield from streams
|
|
289
|
+
|
|
290
|
+
@staticmethod
|
|
291
|
+
def _pipe(
|
|
292
|
+
decode: typing.Callable[[bytes], typing.Iterator[bytes]],
|
|
293
|
+
upstream: typing.Iterator[bytes],
|
|
294
|
+
) -> typing.Iterator[bytes]:
|
|
295
|
+
for chunk in upstream:
|
|
296
|
+
yield from decode(chunk)
|
|
256
297
|
|
|
257
298
|
|
|
258
299
|
class ByteChunker:
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import codecs
|
|
4
|
+
import contextlib
|
|
4
5
|
import datetime
|
|
5
6
|
import email.message
|
|
6
7
|
import json as jsonlib
|
|
7
8
|
import re
|
|
8
9
|
import typing
|
|
9
10
|
import urllib.request
|
|
10
|
-
from collections.abc import Mapping
|
|
11
|
+
from collections.abc import AsyncGenerator, Mapping
|
|
11
12
|
from http.cookiejar import Cookie, CookieJar
|
|
12
13
|
|
|
13
14
|
from ._content import ByteStream, UnattachedStream, encode_request, encode_response
|
|
@@ -889,12 +890,12 @@ class Response:
|
|
|
889
890
|
chunker = ByteChunker(chunk_size=chunk_size)
|
|
890
891
|
with request_context(request=self._request):
|
|
891
892
|
for raw_bytes in self.iter_raw():
|
|
892
|
-
decoded
|
|
893
|
+
for decoded in decoder.decode(raw_bytes):
|
|
894
|
+
for chunk in chunker.decode(decoded):
|
|
895
|
+
yield chunk
|
|
896
|
+
for decoded in decoder.flush():
|
|
893
897
|
for chunk in chunker.decode(decoded):
|
|
894
|
-
yield chunk
|
|
895
|
-
decoded = decoder.flush()
|
|
896
|
-
for chunk in chunker.decode(decoded):
|
|
897
|
-
yield chunk # pragma: no cover
|
|
898
|
+
yield chunk # pragma: no cover
|
|
898
899
|
for chunk in chunker.flush():
|
|
899
900
|
yield chunk
|
|
900
901
|
|
|
@@ -941,16 +942,17 @@ class Response:
|
|
|
941
942
|
self._num_bytes_downloaded = 0
|
|
942
943
|
chunker = ByteChunker(chunk_size=chunk_size)
|
|
943
944
|
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
self.
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
for chunk in chunker.flush():
|
|
951
|
-
yield chunk
|
|
945
|
+
try:
|
|
946
|
+
with request_context(request=self._request):
|
|
947
|
+
for raw_stream_bytes in self.stream:
|
|
948
|
+
self._num_bytes_downloaded += len(raw_stream_bytes)
|
|
949
|
+
for chunk in chunker.decode(raw_stream_bytes):
|
|
950
|
+
yield chunk
|
|
952
951
|
|
|
953
|
-
|
|
952
|
+
for chunk in chunker.flush():
|
|
953
|
+
yield chunk
|
|
954
|
+
finally:
|
|
955
|
+
self.close()
|
|
954
956
|
|
|
955
957
|
def close(self) -> None:
|
|
956
958
|
"""
|
|
@@ -970,10 +972,11 @@ class Response:
|
|
|
970
972
|
Read and return the response content.
|
|
971
973
|
"""
|
|
972
974
|
if not hasattr(self, "_content"):
|
|
973
|
-
|
|
975
|
+
async with contextlib.aclosing(self.aiter_bytes()) as parts:
|
|
976
|
+
self._content = b"".join([part async for part in parts])
|
|
974
977
|
return self._content
|
|
975
978
|
|
|
976
|
-
async def aiter_bytes(self, chunk_size: int | None = None) -> typing.
|
|
979
|
+
async def aiter_bytes(self, chunk_size: int | None = None) -> typing.AsyncGenerator[bytes, None]:
|
|
977
980
|
"""
|
|
978
981
|
A byte-iterator over the decoded response content.
|
|
979
982
|
This allows us to handle gzip, deflate, brotli, and zstd encoded responses.
|
|
@@ -986,13 +989,14 @@ class Response:
|
|
|
986
989
|
decoder = self._get_content_decoder()
|
|
987
990
|
chunker = ByteChunker(chunk_size=chunk_size)
|
|
988
991
|
with request_context(request=self._request):
|
|
989
|
-
async
|
|
990
|
-
|
|
992
|
+
async with contextlib.aclosing(self.aiter_raw()) as raw_stream:
|
|
993
|
+
async for raw_bytes in raw_stream:
|
|
994
|
+
for decoded in decoder.decode(raw_bytes):
|
|
995
|
+
for chunk in chunker.decode(decoded):
|
|
996
|
+
yield chunk
|
|
997
|
+
for decoded in decoder.flush():
|
|
991
998
|
for chunk in chunker.decode(decoded):
|
|
992
|
-
yield chunk
|
|
993
|
-
decoded = decoder.flush()
|
|
994
|
-
for chunk in chunker.decode(decoded):
|
|
995
|
-
yield chunk # pragma: no cover
|
|
999
|
+
yield chunk # pragma: no cover
|
|
996
1000
|
for chunk in chunker.flush():
|
|
997
1001
|
yield chunk
|
|
998
1002
|
|
|
@@ -1024,7 +1028,7 @@ class Response:
|
|
|
1024
1028
|
for line in decoder.flush():
|
|
1025
1029
|
yield line
|
|
1026
1030
|
|
|
1027
|
-
async def aiter_raw(self, chunk_size: int | None = None) -> typing.
|
|
1031
|
+
async def aiter_raw(self, chunk_size: int | None = None) -> typing.AsyncGenerator[bytes, None]:
|
|
1028
1032
|
"""
|
|
1029
1033
|
A byte-iterator over the raw response content.
|
|
1030
1034
|
"""
|
|
@@ -1039,16 +1043,20 @@ class Response:
|
|
|
1039
1043
|
self._num_bytes_downloaded = 0
|
|
1040
1044
|
chunker = ByteChunker(chunk_size=chunk_size)
|
|
1041
1045
|
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
for
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
yield chunk
|
|
1046
|
+
stream = self.stream.__aiter__()
|
|
1047
|
+
try:
|
|
1048
|
+
with request_context(request=self._request):
|
|
1049
|
+
async for raw_stream_bytes in stream:
|
|
1050
|
+
self._num_bytes_downloaded += len(raw_stream_bytes)
|
|
1051
|
+
for chunk in chunker.decode(raw_stream_bytes):
|
|
1052
|
+
yield chunk
|
|
1050
1053
|
|
|
1051
|
-
|
|
1054
|
+
for chunk in chunker.flush():
|
|
1055
|
+
yield chunk
|
|
1056
|
+
finally:
|
|
1057
|
+
if isinstance(stream, AsyncGenerator):
|
|
1058
|
+
await stream.aclose()
|
|
1059
|
+
await self.aclose()
|
|
1052
1060
|
|
|
1053
1061
|
async def aclose(self) -> None:
|
|
1054
1062
|
"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|