python-httpfile 0.0.5.3__py3-none-any.whl → 0.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- httpfile/__init__.py +1520 -805
- {python_httpfile-0.0.5.3.dist-info → python_httpfile-0.0.6.dist-info}/METADATA +11 -10
- python_httpfile-0.0.6.dist-info/RECORD +6 -0
- {python_httpfile-0.0.5.3.dist-info → python_httpfile-0.0.6.dist-info}/WHEEL +1 -1
- python_httpfile-0.0.5.3.dist-info/RECORD +0 -6
- {python_httpfile-0.0.5.3.dist-info → python_httpfile-0.0.6.dist-info}/licenses/LICENSE +0 -0
httpfile/__init__.py
CHANGED
|
@@ -4,156 +4,113 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
__author__ = "ChenyangGao <https://chenyanggao.github.io>"
|
|
7
|
-
__version__ = (0, 0,
|
|
8
|
-
__all__ = ["HTTPFileReader", "AsyncHTTPFileReader"]
|
|
7
|
+
__version__ = (0, 0, 6)
|
|
8
|
+
__all__ = ["HTTPFileReader", "AsyncHTTPFileReader", "MultipartHTTPFileReader", "AsyncMultipartHTTPFileReader"]
|
|
9
9
|
|
|
10
|
-
import
|
|
11
|
-
|
|
12
|
-
from collections.abc import Awaitable, Callable, Mapping
|
|
13
|
-
from functools import cached_property, partial
|
|
14
|
-
from importlib.util import find_spec
|
|
10
|
+
from collections.abc import AsyncIterator, Awaitable, Buffer, Callable, Iterable, Iterator
|
|
11
|
+
from functools import cached_property
|
|
15
12
|
from inspect import isawaitable
|
|
16
13
|
from io import (
|
|
17
|
-
BufferedReader, RawIOBase, TextIOWrapper, UnsupportedOperation,
|
|
14
|
+
BufferedReader, BytesIO, RawIOBase, TextIOWrapper, UnsupportedOperation,
|
|
15
|
+
DEFAULT_BUFFER_SIZE,
|
|
18
16
|
)
|
|
19
|
-
|
|
20
|
-
from shutil import COPY_BUFSIZE # type: ignore
|
|
17
|
+
try:
|
|
18
|
+
from shutil import COPY_BUFSIZE # type: ignore
|
|
19
|
+
except ImportError:
|
|
20
|
+
COPY_BUFSIZE = 1 << 16
|
|
21
21
|
from typing import cast, overload, Any, BinaryIO, Literal, Self
|
|
22
|
-
from types import MappingProxyType
|
|
23
22
|
from warnings import warn
|
|
24
23
|
|
|
24
|
+
from argtools import argcount
|
|
25
25
|
from asynctools import ensure_async, run_async
|
|
26
|
-
from
|
|
27
|
-
from
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
if
|
|
59
|
-
|
|
60
|
-
if
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if dont_read:
|
|
68
|
-
return -1
|
|
69
|
-
total = 0
|
|
70
|
-
if hasattr(file, "readinto"):
|
|
71
|
-
readinto = file.readinto
|
|
72
|
-
buf = bytearray(COPY_BUFSIZE)
|
|
73
|
-
while (size := readinto(buf)):
|
|
74
|
-
total += size
|
|
75
|
-
elif hasattr(file, "read"):
|
|
76
|
-
read = file.read
|
|
77
|
-
while (chunk := read(COPY_BUFSIZE)):
|
|
78
|
-
total += len(chunk)
|
|
79
|
-
else:
|
|
80
|
-
return -1
|
|
81
|
-
return total
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
class HTTPFileReader(RawIOBase, BinaryIO):
|
|
85
|
-
url: str | Callable[[], str]
|
|
86
|
-
response: Any
|
|
87
|
-
length: int
|
|
88
|
-
chunked: bool
|
|
89
|
-
start: int
|
|
90
|
-
urlopen: Callable
|
|
91
|
-
headers: Mapping
|
|
92
|
-
seek_threshold: int
|
|
93
|
-
_seekable: bool
|
|
26
|
+
from errno2 import errno
|
|
27
|
+
from filewrap import (
|
|
28
|
+
buffer_length, bytes_iter_to_reader, bytes_iter_to_async_reader,
|
|
29
|
+
AsyncBufferedReader, AsyncTextIOWrapper,
|
|
30
|
+
)
|
|
31
|
+
from http_request import SupportsGeturl
|
|
32
|
+
from http_response import (
|
|
33
|
+
get_filename, get_range, get_status_code, get_total_length, is_range_request,
|
|
34
|
+
)
|
|
35
|
+
from yarl import URL
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def coalesce(*vals, base=None):
|
|
39
|
+
for v in vals:
|
|
40
|
+
if v is not base:
|
|
41
|
+
return v
|
|
42
|
+
return base
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def geturl(url, /) -> str:
|
|
46
|
+
if callable(url):
|
|
47
|
+
url = url()
|
|
48
|
+
if isinstance(url, SupportsGeturl):
|
|
49
|
+
return url.geturl()
|
|
50
|
+
return str(url)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def ageturl(url, /) -> str:
|
|
54
|
+
if callable(url):
|
|
55
|
+
url = url()
|
|
56
|
+
if isawaitable(url):
|
|
57
|
+
url = await url
|
|
58
|
+
if isinstance(url, SupportsGeturl):
|
|
59
|
+
url = url.geturl()
|
|
60
|
+
if isawaitable(url):
|
|
61
|
+
url = await url
|
|
62
|
+
return url
|
|
63
|
+
return str(url)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class HTTPFileReader[Response](RawIOBase, BinaryIO):
|
|
94
67
|
|
|
95
68
|
def __init__(
|
|
96
69
|
self,
|
|
97
70
|
/,
|
|
98
|
-
url:
|
|
99
|
-
|
|
71
|
+
url: (
|
|
72
|
+
str | SupportsGeturl | URL |
|
|
73
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
|
|
74
|
+
),
|
|
100
75
|
start: int = 0,
|
|
101
|
-
# NOTE: If the offset of the forward seek is not higher than this value,
|
|
102
|
-
# it will be directly read and discarded, default to 1 MB
|
|
103
76
|
seek_threshold: int = 1 << 20,
|
|
104
|
-
|
|
77
|
+
request: None | Callable[..., Response] = None,
|
|
78
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
79
|
+
**request_kwargs,
|
|
105
80
|
):
|
|
106
|
-
if
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
if not rng:
|
|
132
|
-
raise OSError(errno.ESPIPE, "non-seekable")
|
|
133
|
-
start = rng[0]
|
|
134
|
-
self.__dict__.update(
|
|
135
|
-
url = url,
|
|
136
|
-
response = response,
|
|
137
|
-
length = get_total_length(response) or 0,
|
|
138
|
-
chunked = is_chunked(response),
|
|
139
|
-
start = start,
|
|
140
|
-
closed = False,
|
|
141
|
-
urlopen = urlopen,
|
|
142
|
-
headers = MappingProxyType(headers),
|
|
143
|
-
seek_threshold = max(seek_threshold, 0),
|
|
144
|
-
_seekable = is_range_request(response),
|
|
145
|
-
)
|
|
81
|
+
if start < 0:
|
|
82
|
+
raise ValueError("`start` cannot be < 0")
|
|
83
|
+
self._closed = False
|
|
84
|
+
self._pos = start
|
|
85
|
+
self._url = url
|
|
86
|
+
self.seek_threshold = max(seek_threshold, 0)
|
|
87
|
+
if request is None:
|
|
88
|
+
from urllib3_future_request import request_sync as request # type: ignore
|
|
89
|
+
self.request = cast(Callable[..., Response], request)
|
|
90
|
+
if isinstance(get_file, str):
|
|
91
|
+
if get_file:
|
|
92
|
+
is_method = get_file.endswith("()")
|
|
93
|
+
attrs = get_file.removesuffix("()").split(".")
|
|
94
|
+
def get_file(file, /):
|
|
95
|
+
for attr in attrs:
|
|
96
|
+
file = getattr(file, attr)
|
|
97
|
+
if is_method:
|
|
98
|
+
file = file()
|
|
99
|
+
return file
|
|
100
|
+
else:
|
|
101
|
+
get_file = lambda file, /: file
|
|
102
|
+
self.get_file = get_file
|
|
103
|
+
self.request_kwargs = request_kwargs
|
|
104
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
105
|
+
headers["accept-encoding"] = "identity"
|
|
146
106
|
|
|
147
107
|
def __del__(self, /):
|
|
148
|
-
|
|
149
|
-
self.close()
|
|
150
|
-
except:
|
|
151
|
-
pass
|
|
108
|
+
self.close()
|
|
152
109
|
|
|
153
|
-
def __enter__(self, /):
|
|
110
|
+
def __enter__(self, /) -> Self:
|
|
154
111
|
return self
|
|
155
112
|
|
|
156
|
-
def __exit__(self, /, *
|
|
113
|
+
def __exit__(self, /, *_) -> None:
|
|
157
114
|
self.close()
|
|
158
115
|
|
|
159
116
|
def __iter__(self, /) -> Self:
|
|
@@ -163,215 +120,378 @@ class HTTPFileReader(RawIOBase, BinaryIO):
|
|
|
163
120
|
return self.length
|
|
164
121
|
|
|
165
122
|
def __next__(self, /) -> bytes:
|
|
166
|
-
line
|
|
167
|
-
if line:
|
|
123
|
+
if line := self.readline():
|
|
168
124
|
return line
|
|
169
|
-
|
|
170
|
-
raise StopIteration
|
|
125
|
+
raise StopIteration
|
|
171
126
|
|
|
172
127
|
def __repr__(self, /) -> str:
|
|
173
128
|
cls = type(self)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
129
|
+
kwargs = {
|
|
130
|
+
"url": self._url,
|
|
131
|
+
"start": self._pos,
|
|
132
|
+
"seek_threshold": self.seek_threshold,
|
|
133
|
+
"request": self.request,
|
|
134
|
+
"get_file": self.get_file,
|
|
135
|
+
**self.request_kwargs,
|
|
136
|
+
}
|
|
137
|
+
return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
|
|
179
138
|
|
|
180
|
-
|
|
181
|
-
|
|
139
|
+
@property
|
|
140
|
+
def closed(self, /) -> bool:
|
|
141
|
+
return self._closed
|
|
182
142
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
143
|
+
@property
|
|
144
|
+
def response_closed(self, /) -> bool:
|
|
145
|
+
file = self.__dict__.get("response")
|
|
146
|
+
while file:
|
|
147
|
+
if hasattr(file, "closed"):
|
|
148
|
+
return file.closed
|
|
149
|
+
elif hasattr(file, "is_closed"):
|
|
150
|
+
closed = file.is_closed
|
|
151
|
+
if callable(closed):
|
|
152
|
+
closed = closed()
|
|
153
|
+
return closed
|
|
154
|
+
file = getattr(file, "raw", None)
|
|
155
|
+
return self.closed
|
|
156
|
+
|
|
157
|
+
def _init_info(self, /, key: str = "length"):
|
|
158
|
+
response = self.__dict__.get("response")
|
|
159
|
+
if response is None:
|
|
160
|
+
from urllib3_future_request import request_sync as request
|
|
161
|
+
request_kwargs = dict(self.request_kwargs, parse=...)
|
|
162
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
163
|
+
headers["range"] = "bytes=0-0"
|
|
164
|
+
response = request(self._geturl(), **request_kwargs)
|
|
165
|
+
self._seekable = not is_range_request(response)
|
|
166
|
+
self.length = coalesce(get_total_length(response), -1)
|
|
167
|
+
match key:
|
|
168
|
+
case "length":
|
|
169
|
+
return self.length
|
|
170
|
+
case "seekable":
|
|
171
|
+
return self._seekable
|
|
186
172
|
|
|
187
173
|
@cached_property
|
|
188
|
-
def
|
|
189
|
-
|
|
190
|
-
self.file.tell()
|
|
191
|
-
return True
|
|
192
|
-
except (AttributeError, TypeError, OSError):
|
|
193
|
-
return False
|
|
174
|
+
def _seekable(self, /) -> bool:
|
|
175
|
+
return self._init_info("seekable")
|
|
194
176
|
|
|
195
|
-
@
|
|
196
|
-
def
|
|
197
|
-
return self.
|
|
177
|
+
@cached_property
|
|
178
|
+
def length(self, /) -> int:
|
|
179
|
+
return self._init_info("length")
|
|
180
|
+
|
|
181
|
+
@cached_property
|
|
182
|
+
def response(self, /):
|
|
183
|
+
self.reconnect()
|
|
184
|
+
return self.__dict__.get("response")
|
|
198
185
|
|
|
199
186
|
@property
|
|
200
|
-
def
|
|
201
|
-
return
|
|
187
|
+
def mode(self, /) -> str:
|
|
188
|
+
return "rb"
|
|
189
|
+
|
|
190
|
+
@property
|
|
191
|
+
def name(self, /) -> str:
|
|
192
|
+
return get_filename(self.response)
|
|
193
|
+
|
|
194
|
+
def _get_file(self, /):
|
|
195
|
+
response = self.__dict__.get("response")
|
|
196
|
+
if not response:
|
|
197
|
+
self.reconnect()
|
|
198
|
+
response = self.__dict__.get("response")
|
|
199
|
+
if not response:
|
|
200
|
+
return None
|
|
201
|
+
if get_file := self.get_file or (getattr(response, "get_file", None)):
|
|
202
|
+
file = get_file(response)
|
|
203
|
+
if hasattr(file, "read"):
|
|
204
|
+
return file
|
|
205
|
+
elif isinstance(file, Iterator):
|
|
206
|
+
return bytes_iter_to_reader(file)
|
|
207
|
+
else:
|
|
208
|
+
if file := getattr(response, "file", None):
|
|
209
|
+
return file
|
|
210
|
+
if hasattr(response, "read"):
|
|
211
|
+
read = response.read
|
|
212
|
+
if argcount(read):
|
|
213
|
+
return response
|
|
214
|
+
for attr in (
|
|
215
|
+
"iter_bytes", "iter_chunks", "iter_chunked", "iter_stream",
|
|
216
|
+
"iter_body", "iter_raw", "iter_content",
|
|
217
|
+
):
|
|
218
|
+
method = getattr(response, attr, None)
|
|
219
|
+
if method is not None:
|
|
220
|
+
file = method()
|
|
221
|
+
if isinstance(file, Iterator):
|
|
222
|
+
return bytes_iter_to_reader(file)
|
|
223
|
+
for attr in ("stream", "body", "raw", "content"):
|
|
224
|
+
file = getattr(response, attr, None)
|
|
225
|
+
if file is not None:
|
|
226
|
+
if hasattr(file, "read") and argcount(getattr(file, "read")):
|
|
227
|
+
return file
|
|
228
|
+
if callable(file):
|
|
229
|
+
file = file()
|
|
230
|
+
if isinstance(file, Buffer):
|
|
231
|
+
return BytesIO(file)
|
|
232
|
+
elif isinstance(file, Iterator):
|
|
233
|
+
return bytes_iter_to_reader(file)
|
|
234
|
+
raise TypeError("can't determine how to `read`")
|
|
235
|
+
|
|
236
|
+
def _geturl(self, /) -> str:
|
|
237
|
+
return geturl(self._url)
|
|
202
238
|
|
|
203
239
|
def close(self, /):
|
|
204
240
|
if not self.closed:
|
|
205
|
-
self.
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
241
|
+
if response := self.__dict__.get("response"):
|
|
242
|
+
try:
|
|
243
|
+
response.close()
|
|
244
|
+
except (AttributeError, TypeError):
|
|
245
|
+
pass
|
|
246
|
+
self._closed = True
|
|
211
247
|
|
|
212
248
|
def fileno(self, /) -> int:
|
|
213
|
-
|
|
249
|
+
file = self.__dict__.get("response")
|
|
250
|
+
while file:
|
|
251
|
+
if hasattr(file, "fileno"):
|
|
252
|
+
return file.fileno()
|
|
253
|
+
file = getattr(file, "raw", None)
|
|
254
|
+
return 0
|
|
214
255
|
|
|
215
256
|
def flush(self, /):
|
|
216
|
-
if self.closed:
|
|
217
|
-
|
|
218
|
-
|
|
257
|
+
if not self.closed:
|
|
258
|
+
file = self.__dict__.get("response")
|
|
259
|
+
while file:
|
|
260
|
+
if hasattr(file, "flush"):
|
|
261
|
+
return file.flush()
|
|
262
|
+
file = getattr(file, "raw", None)
|
|
219
263
|
|
|
220
264
|
def isatty(self, /) -> bool:
|
|
221
265
|
return False
|
|
222
266
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
267
|
+
def _readinto(self, buffer: Buffer, /) -> int:
|
|
268
|
+
read = self.file.read
|
|
269
|
+
m = memoryview(buffer).cast("B")
|
|
270
|
+
size = len(m)
|
|
271
|
+
start = 0
|
|
272
|
+
last_intv = size - size % COPY_BUFSIZE
|
|
273
|
+
while start < size:
|
|
274
|
+
if start >= last_intv:
|
|
275
|
+
delta = size - start
|
|
276
|
+
else:
|
|
277
|
+
delta = COPY_BUFSIZE
|
|
278
|
+
data = read(delta)
|
|
279
|
+
data_len = len(data)
|
|
280
|
+
m[:data_len] = data
|
|
281
|
+
start += data_len
|
|
282
|
+
if data_len < delta:
|
|
283
|
+
break
|
|
284
|
+
return start
|
|
230
285
|
|
|
231
|
-
def
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
if size == 0 or not self.chunked and self.tell() >= self.length:
|
|
235
|
-
return b""
|
|
236
|
-
if self.file_closed:
|
|
237
|
-
self.reconnect()
|
|
286
|
+
def _readline(self, size: None | int = -1, /) -> bytes:
|
|
287
|
+
read = self.file.read
|
|
288
|
+
buf = bytearray()
|
|
238
289
|
if size is None or size < 0:
|
|
239
|
-
|
|
290
|
+
while b := read(1):
|
|
291
|
+
buf += b
|
|
292
|
+
if b == b"\n":
|
|
293
|
+
break
|
|
294
|
+
else:
|
|
295
|
+
while size and (b := read(1)):
|
|
296
|
+
buf += b
|
|
297
|
+
if b == b"\n":
|
|
298
|
+
break
|
|
299
|
+
size -= 1
|
|
300
|
+
return bytes(buf)
|
|
301
|
+
|
|
302
|
+
def _readlines(self, hint: int = -1, /) -> list[bytes]:
|
|
303
|
+
try:
|
|
304
|
+
readline = self.file.readline
|
|
305
|
+
except AttributeError:
|
|
306
|
+
readline = self._readline
|
|
307
|
+
ls: list[bytes]
|
|
308
|
+
add = ls.append
|
|
309
|
+
if hint <= 0:
|
|
310
|
+
while l := readline():
|
|
311
|
+
add(l)
|
|
240
312
|
else:
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
return
|
|
313
|
+
while hint >= 0 and (l := readline()):
|
|
314
|
+
add(l)
|
|
315
|
+
hint -= len(l)
|
|
316
|
+
return ls
|
|
245
317
|
|
|
246
318
|
def readable(self, /) -> bool:
|
|
247
319
|
return True
|
|
248
320
|
|
|
249
|
-
def
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
if
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
321
|
+
def read(self, size: int = -1, /) -> bytes:
|
|
322
|
+
pos = self._pos
|
|
323
|
+
if size == 0:
|
|
324
|
+
return b""
|
|
325
|
+
length = self.__dict__.get("length")
|
|
326
|
+
if length is not None:
|
|
327
|
+
if length == 0 or length > 0 and pos >= length:
|
|
328
|
+
return b""
|
|
329
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
330
|
+
self.reconnect(pos)
|
|
331
|
+
if file := self.file:
|
|
332
|
+
try:
|
|
333
|
+
if size is None or size < 0:
|
|
334
|
+
data = file.read()
|
|
335
|
+
else:
|
|
336
|
+
data = file.read(size)
|
|
337
|
+
if data:
|
|
338
|
+
self._pos = pos + len(data)
|
|
264
339
|
return data
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
340
|
+
except:
|
|
341
|
+
self._pos = pos
|
|
342
|
+
self.close()
|
|
343
|
+
raise
|
|
344
|
+
return b""
|
|
345
|
+
|
|
346
|
+
def readinto(self, buffer: Buffer, /) -> int:
|
|
347
|
+
pos = self._pos
|
|
348
|
+
if not buffer_length(buffer):
|
|
349
|
+
return 0
|
|
350
|
+
length = self.__dict__.get("length")
|
|
351
|
+
if length is not None:
|
|
352
|
+
if length == 0 or length > 0 and pos >= length:
|
|
353
|
+
return 0
|
|
354
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
355
|
+
self.reconnect(pos)
|
|
356
|
+
if file := self.file:
|
|
357
|
+
try:
|
|
358
|
+
readinto = file.readinto
|
|
359
|
+
except AttributeError:
|
|
360
|
+
readinto = self._readinto
|
|
361
|
+
try:
|
|
362
|
+
size = readinto(buffer)
|
|
363
|
+
if size:
|
|
364
|
+
self._pos = pos + size
|
|
365
|
+
return size
|
|
366
|
+
except:
|
|
367
|
+
self._pos = pos
|
|
368
|
+
self.close()
|
|
369
|
+
raise
|
|
370
|
+
return 0
|
|
269
371
|
|
|
270
372
|
def readline(self, size: None | int = -1, /) -> bytes:
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
if size == 0 or not self.chunked and self.tell() >= self.length:
|
|
373
|
+
pos = self._pos
|
|
374
|
+
if size == 0:
|
|
274
375
|
return b""
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
376
|
+
length = self.__dict__.get("length")
|
|
377
|
+
if length is not None:
|
|
378
|
+
if length == 0 or length > 0 and pos >= length:
|
|
379
|
+
return b""
|
|
380
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
381
|
+
self.reconnect(pos)
|
|
382
|
+
if file := self.file:
|
|
383
|
+
try:
|
|
384
|
+
readline = file.readline
|
|
385
|
+
except AttributeError:
|
|
386
|
+
readline = self._readline
|
|
387
|
+
try:
|
|
388
|
+
if size is None or size < 0:
|
|
389
|
+
data = readline()
|
|
390
|
+
else:
|
|
391
|
+
data = readline(size)
|
|
392
|
+
if data:
|
|
393
|
+
self._pos = pos + len(data)
|
|
394
|
+
return data
|
|
395
|
+
except:
|
|
396
|
+
self._pos = pos
|
|
397
|
+
self.close()
|
|
398
|
+
raise
|
|
399
|
+
return b""
|
|
285
400
|
|
|
286
401
|
def readlines(self, hint: int = -1, /) -> list[bytes]:
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
if
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
402
|
+
pos = self._pos
|
|
403
|
+
length = self.__dict__.get("length")
|
|
404
|
+
if length is not None:
|
|
405
|
+
if length == 0 or length > 0 and pos >= length:
|
|
406
|
+
return []
|
|
407
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
408
|
+
self.reconnect(pos)
|
|
409
|
+
if file := self.file:
|
|
410
|
+
try:
|
|
411
|
+
readlines = file.readlines
|
|
412
|
+
except AttributeError:
|
|
413
|
+
readlines = self._readlines
|
|
414
|
+
try:
|
|
415
|
+
ls = readlines(hint)
|
|
416
|
+
if ls:
|
|
417
|
+
self._pos = pos + sum(map(len, ls))
|
|
418
|
+
return ls
|
|
419
|
+
except:
|
|
420
|
+
self._pos = pos
|
|
421
|
+
self.close()
|
|
422
|
+
raise
|
|
423
|
+
return []
|
|
298
424
|
|
|
299
425
|
def reconnect(self, /, start: None | int = None) -> int:
|
|
300
|
-
if not self._seekable:
|
|
301
|
-
if start is None and self.tell() or start:
|
|
302
|
-
raise OSError(errno.EOPNOTSUPP, "Unsupport for reconnection of non-seekable streams.")
|
|
303
|
-
start = 0
|
|
304
426
|
if start is None:
|
|
305
|
-
start = self.
|
|
427
|
+
start = self._pos
|
|
428
|
+
if start and not self.seekable():
|
|
429
|
+
raise OSError(errno.EOPNOTSUPP, "unsupport for reconnection of non-seekable streams")
|
|
430
|
+
request_kwargs = self.request_kwargs
|
|
431
|
+
headers = request_kwargs["headers"] = request_kwargs["headers"]
|
|
432
|
+
if start >= 0:
|
|
433
|
+
headers["range"] = f"bytes={start}-"
|
|
306
434
|
elif start < 0:
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
if
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
start=
|
|
325
|
-
|
|
326
|
-
)
|
|
435
|
+
headers["range"] = f"bytes={start}"
|
|
436
|
+
self.close()
|
|
437
|
+
response = self.request(self._geturl(), **request_kwargs)
|
|
438
|
+
status_code = get_status_code(response)
|
|
439
|
+
if not 200 <= status_code < 300:
|
|
440
|
+
raise OSError(
|
|
441
|
+
errno.EIO,
|
|
442
|
+
{
|
|
443
|
+
"code": status_code,
|
|
444
|
+
"response": response,
|
|
445
|
+
"reason": "status code must be in the `range(200, 300)`",
|
|
446
|
+
},
|
|
447
|
+
)
|
|
448
|
+
if start:
|
|
449
|
+
rng = get_range(response)
|
|
450
|
+
if not rng:
|
|
451
|
+
raise OSError(errno.ESPIPE, "non-seekable")
|
|
452
|
+
start = self._pos = rng[0]
|
|
453
|
+
self.response = response
|
|
454
|
+
self.file = self._get_file()
|
|
455
|
+
self._closed = False
|
|
327
456
|
return start
|
|
328
457
|
|
|
329
458
|
def seek(self, pos: int, whence: int = 0, /) -> int:
|
|
330
|
-
if self.
|
|
331
|
-
raise ValueError("I/O operation on closed file.")
|
|
332
|
-
if not self._seekable:
|
|
459
|
+
if not self.seekable():
|
|
333
460
|
raise OSError(errno.EINVAL, "not a seekable stream")
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
while size > COPY_BUFSIZE:
|
|
347
|
-
readinto(buf)
|
|
348
|
-
size -= COPY_BUFSIZE
|
|
349
|
-
self.read(size)
|
|
350
|
-
else:
|
|
351
|
-
self.reconnect(pos)
|
|
461
|
+
old_pos = self._pos
|
|
462
|
+
match whence:
|
|
463
|
+
case 1:
|
|
464
|
+
pos = old_pos + pos
|
|
465
|
+
case 2:
|
|
466
|
+
length = self.__dict__.get("length")
|
|
467
|
+
if length is None:
|
|
468
|
+
length = self._init_info()
|
|
469
|
+
pos += length
|
|
470
|
+
if pos < 0:
|
|
471
|
+
pos = 0
|
|
472
|
+
if pos == old_pos:
|
|
352
473
|
return pos
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
474
|
+
if (self.__dict__.get("response") and
|
|
475
|
+
self.file and
|
|
476
|
+
not self.closed and
|
|
477
|
+
not self.response_closed and
|
|
478
|
+
pos > old_pos and
|
|
479
|
+
(size := pos - old_pos) <= self.seek_threshold
|
|
480
|
+
):
|
|
481
|
+
read = self.read
|
|
482
|
+
while size > COPY_BUFSIZE:
|
|
483
|
+
read(COPY_BUFSIZE)
|
|
484
|
+
size -= COPY_BUFSIZE
|
|
485
|
+
read(size)
|
|
486
|
+
return self.reconnect(pos)
|
|
361
487
|
|
|
362
488
|
def seekable(self, /) -> bool:
|
|
363
489
|
return self._seekable
|
|
364
490
|
|
|
365
491
|
def tell(self, /) -> int:
|
|
366
|
-
|
|
367
|
-
start = self.start
|
|
368
|
-
if start >= self.length:
|
|
369
|
-
return start
|
|
370
|
-
return start + self.file.tell()
|
|
371
|
-
else:
|
|
372
|
-
return self.start
|
|
492
|
+
return self._pos
|
|
373
493
|
|
|
374
|
-
def truncate(self, size: None | int = None, /):
|
|
494
|
+
def truncate(self, size: None | int = None, /) -> int:
|
|
375
495
|
raise UnsupportedOperation(errno.ENOTSUP, "truncate")
|
|
376
496
|
|
|
377
497
|
def writable(self, /) -> bool:
|
|
@@ -388,17 +508,21 @@ class HTTPFileReader(RawIOBase, BinaryIO):
|
|
|
388
508
|
def open(
|
|
389
509
|
cls,
|
|
390
510
|
/,
|
|
391
|
-
url:
|
|
511
|
+
url: (
|
|
512
|
+
str | SupportsGeturl | URL |
|
|
513
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
|
|
514
|
+
),
|
|
392
515
|
mode: Literal["br", "rb"],
|
|
393
516
|
*,
|
|
394
517
|
buffering: None | int = None,
|
|
395
518
|
encoding: None | str = None,
|
|
396
519
|
errors: None | str = None,
|
|
397
520
|
newline: None | str = None,
|
|
398
|
-
headers: None | Mapping = None,
|
|
399
521
|
start: int = 0,
|
|
400
522
|
seek_threshold: int = 1 << 20,
|
|
401
|
-
|
|
523
|
+
request: None | Callable[..., Response] = None,
|
|
524
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
525
|
+
**request_kwargs,
|
|
402
526
|
) -> Self | BufferedReader:
|
|
403
527
|
...
|
|
404
528
|
@overload
|
|
@@ -406,46 +530,55 @@ class HTTPFileReader(RawIOBase, BinaryIO):
|
|
|
406
530
|
def open(
|
|
407
531
|
cls,
|
|
408
532
|
/,
|
|
409
|
-
url:
|
|
533
|
+
url: (
|
|
534
|
+
str | SupportsGeturl | URL |
|
|
535
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
|
|
536
|
+
),
|
|
410
537
|
mode: Literal["r", "rt", "tr"] = "r",
|
|
411
538
|
*,
|
|
412
539
|
buffering: None | int = None,
|
|
413
540
|
encoding: None | str = None,
|
|
414
541
|
errors: None | str = None,
|
|
415
542
|
newline: None | str = None,
|
|
416
|
-
headers: None | Mapping = None,
|
|
417
543
|
start: int = 0,
|
|
418
544
|
seek_threshold: int = 1 << 20,
|
|
419
|
-
|
|
545
|
+
request: None | Callable[..., Response] = None,
|
|
546
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
547
|
+
**request_kwargs,
|
|
420
548
|
) -> TextIOWrapper:
|
|
421
549
|
...
|
|
422
550
|
@classmethod
|
|
423
551
|
def open(
|
|
424
552
|
cls,
|
|
425
553
|
/,
|
|
426
|
-
url:
|
|
554
|
+
url: (
|
|
555
|
+
str | SupportsGeturl | URL |
|
|
556
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
|
|
557
|
+
),
|
|
427
558
|
mode: Literal["r", "rt", "tr", "br", "rb"] = "r",
|
|
428
559
|
*,
|
|
429
560
|
buffering: None | int = None,
|
|
430
561
|
encoding: None | str = None,
|
|
431
562
|
errors: None | str = None,
|
|
432
563
|
newline: None | str = None,
|
|
433
|
-
headers: None | Mapping = None,
|
|
434
564
|
start: int = 0,
|
|
435
565
|
seek_threshold: int = 1 << 20,
|
|
436
|
-
|
|
566
|
+
request: None | Callable[..., Response] = None,
|
|
567
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
568
|
+
**request_kwargs,
|
|
437
569
|
) -> Self | BufferedReader | TextIOWrapper:
|
|
438
570
|
file = cls(
|
|
439
571
|
url=url,
|
|
440
|
-
headers=headers,
|
|
441
572
|
start=start,
|
|
442
573
|
seek_threshold=seek_threshold,
|
|
443
|
-
|
|
574
|
+
request=request,
|
|
575
|
+
get_file=get_file,
|
|
576
|
+
**request_kwargs,
|
|
444
577
|
)
|
|
445
578
|
if mode not in ("r", "rt", "tr", "rb", "br"):
|
|
446
579
|
raise OSError(errno.EINVAL, f"invalid (or unsupported) mode: {mode!r}")
|
|
447
580
|
return file.wrap(
|
|
448
|
-
text_mode="b" not in mode, # type: ignore
|
|
581
|
+
text_mode="b" not in cast(str, mode), # type: ignore
|
|
449
582
|
buffering=buffering,
|
|
450
583
|
encoding=encoding,
|
|
451
584
|
errors=errors,
|
|
@@ -507,6 +640,8 @@ class HTTPFileReader(RawIOBase, BinaryIO):
|
|
|
507
640
|
line_buffering = True
|
|
508
641
|
else:
|
|
509
642
|
buffer_size = buffering
|
|
643
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
644
|
+
self.reconnect()
|
|
510
645
|
raw = self
|
|
511
646
|
buffer: BufferedReader = BufferedReader(raw, buffer_size)
|
|
512
647
|
if text_mode:
|
|
@@ -521,34 +656,81 @@ class HTTPFileReader(RawIOBase, BinaryIO):
|
|
|
521
656
|
return buffer
|
|
522
657
|
|
|
523
658
|
|
|
524
|
-
class AsyncHTTPFileReader(
|
|
525
|
-
url: str | Callable[[], Awaitable[str]] # type: ignore
|
|
659
|
+
class AsyncHTTPFileReader[Response](RawIOBase, BinaryIO):
|
|
526
660
|
|
|
527
661
|
def __init__(
|
|
528
662
|
self,
|
|
529
663
|
/,
|
|
530
|
-
url:
|
|
531
|
-
|
|
664
|
+
url: (
|
|
665
|
+
str | SupportsGeturl | URL |
|
|
666
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
667
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
|
|
668
|
+
),
|
|
532
669
|
start: int = 0,
|
|
533
670
|
seek_threshold: int = 1 << 20,
|
|
534
|
-
|
|
671
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
672
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
673
|
+
**request_kwargs,
|
|
535
674
|
):
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
675
|
+
if start < 0:
|
|
676
|
+
raise ValueError("`start` cannot be < 0")
|
|
677
|
+
self._closed = False
|
|
678
|
+
self._pos = start
|
|
679
|
+
self._url = url
|
|
680
|
+
self.seek_threshold = max(seek_threshold, 0)
|
|
681
|
+
if request is None:
|
|
682
|
+
from urllib3_future_request import request_async as request # type: ignore
|
|
683
|
+
self.request = cast(Callable[..., Awaitable[Response]], request)
|
|
684
|
+
if isinstance(get_file, str):
|
|
685
|
+
if get_file:
|
|
686
|
+
is_method = get_file.endswith("()")
|
|
687
|
+
attrs = get_file.removesuffix("()").split(".")
|
|
688
|
+
def get_file(file, /):
|
|
689
|
+
for attr in attrs:
|
|
690
|
+
file = getattr(file, attr)
|
|
691
|
+
if is_method:
|
|
692
|
+
file = file()
|
|
693
|
+
return file
|
|
694
|
+
else:
|
|
695
|
+
get_file = lambda file, /: file
|
|
696
|
+
self.get_file = get_file
|
|
697
|
+
self.request_kwargs = request_kwargs
|
|
698
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
699
|
+
headers["accept-encoding"] = "identity"
|
|
700
|
+
|
|
701
|
+
@classmethod
|
|
702
|
+
async def new(
|
|
703
|
+
cls,
|
|
704
|
+
/,
|
|
705
|
+
url: (
|
|
706
|
+
str | SupportsGeturl | URL |
|
|
707
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
708
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
|
|
709
|
+
),
|
|
710
|
+
start: int = 0,
|
|
711
|
+
seek_threshold: int = 1 << 20,
|
|
712
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
713
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
714
|
+
**request_kwargs,
|
|
715
|
+
) -> Self:
|
|
716
|
+
self = cls(
|
|
717
|
+
url,
|
|
540
718
|
start=start,
|
|
541
719
|
seek_threshold=seek_threshold,
|
|
542
|
-
|
|
543
|
-
|
|
720
|
+
request=request,
|
|
721
|
+
get_file=get_file,
|
|
722
|
+
**request_kwargs,
|
|
723
|
+
)
|
|
724
|
+
await self.reconnect()
|
|
725
|
+
return self
|
|
726
|
+
|
|
727
|
+
def __del__(self, /):
|
|
728
|
+
self.close()
|
|
544
729
|
|
|
545
730
|
async def __aenter__(self, /) -> Self:
|
|
546
|
-
if _initing := self.__dict__.get("_initing"):
|
|
547
|
-
await _initing
|
|
548
|
-
self.__dict__.pop("_initing", None)
|
|
549
731
|
return self
|
|
550
732
|
|
|
551
|
-
async def __aexit__(self, /, *
|
|
733
|
+
async def __aexit__(self, /, *_):
|
|
552
734
|
await self.aclose()
|
|
553
735
|
|
|
554
736
|
def __aiter__(self, /) -> Self:
|
|
@@ -557,282 +739,456 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
557
739
|
async def __anext__(self, /) -> bytes:
|
|
558
740
|
if line := await self.readline():
|
|
559
741
|
return line
|
|
560
|
-
|
|
561
|
-
raise StopAsyncIteration
|
|
742
|
+
raise StopAsyncIteration
|
|
562
743
|
|
|
563
|
-
|
|
564
|
-
self
|
|
565
|
-
/,
|
|
566
|
-
url: str | Callable[[], str] | Callable[[], Awaitable[str]],
|
|
567
|
-
headers: None | Mapping = None,
|
|
568
|
-
start: int = 0,
|
|
569
|
-
seek_threshold: int = 1 << 20,
|
|
570
|
-
urlopen = None,
|
|
571
|
-
):
|
|
572
|
-
if headers:
|
|
573
|
-
headers = {**headers, "accept-encoding": "identity"}
|
|
574
|
-
else:
|
|
575
|
-
headers = {"accept-encoding": "identity"}
|
|
576
|
-
if start > 0:
|
|
577
|
-
headers["range"] = f"bytes={start}-"
|
|
578
|
-
elif start < 0:
|
|
579
|
-
headers["range"] = f"bytes={start}"
|
|
580
|
-
if urlopen is None:
|
|
581
|
-
from http_request.extension.request import urlopen
|
|
582
|
-
if callable(url):
|
|
583
|
-
geturl = url
|
|
584
|
-
async def url() -> str:
|
|
585
|
-
url = geturl()
|
|
586
|
-
if isawaitable(url):
|
|
587
|
-
url = await url
|
|
588
|
-
headers_extra = getattr(url, "headers")
|
|
589
|
-
if headers_extra:
|
|
590
|
-
headers.update(headers_extra)
|
|
591
|
-
return cast(str, url)
|
|
592
|
-
elif hasattr(url, "headers"):
|
|
593
|
-
headers_extra = getattr(url, "headers")
|
|
594
|
-
if headers_extra:
|
|
595
|
-
headers.update(headers_extra)
|
|
596
|
-
urlopen = ensure_async(urlopen, threaded=True)
|
|
597
|
-
response = await urlopen((await url()) if callable(url) else url, headers=headers)
|
|
598
|
-
if start:
|
|
599
|
-
rng = get_range(response)
|
|
600
|
-
if not rng:
|
|
601
|
-
raise OSError(errno.ESPIPE, "non-seekable")
|
|
602
|
-
start = rng[0]
|
|
603
|
-
self.__dict__.update(
|
|
604
|
-
url = url,
|
|
605
|
-
response = response,
|
|
606
|
-
length = get_total_length(response) or 0,
|
|
607
|
-
chunked = is_chunked(response),
|
|
608
|
-
start = start,
|
|
609
|
-
urlopen = urlopen,
|
|
610
|
-
headers = MappingProxyType(headers),
|
|
611
|
-
seek_threshold = max(seek_threshold, 0),
|
|
612
|
-
_seekable = is_range_request(response),
|
|
613
|
-
)
|
|
614
|
-
self.__dict__.pop("_initing", None)
|
|
744
|
+
def __len__(self, /) -> int:
|
|
745
|
+
return self.length
|
|
615
746
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
747
|
+
def __repr__(self, /) -> str:
|
|
748
|
+
cls = type(self)
|
|
749
|
+
kwargs = {
|
|
750
|
+
"url": self._url,
|
|
751
|
+
"start": self._pos,
|
|
752
|
+
"seek_threshold": self.seek_threshold,
|
|
753
|
+
"request": self.request,
|
|
754
|
+
"get_file": self.get_file,
|
|
755
|
+
**self.request_kwargs,
|
|
756
|
+
}
|
|
757
|
+
return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
|
|
620
758
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
759
|
+
@property
|
|
760
|
+
def closed(self, /) -> bool:
|
|
761
|
+
return self._closed
|
|
762
|
+
|
|
763
|
+
@property
|
|
764
|
+
def response_closed(self, /) -> bool:
|
|
765
|
+
file = self.__dict__.get("response")
|
|
766
|
+
while file:
|
|
767
|
+
if hasattr(file, "closed"):
|
|
768
|
+
return file.closed
|
|
769
|
+
elif hasattr(file, "is_closed"):
|
|
770
|
+
closed = file.is_closed
|
|
771
|
+
if callable(closed):
|
|
772
|
+
closed = closed()
|
|
773
|
+
return closed
|
|
774
|
+
file = getattr(file, "raw", None)
|
|
775
|
+
return self.closed
|
|
776
|
+
|
|
777
|
+
async def _init_info(self, /, key: str = "length"):
|
|
778
|
+
response = self.__dict__.get("response")
|
|
779
|
+
if response is None:
|
|
780
|
+
from urllib3_future_request import request_async as request
|
|
781
|
+
request_kwargs = dict(self.request_kwargs, parse=...)
|
|
782
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
783
|
+
headers["range"] = "bytes=0-0"
|
|
784
|
+
response = await request(await self._geturl(), **request_kwargs)
|
|
785
|
+
self._seekable = not is_range_request(response)
|
|
786
|
+
self.length = coalesce(get_total_length(response), -1)
|
|
787
|
+
match key:
|
|
788
|
+
case "length":
|
|
789
|
+
return self.length
|
|
790
|
+
case "seekable":
|
|
791
|
+
return self._seekable
|
|
792
|
+
|
|
793
|
+
@cached_property
|
|
794
|
+
def _seekable(self, /) -> bool:
|
|
795
|
+
return run_async(self._init_info("seekable"))
|
|
796
|
+
|
|
797
|
+
@cached_property
|
|
798
|
+
def length(self, /) -> int:
|
|
799
|
+
return run_async(self._init_info("length"))
|
|
800
|
+
|
|
801
|
+
@cached_property
|
|
802
|
+
def response(self, /):
|
|
803
|
+
run_async(self.reconnect())
|
|
804
|
+
return self.__dict__.get("response")
|
|
805
|
+
|
|
806
|
+
@property
|
|
807
|
+
def mode(self, /) -> str:
|
|
808
|
+
return "rb"
|
|
809
|
+
|
|
810
|
+
@property
|
|
811
|
+
def name(self, /) -> str:
|
|
812
|
+
return get_filename(self.response)
|
|
813
|
+
|
|
814
|
+
async def _get_file(self, /):
|
|
815
|
+
response = self.__dict__.get("response")
|
|
816
|
+
if not response:
|
|
817
|
+
await self.reconnect()
|
|
818
|
+
response = self.__dict__.get("response")
|
|
819
|
+
if not response:
|
|
820
|
+
return None
|
|
821
|
+
if get_file := self.get_file or (getattr(response, "get_file", None)):
|
|
822
|
+
file = get_file(response)
|
|
823
|
+
if isawaitable(file):
|
|
824
|
+
file = await file
|
|
825
|
+
if hasattr(file, "aread") or hasattr(file, "read"):
|
|
826
|
+
return file
|
|
827
|
+
elif isinstance(file, (Iterator, AsyncIterator)):
|
|
828
|
+
return bytes_iter_to_async_reader(file, threaded=True)
|
|
829
|
+
else:
|
|
830
|
+
if file := getattr(response, "file", None):
|
|
831
|
+
return file
|
|
832
|
+
for attr in ("aread", "read"):
|
|
833
|
+
method = getattr(response, attr, None)
|
|
834
|
+
if method is not None and argcount(method):
|
|
835
|
+
return response
|
|
836
|
+
for attr in (
|
|
837
|
+
"aiter_bytes", "aiter_chunks", "aiter_chunked", "aiter_stream",
|
|
838
|
+
"aiter_body", "aiter_raw", "aiter_content",
|
|
839
|
+
"iter_bytes", "iter_chunks", "iter_chunked", "iter_stream",
|
|
840
|
+
"iter_body", "iter_raw", "iter_content",
|
|
841
|
+
):
|
|
842
|
+
method = getattr(response, attr, None)
|
|
843
|
+
if method is not None:
|
|
844
|
+
file = method()
|
|
845
|
+
if isinstance(file, (Iterator, AsyncIterator)):
|
|
846
|
+
return bytes_iter_to_async_reader(file, threaded=True)
|
|
847
|
+
for attr in (
|
|
848
|
+
"astream", "abody", "araw", "acontent",
|
|
849
|
+
"stream", "body", "raw", "content",
|
|
850
|
+
):
|
|
851
|
+
file = getattr(response, attr, None)
|
|
852
|
+
if file is not None:
|
|
853
|
+
if isawaitable(file):
|
|
854
|
+
file = await file
|
|
855
|
+
for attr in ("aread", "read"):
|
|
856
|
+
method = getattr(file, attr, None)
|
|
857
|
+
if method is not None and argcount(method):
|
|
858
|
+
return file
|
|
859
|
+
if callable(file):
|
|
860
|
+
file = file()
|
|
861
|
+
if isawaitable(file):
|
|
862
|
+
file = await file
|
|
863
|
+
if isinstance(file, Buffer):
|
|
864
|
+
return BytesIO(file)
|
|
865
|
+
elif isinstance(file, (Iterator, AsyncIterator)):
|
|
866
|
+
return bytes_iter_to_async_reader(file, threaded=True)
|
|
867
|
+
raise TypeError("can't determine how to `read`")
|
|
868
|
+
|
|
869
|
+
async def _geturl(self, /) -> str:
|
|
870
|
+
return await ageturl(self._url)
|
|
871
|
+
|
|
872
|
+
async def close_response(self, /):
|
|
873
|
+
if response := self.__dict__.get("response"):
|
|
624
874
|
try:
|
|
625
|
-
ret =
|
|
875
|
+
ret = response.aclose()
|
|
626
876
|
except (AttributeError, TypeError):
|
|
627
877
|
try:
|
|
628
|
-
ret =
|
|
878
|
+
ret = response.close()
|
|
629
879
|
except (AttributeError, TypeError):
|
|
630
880
|
return
|
|
631
881
|
if isawaitable(ret):
|
|
632
|
-
|
|
882
|
+
await ret
|
|
633
883
|
|
|
634
|
-
async def
|
|
884
|
+
async def aclose(self, /):
|
|
885
|
+
if not self.closed:
|
|
886
|
+
await self.close_response()
|
|
887
|
+
self._closed = True
|
|
888
|
+
|
|
889
|
+
def close(self, /):
|
|
890
|
+
from asynctools import run_async
|
|
891
|
+
run_async(self.aclose())
|
|
892
|
+
|
|
893
|
+
def fileno(self, /) -> int:
|
|
894
|
+
file = self.__dict__.get("response")
|
|
895
|
+
while file:
|
|
896
|
+
if hasattr(file, "fileno"):
|
|
897
|
+
return file.fileno()
|
|
898
|
+
file = getattr(file, "raw", None)
|
|
899
|
+
return 0
|
|
900
|
+
|
|
901
|
+
async def flush(self, /):
|
|
902
|
+
if not self.closed:
|
|
903
|
+
file = self.__dict__.get("response")
|
|
904
|
+
while file:
|
|
905
|
+
if hasattr(file, "flush"):
|
|
906
|
+
ret = file.flush()
|
|
907
|
+
if isawaitable(ret):
|
|
908
|
+
ret = await ret
|
|
909
|
+
return ret
|
|
910
|
+
file = getattr(file, "raw", None)
|
|
911
|
+
|
|
912
|
+
def isatty(self, /) -> bool:
|
|
913
|
+
return False
|
|
914
|
+
|
|
915
|
+
def _get_read(self, /):
|
|
916
|
+
try:
|
|
917
|
+
return self.file.aread
|
|
918
|
+
except AttributeError:
|
|
919
|
+
return ensure_async(self.file.read, threaded=True)
|
|
920
|
+
|
|
921
|
+
async def _readinto(self, buffer: Buffer, /) -> int:
|
|
922
|
+
read = self._get_read()
|
|
923
|
+
m = memoryview(buffer).cast("B")
|
|
924
|
+
size = len(m)
|
|
925
|
+
start = 0
|
|
926
|
+
last_intv = size - size % COPY_BUFSIZE
|
|
927
|
+
while start < size:
|
|
928
|
+
if start >= last_intv:
|
|
929
|
+
delta = size - start
|
|
930
|
+
else:
|
|
931
|
+
delta = COPY_BUFSIZE
|
|
932
|
+
data = await read(delta)
|
|
933
|
+
data_len = len(data)
|
|
934
|
+
m[:data_len] = data
|
|
935
|
+
start += data_len
|
|
936
|
+
if data_len < delta:
|
|
937
|
+
break
|
|
938
|
+
return start
|
|
939
|
+
|
|
940
|
+
async def _readline(self, size: None | int = -1, /) -> bytes:
|
|
941
|
+
read = self._get_read()
|
|
942
|
+
buf = bytearray()
|
|
943
|
+
if size is None or size < 0:
|
|
944
|
+
while b := await read(1):
|
|
945
|
+
buf += b
|
|
946
|
+
if b == b"\n":
|
|
947
|
+
break
|
|
948
|
+
else:
|
|
949
|
+
while size and (b := await read(1)):
|
|
950
|
+
buf += b
|
|
951
|
+
if b == b"\n":
|
|
952
|
+
break
|
|
953
|
+
size -= 1
|
|
954
|
+
return bytes(buf)
|
|
955
|
+
|
|
956
|
+
async def _readlines(self, hint: int = -1, /) -> list[bytes]:
|
|
635
957
|
try:
|
|
636
|
-
|
|
637
|
-
except
|
|
958
|
+
readline = self.file.areadline
|
|
959
|
+
except AttributeError:
|
|
638
960
|
try:
|
|
639
|
-
|
|
640
|
-
except
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
961
|
+
readline = ensure_async(self.file.readline, threaded=True)
|
|
962
|
+
except AttributeError:
|
|
963
|
+
readline = self._readline
|
|
964
|
+
ls: list[bytes]
|
|
965
|
+
add = ls.append
|
|
966
|
+
if hint <= 0:
|
|
967
|
+
while l := await readline():
|
|
968
|
+
add(l)
|
|
969
|
+
else:
|
|
970
|
+
while hint >= 0 and (l := await readline()):
|
|
971
|
+
add(l)
|
|
972
|
+
hint -= len(l)
|
|
973
|
+
return ls
|
|
644
974
|
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
raise ValueError("I/O operation on closed file.")
|
|
648
|
-
if _initing := self.__dict__.get("_initing"):
|
|
649
|
-
await _initing
|
|
650
|
-
self.__dict__.pop("_initing", None)
|
|
651
|
-
return await ensure_async(self.file.flush, threaded=True)()
|
|
975
|
+
def readable(self, /) -> bool:
|
|
976
|
+
return True
|
|
652
977
|
|
|
653
978
|
async def read(self, size: int = -1, /) -> bytes: # type: ignore
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
if _initing := self.__dict__.get("_initing"):
|
|
657
|
-
await _initing
|
|
658
|
-
self.__dict__.pop("_initing", None)
|
|
659
|
-
if size == 0 or not self.chunked and self.tell() >= self.length:
|
|
979
|
+
pos = self._pos
|
|
980
|
+
if size == 0:
|
|
660
981
|
return b""
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
982
|
+
length = self.__dict__.get("length")
|
|
983
|
+
if length is not None:
|
|
984
|
+
if length == 0 or length > 0 and pos >= length:
|
|
985
|
+
return b""
|
|
986
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
987
|
+
await self.reconnect(pos)
|
|
988
|
+
if self.file:
|
|
989
|
+
read = self._get_read()
|
|
990
|
+
try:
|
|
991
|
+
if size is None or size < 0:
|
|
992
|
+
data = await read()
|
|
993
|
+
else:
|
|
994
|
+
data = await read(size)
|
|
995
|
+
if data:
|
|
996
|
+
self._pos = pos + len(data)
|
|
997
|
+
return data
|
|
998
|
+
except:
|
|
999
|
+
self._pos = pos
|
|
1000
|
+
await self.aclose()
|
|
1001
|
+
raise
|
|
1002
|
+
return b""
|
|
671
1003
|
|
|
672
1004
|
async def readinto(self, buffer, /) -> int: # type: ignore
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
if _initing := self.__dict__.get("_initing"):
|
|
676
|
-
await _initing
|
|
677
|
-
self.__dict__.pop("_initing", None)
|
|
678
|
-
if not buffer or not self.chunked and self.tell() >= self.length:
|
|
1005
|
+
pos = self._pos
|
|
1006
|
+
if not buffer_length(buffer):
|
|
679
1007
|
return 0
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
1008
|
+
length = self.__dict__.get("length")
|
|
1009
|
+
if length is not None:
|
|
1010
|
+
if length == 0 or length > 0 and pos >= length:
|
|
1011
|
+
return 0
|
|
1012
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
1013
|
+
await self.reconnect(pos)
|
|
1014
|
+
if file := self.file:
|
|
1015
|
+
try:
|
|
1016
|
+
readinto = file.areadinto
|
|
1017
|
+
except AttributeError:
|
|
1018
|
+
try:
|
|
1019
|
+
readinto = ensure_async(file.readinto, threaded=True)
|
|
1020
|
+
except AttributeError:
|
|
1021
|
+
readinto = self._readinto
|
|
1022
|
+
try:
|
|
1023
|
+
size = await readinto(buffer)
|
|
1024
|
+
if size:
|
|
1025
|
+
self._pos = pos + size
|
|
690
1026
|
return size
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
1027
|
+
except:
|
|
1028
|
+
self._pos = pos
|
|
1029
|
+
await self.aclose()
|
|
1030
|
+
raise
|
|
1031
|
+
return 0
|
|
695
1032
|
|
|
696
1033
|
async def readline(self, size: None | int = -1, /) -> bytes: # type: ignore
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
if _initing := self.__dict__.get("_initing"):
|
|
700
|
-
await _initing
|
|
701
|
-
self.__dict__.pop("_initing", None)
|
|
702
|
-
if size == 0 or not self.chunked and self.tell() >= self.length:
|
|
1034
|
+
pos = self._pos
|
|
1035
|
+
if size == 0:
|
|
703
1036
|
return b""
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
1037
|
+
length = self.__dict__.get("length")
|
|
1038
|
+
if length is not None:
|
|
1039
|
+
if length == 0 or length > 0 and pos >= length:
|
|
1040
|
+
return b""
|
|
1041
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
1042
|
+
await self.reconnect(pos)
|
|
1043
|
+
if file := self.file:
|
|
1044
|
+
try:
|
|
1045
|
+
readline = file.areadline
|
|
1046
|
+
except AttributeError:
|
|
1047
|
+
try:
|
|
1048
|
+
readline = ensure_async(file.readline, threaded=True)
|
|
1049
|
+
except AttributeError:
|
|
1050
|
+
readline = self._readline
|
|
1051
|
+
try:
|
|
1052
|
+
if size is None or size < 0:
|
|
1053
|
+
data = await readline()
|
|
1054
|
+
else:
|
|
1055
|
+
data = await readline(size)
|
|
1056
|
+
if data:
|
|
1057
|
+
self._pos = pos + len(data)
|
|
1058
|
+
return data
|
|
1059
|
+
except:
|
|
1060
|
+
self._pos = pos
|
|
1061
|
+
await self.aclose()
|
|
1062
|
+
raise
|
|
1063
|
+
return b""
|
|
714
1064
|
|
|
715
1065
|
async def readlines(self, hint: int = -1, /) -> list[bytes]: # type: ignore
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
if
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
if not self.
|
|
722
|
-
|
|
723
|
-
if self.
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
1066
|
+
pos = self._pos
|
|
1067
|
+
length = self.__dict__.get("length")
|
|
1068
|
+
if length is not None:
|
|
1069
|
+
if length == 0 or length > 0 and pos >= length:
|
|
1070
|
+
return []
|
|
1071
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
1072
|
+
await self.reconnect(pos)
|
|
1073
|
+
if file := self.file:
|
|
1074
|
+
try:
|
|
1075
|
+
readlines = file.areadlines
|
|
1076
|
+
except AttributeError:
|
|
1077
|
+
try:
|
|
1078
|
+
readlines = ensure_async(file.readlines, threaded=True)
|
|
1079
|
+
except AttributeError:
|
|
1080
|
+
readlines = self._readlines
|
|
1081
|
+
try:
|
|
1082
|
+
ls = await readlines(hint)
|
|
1083
|
+
if ls:
|
|
1084
|
+
self._pos = pos + sum(map(len, ls))
|
|
1085
|
+
return ls
|
|
1086
|
+
except:
|
|
1087
|
+
self._pos = pos
|
|
1088
|
+
await self.aclose()
|
|
1089
|
+
raise
|
|
1090
|
+
return []
|
|
1091
|
+
|
|
1092
|
+
async def reconnect(self, /, start: None | int = None) -> int:
|
|
739
1093
|
if start is None:
|
|
740
|
-
start = self.
|
|
1094
|
+
start = self._pos
|
|
1095
|
+
if start and not self.seekable():
|
|
1096
|
+
raise OSError(errno.EOPNOTSUPP, "unsupport for reconnection of non-seekable streams")
|
|
1097
|
+
request_kwargs = self.request_kwargs
|
|
1098
|
+
headers = request_kwargs["headers"] = request_kwargs["headers"]
|
|
1099
|
+
if start >= 0:
|
|
1100
|
+
headers["range"] = f"bytes={start}-"
|
|
741
1101
|
elif start < 0:
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
if
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
start=
|
|
760
|
-
|
|
761
|
-
)
|
|
1102
|
+
headers["range"] = f"bytes={start}"
|
|
1103
|
+
await self.aclose()
|
|
1104
|
+
response = await self.request(await self._geturl(), **request_kwargs)
|
|
1105
|
+
status_code = get_status_code(response)
|
|
1106
|
+
if not 200 <= status_code < 300:
|
|
1107
|
+
raise OSError(
|
|
1108
|
+
errno.EIO,
|
|
1109
|
+
{
|
|
1110
|
+
"code": status_code,
|
|
1111
|
+
"response": response,
|
|
1112
|
+
"reason": "status code must be in the `range(200, 300)`",
|
|
1113
|
+
},
|
|
1114
|
+
)
|
|
1115
|
+
if start:
|
|
1116
|
+
rng = get_range(response)
|
|
1117
|
+
if not rng:
|
|
1118
|
+
raise OSError(errno.ESPIPE, "non-seekable")
|
|
1119
|
+
start = self._pos = rng[0]
|
|
1120
|
+
self.response = response
|
|
1121
|
+
self.file = await self._get_file()
|
|
1122
|
+
self._closed = False
|
|
762
1123
|
return start
|
|
763
1124
|
|
|
764
1125
|
async def seek(self, pos: int, whence: int = 0, /) -> int: # type: ignore
|
|
765
|
-
if self.
|
|
766
|
-
raise ValueError("I/O operation on closed file.")
|
|
767
|
-
if _initing := self.__dict__.get("_initing"):
|
|
768
|
-
await _initing
|
|
769
|
-
self.__dict__.pop("_initing", None)
|
|
770
|
-
if not self._seekable:
|
|
1126
|
+
if not self.seekable():
|
|
771
1127
|
raise OSError(errno.EINVAL, "not a seekable stream")
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
return await self.seek(self.length + pos)
|
|
797
|
-
else:
|
|
798
|
-
raise OSError(errno.EINVAL, f"whence value unsupported: {whence!r}")
|
|
1128
|
+
old_pos = self._pos
|
|
1129
|
+
match whence:
|
|
1130
|
+
case 1:
|
|
1131
|
+
pos = old_pos + pos
|
|
1132
|
+
case 2:
|
|
1133
|
+
length = self.__dict__.get("length")
|
|
1134
|
+
if length is None:
|
|
1135
|
+
length = await self._init_info()
|
|
1136
|
+
pos += length
|
|
1137
|
+
if pos < 0:
|
|
1138
|
+
pos = 0
|
|
1139
|
+
if (self.__dict__.get("response") and
|
|
1140
|
+
self.file and
|
|
1141
|
+
not self.closed and
|
|
1142
|
+
not self.response_closed and
|
|
1143
|
+
pos > old_pos and
|
|
1144
|
+
(size := pos - old_pos) <= self.seek_threshold
|
|
1145
|
+
):
|
|
1146
|
+
read = self._get_read()
|
|
1147
|
+
while size > COPY_BUFSIZE:
|
|
1148
|
+
await read(COPY_BUFSIZE)
|
|
1149
|
+
size -= COPY_BUFSIZE
|
|
1150
|
+
await read(size)
|
|
1151
|
+
return await self.reconnect(pos)
|
|
799
1152
|
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
headers: None | Mapping = None,
|
|
806
|
-
start: int = 0,
|
|
807
|
-
seek_threshold: int = 1 << 20,
|
|
808
|
-
urlopen = None,
|
|
809
|
-
) -> Self: # type: ignore
|
|
810
|
-
self = cls.__new__(cls)
|
|
811
|
-
await self.__ainit__(
|
|
812
|
-
url=url,
|
|
813
|
-
headers=headers,
|
|
814
|
-
start=start,
|
|
815
|
-
seek_threshold=seek_threshold,
|
|
816
|
-
urlopen=urlopen,
|
|
817
|
-
)
|
|
818
|
-
return self
|
|
1153
|
+
def seekable(self, /) -> bool:
|
|
1154
|
+
return self._seekable
|
|
1155
|
+
|
|
1156
|
+
def tell(self, /) -> int:
|
|
1157
|
+
return self._pos
|
|
819
1158
|
|
|
820
|
-
|
|
1159
|
+
async def truncate(self, size: None | int = None, /) -> int: # type: ignore
|
|
1160
|
+
raise UnsupportedOperation(errno.ENOTSUP, "truncate")
|
|
1161
|
+
|
|
1162
|
+
async def writable(self, /) -> bool: # type: ignore
|
|
1163
|
+
return False
|
|
1164
|
+
|
|
1165
|
+
async def write(self, b, /) -> int: # type: ignore
|
|
1166
|
+
raise UnsupportedOperation(errno.ENOTSUP, "write")
|
|
1167
|
+
|
|
1168
|
+
async def writelines(self, lines, /): # type: ignore
|
|
1169
|
+
raise UnsupportedOperation(errno.ENOTSUP, "writelines")
|
|
1170
|
+
|
|
1171
|
+
@overload
|
|
821
1172
|
@classmethod
|
|
822
1173
|
async def open(
|
|
823
1174
|
cls,
|
|
824
1175
|
/,
|
|
825
|
-
url:
|
|
1176
|
+
url: (
|
|
1177
|
+
str | SupportsGeturl | URL |
|
|
1178
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1179
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
|
|
1180
|
+
),
|
|
826
1181
|
mode: Literal["br", "rb"],
|
|
827
1182
|
*,
|
|
828
1183
|
buffering: None | int = None,
|
|
829
1184
|
encoding: None | str = None,
|
|
830
1185
|
errors: None | str = None,
|
|
831
1186
|
newline: None | str = None,
|
|
832
|
-
headers: None | Mapping = None,
|
|
833
1187
|
start: int = 0,
|
|
834
1188
|
seek_threshold: int = 1 << 20,
|
|
835
|
-
|
|
1189
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
1190
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1191
|
+
**request_kwargs,
|
|
836
1192
|
) -> Self | AsyncBufferedReader:
|
|
837
1193
|
...
|
|
838
1194
|
@overload
|
|
@@ -840,45 +1196,57 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
840
1196
|
async def open(
|
|
841
1197
|
cls,
|
|
842
1198
|
/,
|
|
843
|
-
url:
|
|
1199
|
+
url: (
|
|
1200
|
+
str | SupportsGeturl | URL |
|
|
1201
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1202
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
|
|
1203
|
+
),
|
|
844
1204
|
mode: Literal["r", "rt", "tr"] = "r",
|
|
845
1205
|
*,
|
|
846
1206
|
buffering: None | int = None,
|
|
847
1207
|
encoding: None | str = None,
|
|
848
1208
|
errors: None | str = None,
|
|
849
1209
|
newline: None | str = None,
|
|
850
|
-
headers: None | Mapping = None,
|
|
851
1210
|
start: int = 0,
|
|
852
1211
|
seek_threshold: int = 1 << 20,
|
|
853
|
-
|
|
1212
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
1213
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1214
|
+
**request_kwargs,
|
|
854
1215
|
) -> AsyncTextIOWrapper:
|
|
855
1216
|
...
|
|
856
1217
|
@classmethod
|
|
857
1218
|
async def open(
|
|
858
1219
|
cls,
|
|
859
1220
|
/,
|
|
860
|
-
url:
|
|
1221
|
+
url: (
|
|
1222
|
+
str | SupportsGeturl | URL |
|
|
1223
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1224
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
|
|
1225
|
+
),
|
|
861
1226
|
mode: Literal["r", "rt", "tr", "br", "rb"] = "r",
|
|
1227
|
+
*,
|
|
862
1228
|
buffering: None | int = None,
|
|
863
1229
|
encoding: None | str = None,
|
|
864
1230
|
errors: None | str = None,
|
|
865
1231
|
newline: None | str = None,
|
|
866
|
-
headers: None | Mapping = None,
|
|
867
1232
|
start: int = 0,
|
|
868
1233
|
seek_threshold: int = 1 << 20,
|
|
869
|
-
|
|
1234
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
1235
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1236
|
+
**request_kwargs,
|
|
870
1237
|
) -> Self | AsyncBufferedReader | AsyncTextIOWrapper:
|
|
871
|
-
file =
|
|
872
|
-
url
|
|
873
|
-
headers=headers,
|
|
1238
|
+
file = cls(
|
|
1239
|
+
url,
|
|
874
1240
|
start=start,
|
|
875
1241
|
seek_threshold=seek_threshold,
|
|
876
|
-
|
|
1242
|
+
request=request,
|
|
1243
|
+
get_file=get_file,
|
|
1244
|
+
**request_kwargs,
|
|
877
1245
|
)
|
|
878
1246
|
if mode not in ("r", "rt", "tr", "rb", "br"):
|
|
879
1247
|
raise OSError(errno.EINVAL, f"invalid (or unsupported) mode: {mode!r}")
|
|
880
1248
|
return file.wrap(
|
|
881
|
-
text_mode="b" not in mode, # type: ignore
|
|
1249
|
+
text_mode="b" not in cast(str, mode), # type: ignore
|
|
882
1250
|
buffering=buffering,
|
|
883
1251
|
encoding=encoding,
|
|
884
1252
|
errors=errors,
|
|
@@ -886,7 +1254,7 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
886
1254
|
)
|
|
887
1255
|
|
|
888
1256
|
@overload
|
|
889
|
-
def wrap(
|
|
1257
|
+
async def wrap(
|
|
890
1258
|
self,
|
|
891
1259
|
/,
|
|
892
1260
|
text_mode: Literal[False] = False,
|
|
@@ -898,7 +1266,7 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
898
1266
|
) -> Self | AsyncBufferedReader:
|
|
899
1267
|
...
|
|
900
1268
|
@overload
|
|
901
|
-
def wrap(
|
|
1269
|
+
async def wrap(
|
|
902
1270
|
self,
|
|
903
1271
|
/,
|
|
904
1272
|
text_mode: Literal[True],
|
|
@@ -909,7 +1277,7 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
909
1277
|
newline: None | str = None,
|
|
910
1278
|
) -> AsyncTextIOWrapper:
|
|
911
1279
|
...
|
|
912
|
-
def wrap(
|
|
1280
|
+
async def wrap(
|
|
913
1281
|
self,
|
|
914
1282
|
/,
|
|
915
1283
|
text_mode: bool = False,
|
|
@@ -940,6 +1308,8 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
940
1308
|
line_buffering = True
|
|
941
1309
|
else:
|
|
942
1310
|
buffer_size = buffering
|
|
1311
|
+
if "response" not in self.__dict__ or self.closed or self.response_closed:
|
|
1312
|
+
await self.reconnect()
|
|
943
1313
|
raw = self
|
|
944
1314
|
buffer = AsyncBufferedReader(raw, buffer_size)
|
|
945
1315
|
if text_mode:
|
|
@@ -954,278 +1324,623 @@ class AsyncHTTPFileReader(HTTPFileReader):
|
|
|
954
1324
|
return buffer
|
|
955
1325
|
|
|
956
1326
|
|
|
957
|
-
|
|
958
|
-
from urllib.error import HTTPError
|
|
1327
|
+
class MultipartHTTPFileReader[Response](HTTPFileReader[Response]):
|
|
959
1328
|
|
|
960
|
-
|
|
1329
|
+
def __init__(
|
|
1330
|
+
self,
|
|
1331
|
+
/,
|
|
1332
|
+
urls: (
|
|
1333
|
+
str | SupportsGeturl | URL |
|
|
1334
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1335
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1336
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1337
|
+
Iterable[
|
|
1338
|
+
str | SupportsGeturl | URL |
|
|
1339
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1340
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1341
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]]
|
|
1342
|
+
]
|
|
1343
|
+
),
|
|
1344
|
+
start: int = 0,
|
|
1345
|
+
seek_threshold: int = 1 << 20,
|
|
1346
|
+
request: None | Callable[..., Response] = None,
|
|
1347
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1348
|
+
**request_kwargs,
|
|
1349
|
+
):
|
|
1350
|
+
self.seek_threshold = seek_threshold
|
|
1351
|
+
if request is None:
|
|
1352
|
+
from urllib3_future_request import request_sync as request # type: ignore
|
|
1353
|
+
self.request = cast(Callable[..., Response], request)
|
|
1354
|
+
self.get_file = get_file # type: ignore
|
|
1355
|
+
self.request_kwargs = request_kwargs
|
|
1356
|
+
if (isinstance(urls, (str, SupportsGeturl, URL)) or
|
|
1357
|
+
callable(urls) or
|
|
1358
|
+
isinstance(urls, tuple) and urls and isinstance(urls[0], int)
|
|
1359
|
+
):
|
|
1360
|
+
urls = [urls]
|
|
1361
|
+
else:
|
|
1362
|
+
urls = list(urls) # type: ignore
|
|
961
1363
|
|
|
962
|
-
|
|
1364
|
+
from urllib3_future_request import request_sync
|
|
963
1365
|
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
from urllib3 import request
|
|
975
|
-
from urllib3.poolmanager import PoolManager
|
|
976
|
-
|
|
977
|
-
if isinstance(urlopen, PoolManager):
|
|
978
|
-
urlopen = partial(urlopen.request, "GET", preload_content=False)
|
|
979
|
-
elif urlopen is None:
|
|
980
|
-
if _urllib3_urlopen is None:
|
|
981
|
-
urlopen = _urllib3_urlopen = partial(cast(Callable, request), "GET", preload_content=False)
|
|
982
|
-
else:
|
|
983
|
-
urlopen = _urllib3_urlopen
|
|
1366
|
+
self._parts: list[dict] = []
|
|
1367
|
+
add_part = self._parts.append
|
|
1368
|
+
seekable = True
|
|
1369
|
+
length = 0
|
|
1370
|
+
request_kwargs = dict(request_kwargs, parse=...)
|
|
1371
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
1372
|
+
headers["range"] = "bytes=0-0"
|
|
1373
|
+
for url in urls:
|
|
1374
|
+
if isinstance(url, tuple):
|
|
1375
|
+
size, url = url
|
|
984
1376
|
else:
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1377
|
+
url_ = url
|
|
1378
|
+
if callable(url_):
|
|
1379
|
+
url_ = url_()
|
|
1380
|
+
if isinstance(url_, SupportsGeturl):
|
|
1381
|
+
url_ = url_.geturl()
|
|
1382
|
+
else:
|
|
1383
|
+
url_ = str(url_)
|
|
1384
|
+
resp = request_sync(url_, **request_kwargs)
|
|
1385
|
+
if not is_range_request(resp):
|
|
1386
|
+
seekable = False
|
|
1387
|
+
size = get_total_length(resp) or 0
|
|
1388
|
+
if size:
|
|
1389
|
+
start = length
|
|
1390
|
+
length += size
|
|
1391
|
+
stop = length
|
|
1392
|
+
add_part({
|
|
1393
|
+
"url": url,
|
|
1394
|
+
"length": size,
|
|
1395
|
+
"start": start,
|
|
1396
|
+
"stop": stop,
|
|
1397
|
+
})
|
|
1398
|
+
self.length = length
|
|
1399
|
+
if start < 0:
|
|
1400
|
+
start += length
|
|
1401
|
+
if start < 0:
|
|
1402
|
+
start = 0
|
|
1403
|
+
self._pos = start
|
|
1404
|
+
self._seekable = seekable
|
|
1405
|
+
self._closed = False
|
|
1000
1406
|
|
|
1001
|
-
|
|
1407
|
+
def __repr__(self):
|
|
1408
|
+
cls = type(self)
|
|
1409
|
+
kwargs = {
|
|
1410
|
+
"urls": [(d["length"], d["url"]) for d in self._parts],
|
|
1411
|
+
"start": self._pos,
|
|
1412
|
+
"seek_threshold": self.seek_threshold,
|
|
1413
|
+
"request": self.request,
|
|
1414
|
+
"get_file": self.get_file,
|
|
1415
|
+
**self.request_kwargs,
|
|
1416
|
+
}
|
|
1417
|
+
return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
|
|
1002
1418
|
|
|
1419
|
+
@cached_property
|
|
1420
|
+
def response(self, /):
|
|
1421
|
+
return self.file.response
|
|
1003
1422
|
|
|
1004
|
-
|
|
1005
|
-
|
|
1423
|
+
def close(self, /):
|
|
1424
|
+
self.file.close()
|
|
1425
|
+
self._closed = True
|
|
1006
1426
|
|
|
1007
|
-
|
|
1427
|
+
def read(self, size: None | int = -1, /) -> bytes:
|
|
1428
|
+
if size is None or size < 0:
|
|
1429
|
+
size = self.length - self._pos
|
|
1430
|
+
else:
|
|
1431
|
+
size = min(self.length - self._pos, size)
|
|
1432
|
+
if size <= 0:
|
|
1433
|
+
return b""
|
|
1434
|
+
buf = bytearray(size)
|
|
1435
|
+
self.readinto(buf)
|
|
1436
|
+
return bytes(buf)
|
|
1437
|
+
|
|
1438
|
+
def readinto(self, buffer: Buffer, /) -> int:
|
|
1439
|
+
pos = self._pos
|
|
1440
|
+
remaining = min(self.length - pos, buffer_length(buffer))
|
|
1441
|
+
if remaining <= 0:
|
|
1442
|
+
return 0
|
|
1443
|
+
if "file" not in self.__dict__ or self.closed or self.response_closed:
|
|
1444
|
+
self.reconnect(pos)
|
|
1445
|
+
try:
|
|
1446
|
+
readinto = self.file.readinto
|
|
1447
|
+
view = memoryview(buffer).cast("B")[:remaining]
|
|
1448
|
+
size = 0
|
|
1449
|
+
while size < remaining:
|
|
1450
|
+
start_pos = pos + size
|
|
1451
|
+
self.seek(start_pos)
|
|
1452
|
+
size += readinto(view[size:])
|
|
1453
|
+
return size
|
|
1454
|
+
except:
|
|
1455
|
+
self._pos = pos
|
|
1456
|
+
self.close()
|
|
1457
|
+
raise
|
|
1008
1458
|
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
if
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1459
|
+
def readline(self, size: None | int = -1, /) -> bytes:
|
|
1460
|
+
pos = self._pos
|
|
1461
|
+
if size is None or size < 0:
|
|
1462
|
+
size = self.length - pos
|
|
1463
|
+
else:
|
|
1464
|
+
size = min(self.length - pos, size)
|
|
1465
|
+
if size <= 0:
|
|
1466
|
+
return b""
|
|
1467
|
+
if "file" not in self.__dict__ or self.closed or self.response_closed:
|
|
1468
|
+
self.reconnect(pos)
|
|
1469
|
+
try:
|
|
1470
|
+
readline = self.file.readline
|
|
1471
|
+
buf = bytearray()
|
|
1472
|
+
while size > 0:
|
|
1473
|
+
if line := readline(size):
|
|
1474
|
+
buf += line
|
|
1475
|
+
if buf.endswith(b"\n"):
|
|
1476
|
+
break
|
|
1027
1477
|
else:
|
|
1028
|
-
|
|
1478
|
+
self.seek(pos + len(buf))
|
|
1479
|
+
size -= len(line)
|
|
1480
|
+
return bytes(buf)
|
|
1481
|
+
except:
|
|
1482
|
+
self._pos = pos
|
|
1483
|
+
self.close()
|
|
1484
|
+
raise
|
|
1485
|
+
|
|
1486
|
+
def readlines(self, hint: int = -1, /) -> list[bytes]:
|
|
1487
|
+
pos = self._pos
|
|
1488
|
+
readline = self.readline
|
|
1489
|
+
ls: list[bytes]
|
|
1490
|
+
add = ls.append
|
|
1491
|
+
try:
|
|
1492
|
+
if hint <= 0:
|
|
1493
|
+
while l := readline():
|
|
1494
|
+
add(l)
|
|
1029
1495
|
else:
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1496
|
+
while hint >=0. and (l := readline()):
|
|
1497
|
+
add(l)
|
|
1498
|
+
hint -= len(l)
|
|
1499
|
+
return ls
|
|
1500
|
+
except:
|
|
1501
|
+
self._pos = pos
|
|
1502
|
+
self.close()
|
|
1503
|
+
raise
|
|
1504
|
+
|
|
1505
|
+
def read_range(self, size: None | int = -1, /, offset: int = 0) -> bytes:
|
|
1506
|
+
if offset < 0:
|
|
1507
|
+
offset += self.length
|
|
1508
|
+
if offset < 0:
|
|
1509
|
+
offset = 0
|
|
1510
|
+
if size is None or size < 0:
|
|
1511
|
+
size = self.length - offset
|
|
1512
|
+
else:
|
|
1513
|
+
size = min(self.length - offset, size)
|
|
1514
|
+
if size <= 0:
|
|
1515
|
+
return b""
|
|
1516
|
+
buf = bytearray(size)
|
|
1517
|
+
self.readinto_range(buf, offset)
|
|
1518
|
+
return bytes(buf)
|
|
1519
|
+
|
|
1520
|
+
def readinto_range(self, buffer: Buffer, /, offset: int = 0):
|
|
1521
|
+
from urllib3_future_request import request_sync as request
|
|
1522
|
+
|
|
1523
|
+
if offset < 0:
|
|
1524
|
+
offset += self.length
|
|
1525
|
+
if offset < 0:
|
|
1526
|
+
offset = 0
|
|
1527
|
+
remaining = min(self.length - offset, buffer_length(buffer))
|
|
1528
|
+
if remaining <= 0:
|
|
1529
|
+
return 0
|
|
1530
|
+
request_kwargs = dict(self.request_kwargs)
|
|
1531
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
1532
|
+
headers["accept-encoding"] = "identity"
|
|
1533
|
+
view = memoryview(buffer).cast("B")[:remaining]
|
|
1534
|
+
size = 0
|
|
1535
|
+
while size < remaining:
|
|
1536
|
+
for vol in self._parts:
|
|
1537
|
+
if vol["start"] <= offset < vol["stop"]:
|
|
1538
|
+
vol_offset = offset + size - vol["start"]
|
|
1539
|
+
headers["range"] = f"bytes={vol_offset}-"
|
|
1540
|
+
resp = request(vol["url"], **request_kwargs)
|
|
1541
|
+
try:
|
|
1542
|
+
if vol_offset and resp.headers.get("accept-ranges") != "bytes":
|
|
1543
|
+
raise OSError(29, "non-seekable stream")
|
|
1544
|
+
size += resp.readinto(view[size:])
|
|
1545
|
+
finally:
|
|
1546
|
+
resp.close()
|
|
1547
|
+
return size
|
|
1042
1548
|
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1549
|
+
def reconnect(self, /, start: None | int = None) -> int:
|
|
1550
|
+
if start is None:
|
|
1551
|
+
start = self._pos
|
|
1552
|
+
elif start < 0:
|
|
1553
|
+
start += self.length
|
|
1554
|
+
if start < 0:
|
|
1555
|
+
start = 0
|
|
1556
|
+
if start >= self.length:
|
|
1557
|
+
return start
|
|
1558
|
+
for vol in self._parts:
|
|
1559
|
+
if vol["start"] <= start < vol["stop"]:
|
|
1560
|
+
self.close()
|
|
1561
|
+
file = self.file = HTTPFileReader(
|
|
1562
|
+
vol["url"],
|
|
1563
|
+
start=vol["start"] - start,
|
|
1564
|
+
seek_threshold=self.seek_threshold,
|
|
1565
|
+
request=self.request,
|
|
1566
|
+
get_file=self.get_file,
|
|
1567
|
+
**self.request_kwargs,
|
|
1568
|
+
)
|
|
1569
|
+
self.response = file.response
|
|
1570
|
+
self._pos = start
|
|
1571
|
+
self._closed = False
|
|
1572
|
+
return start
|
|
1573
|
+
return start
|
|
1046
1574
|
|
|
1047
|
-
|
|
1575
|
+
def seek(self, pos: int, whence: int = 0, /) -> int:
|
|
1576
|
+
old_pos = self._pos
|
|
1577
|
+
match whence:
|
|
1578
|
+
case 1:
|
|
1579
|
+
pos = old_pos + pos
|
|
1580
|
+
case 2:
|
|
1581
|
+
pos += self.length
|
|
1582
|
+
if pos < 0:
|
|
1583
|
+
pos = 0
|
|
1584
|
+
if pos == old_pos:
|
|
1585
|
+
return pos
|
|
1586
|
+
elif pos >= self.length:
|
|
1587
|
+
self.close()
|
|
1588
|
+
self._pos = pos
|
|
1589
|
+
return pos
|
|
1590
|
+
if hasattr(self, "file"):
|
|
1591
|
+
file = self.file
|
|
1592
|
+
start = old_pos - file._pos
|
|
1593
|
+
stop = start + file.length
|
|
1594
|
+
if start <= pos < stop:
|
|
1595
|
+
file.seek(pos-start)
|
|
1596
|
+
self._pos = pos
|
|
1597
|
+
return pos
|
|
1598
|
+
return self.reconnect(pos)
|
|
1048
1599
|
|
|
1049
1600
|
|
|
1050
|
-
|
|
1051
|
-
_aiohttp_urlopen = None
|
|
1601
|
+
class AsyncMultipartHTTPFileReader[Response](AsyncHTTPFileReader[Response]):
|
|
1052
1602
|
|
|
1053
|
-
|
|
1603
|
+
def __init__(
|
|
1604
|
+
self,
|
|
1605
|
+
/,
|
|
1606
|
+
urls: (
|
|
1607
|
+
str | SupportsGeturl | URL |
|
|
1608
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1609
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
|
|
1610
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1611
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1612
|
+
tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]] |
|
|
1613
|
+
Iterable[
|
|
1614
|
+
str | SupportsGeturl | URL |
|
|
1615
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1616
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
|
|
1617
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1618
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1619
|
+
tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]]
|
|
1620
|
+
]
|
|
1621
|
+
),
|
|
1622
|
+
start: int = 0,
|
|
1623
|
+
seek_threshold: int = 1 << 20,
|
|
1624
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
1625
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1626
|
+
**request_kwargs,
|
|
1627
|
+
):
|
|
1628
|
+
run_async(self.__ainit__(
|
|
1629
|
+
urls,
|
|
1630
|
+
start=start,
|
|
1631
|
+
seek_threshold=seek_threshold,
|
|
1632
|
+
request=request,
|
|
1633
|
+
get_file=get_file,
|
|
1634
|
+
**request_kwargs,
|
|
1635
|
+
))
|
|
1054
1636
|
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1637
|
+
async def __ainit__(
|
|
1638
|
+
self,
|
|
1639
|
+
/,
|
|
1640
|
+
urls: (
|
|
1641
|
+
str | SupportsGeturl | URL |
|
|
1642
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1643
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
|
|
1644
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1645
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1646
|
+
tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]] |
|
|
1647
|
+
Iterable[
|
|
1648
|
+
str | SupportsGeturl | URL |
|
|
1649
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1650
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
|
|
1651
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1652
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1653
|
+
tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]]
|
|
1654
|
+
]
|
|
1655
|
+
),
|
|
1656
|
+
start: int = 0,
|
|
1657
|
+
seek_threshold: int = 1 << 20,
|
|
1658
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
1659
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1660
|
+
**request_kwargs,
|
|
1661
|
+
):
|
|
1662
|
+
self.seek_threshold = seek_threshold
|
|
1663
|
+
if request is None:
|
|
1664
|
+
from urllib3_future_request import request_async as request # type: ignore
|
|
1665
|
+
self.request = cast(Callable[..., Awaitable[Response]], request)
|
|
1666
|
+
self.get_file = get_file # type: ignore
|
|
1667
|
+
self.request_kwargs = request_kwargs
|
|
1668
|
+
if (isinstance(urls, (str, SupportsGeturl, URL)) or
|
|
1669
|
+
callable(urls) or
|
|
1670
|
+
isinstance(urls, tuple) and urls and isinstance(urls[0], int)
|
|
1063
1671
|
):
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
if isinstance(urlopen, ClientSession):
|
|
1068
|
-
urlopen = urlopen.get
|
|
1069
|
-
elif urlopen is None:
|
|
1070
|
-
if _aiohttp_urlopen is None:
|
|
1071
|
-
urlopen = _aiohttp_urlopen = partial(request, "GET")
|
|
1072
|
-
else:
|
|
1073
|
-
urlopen = _aiohttp_urlopen
|
|
1074
|
-
async def urlopen_wrapper(url: str, headers: None | Mapping = headers):
|
|
1075
|
-
ctx = urlopen(url, headers=headers)
|
|
1076
|
-
resp = await ctx.__aenter__()
|
|
1077
|
-
resp.aclose = lambda: ctx.__aexit__(None, None, None)
|
|
1078
|
-
resp.raise_for_status()
|
|
1079
|
-
return resp
|
|
1080
|
-
await super().__ainit__(
|
|
1081
|
-
url=url,
|
|
1082
|
-
headers=headers,
|
|
1083
|
-
start=start,
|
|
1084
|
-
seek_threshold=seek_threshold,
|
|
1085
|
-
urlopen=urlopen_wrapper,
|
|
1086
|
-
)
|
|
1672
|
+
urls = [urls]
|
|
1673
|
+
else:
|
|
1674
|
+
urls = list(urls) # type: ignore
|
|
1087
1675
|
|
|
1088
|
-
|
|
1089
|
-
def file(self, /) -> BinaryIO:
|
|
1090
|
-
return self.response.content
|
|
1676
|
+
from urllib3_future_request import request_async
|
|
1091
1677
|
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1678
|
+
self._parts: list[dict] = []
|
|
1679
|
+
add_part = self._parts.append
|
|
1680
|
+
seekable = True
|
|
1681
|
+
length = 0
|
|
1682
|
+
request_kwargs = dict(request_kwargs, parse=...)
|
|
1683
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
1684
|
+
headers["range"] = "bytes=0-0"
|
|
1685
|
+
for url in urls:
|
|
1686
|
+
if isinstance(url, tuple):
|
|
1687
|
+
size, url = url
|
|
1688
|
+
else:
|
|
1689
|
+
resp = await request_async(await ageturl(url), **request_kwargs)
|
|
1690
|
+
if not is_range_request(resp):
|
|
1691
|
+
seekable = False
|
|
1692
|
+
size = get_total_length(resp) or 0
|
|
1693
|
+
if size:
|
|
1694
|
+
start = length
|
|
1695
|
+
length += size
|
|
1696
|
+
stop = length
|
|
1697
|
+
add_part({
|
|
1698
|
+
"url": url,
|
|
1699
|
+
"length": size,
|
|
1700
|
+
"start": start,
|
|
1701
|
+
"stop": stop,
|
|
1702
|
+
})
|
|
1703
|
+
self.length = length
|
|
1704
|
+
if start < 0:
|
|
1705
|
+
start += length
|
|
1706
|
+
if start < 0:
|
|
1707
|
+
start = 0
|
|
1708
|
+
self._pos = start
|
|
1709
|
+
self._seekable = seekable
|
|
1710
|
+
self._closed = False
|
|
1095
1711
|
|
|
1096
|
-
|
|
1712
|
+
@classmethod
|
|
1713
|
+
async def new( # type: ignore
|
|
1714
|
+
cls,
|
|
1715
|
+
/,
|
|
1716
|
+
urls: (
|
|
1717
|
+
str | SupportsGeturl | URL |
|
|
1718
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1719
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
|
|
1720
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1721
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1722
|
+
tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]] |
|
|
1723
|
+
Iterable[
|
|
1724
|
+
str | SupportsGeturl | URL |
|
|
1725
|
+
Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
|
|
1726
|
+
Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
|
|
1727
|
+
tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
|
|
1728
|
+
tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
|
|
1729
|
+
tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]]
|
|
1730
|
+
]
|
|
1731
|
+
),
|
|
1732
|
+
start: int = 0,
|
|
1733
|
+
seek_threshold: int = 1 << 20,
|
|
1734
|
+
request: None | Callable[..., Awaitable[Response]] = None,
|
|
1735
|
+
get_file: None | str | Callable[[Response], Any] = None,
|
|
1736
|
+
**request_kwargs,
|
|
1737
|
+
) -> Self:
|
|
1738
|
+
self = cls.__new__(cls)
|
|
1739
|
+
await self.__ainit__(
|
|
1740
|
+
urls,
|
|
1741
|
+
start=start,
|
|
1742
|
+
seek_threshold=seek_threshold,
|
|
1743
|
+
request=request,
|
|
1744
|
+
get_file=get_file,
|
|
1745
|
+
**request_kwargs,
|
|
1746
|
+
)
|
|
1747
|
+
return self
|
|
1097
1748
|
|
|
1749
|
+
def __repr__(self):
|
|
1750
|
+
cls = type(self)
|
|
1751
|
+
kwargs = {
|
|
1752
|
+
"urls": [(d["length"], d["url"]) for d in self._parts],
|
|
1753
|
+
"start": self._pos,
|
|
1754
|
+
"seek_threshold": self.seek_threshold,
|
|
1755
|
+
"request": self.request,
|
|
1756
|
+
"get_file": self.get_file,
|
|
1757
|
+
**self.request_kwargs,
|
|
1758
|
+
}
|
|
1759
|
+
return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
|
|
1098
1760
|
|
|
1099
|
-
|
|
1100
|
-
|
|
1761
|
+
@cached_property
|
|
1762
|
+
def response(self, /):
|
|
1763
|
+
return self.file.response
|
|
1101
1764
|
|
|
1102
|
-
def
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
auth = None,
|
|
1106
|
-
follow_redirects: bool = True,
|
|
1107
|
-
stream: bool = True,
|
|
1108
|
-
session = None,
|
|
1109
|
-
**request_kwargs,
|
|
1110
|
-
):
|
|
1111
|
-
if session is None:
|
|
1112
|
-
from httpx import Client
|
|
1113
|
-
if "__del__" not in Client.__dict__:
|
|
1114
|
-
setattr(Client, "__del__", lambda self: self.close())
|
|
1115
|
-
session = Client()
|
|
1116
|
-
resp = session.send(
|
|
1117
|
-
request=session.build_request(
|
|
1118
|
-
method=method,
|
|
1119
|
-
url=url,
|
|
1120
|
-
**request_kwargs,
|
|
1121
|
-
),
|
|
1122
|
-
auth=auth,
|
|
1123
|
-
follow_redirects=follow_redirects,
|
|
1124
|
-
stream=stream,
|
|
1125
|
-
)
|
|
1126
|
-
setattr(resp, "session", session)
|
|
1127
|
-
return resp
|
|
1128
|
-
|
|
1129
|
-
async def httpx_request_async(
|
|
1130
|
-
url,
|
|
1131
|
-
method: str = "GET",
|
|
1132
|
-
auth = None,
|
|
1133
|
-
follow_redirects: bool = True,
|
|
1134
|
-
stream: bool = True,
|
|
1135
|
-
session = None,
|
|
1136
|
-
**request_kwargs,
|
|
1137
|
-
):
|
|
1138
|
-
if session is None:
|
|
1139
|
-
from httpx import AsyncClient
|
|
1140
|
-
if "__del__" not in AsyncClient.__dict__:
|
|
1141
|
-
setattr(AsyncClient, "__del__", lambda self: run_async(self.aclose()))
|
|
1142
|
-
session = AsyncClient()
|
|
1143
|
-
resp = await session.send(
|
|
1144
|
-
request=session.build_request(
|
|
1145
|
-
method=method,
|
|
1146
|
-
url=url,
|
|
1147
|
-
**request_kwargs,
|
|
1148
|
-
),
|
|
1149
|
-
auth=auth,
|
|
1150
|
-
follow_redirects=follow_redirects,
|
|
1151
|
-
stream=stream,
|
|
1152
|
-
)
|
|
1153
|
-
setattr(resp, "session", session)
|
|
1154
|
-
return resp
|
|
1155
|
-
|
|
1156
|
-
class HttpxFileReader(HTTPFileReader):
|
|
1157
|
-
|
|
1158
|
-
def __init__(
|
|
1159
|
-
self,
|
|
1160
|
-
/,
|
|
1161
|
-
url: str | Callable[[], str],
|
|
1162
|
-
headers: None | Mapping = None,
|
|
1163
|
-
start: int = 0,
|
|
1164
|
-
seek_threshold: int = 1 << 20,
|
|
1165
|
-
urlopen = None,
|
|
1166
|
-
):
|
|
1167
|
-
from httpx import Client
|
|
1168
|
-
|
|
1169
|
-
if urlopen is None or isinstance(urlopen, Client):
|
|
1170
|
-
urlopen = partial(httpx_request, session=urlopen)
|
|
1171
|
-
|
|
1172
|
-
def urlopen_wrapper(url: str, headers: None | Mapping = headers):
|
|
1173
|
-
resp = urlopen(url, headers=headers)
|
|
1174
|
-
resp.raise_for_status()
|
|
1175
|
-
file = bytes_iter_to_reader(resp.iter_raw())
|
|
1176
|
-
self.__dict__["file"] = file
|
|
1177
|
-
return resp
|
|
1178
|
-
|
|
1179
|
-
super().__init__(
|
|
1180
|
-
url,
|
|
1181
|
-
headers=headers,
|
|
1182
|
-
start=start,
|
|
1183
|
-
seek_threshold=seek_threshold,
|
|
1184
|
-
urlopen=urlopen_wrapper,
|
|
1185
|
-
)
|
|
1765
|
+
async def aclose(self, /):
|
|
1766
|
+
await self.file.aclose()
|
|
1767
|
+
self._closed = True
|
|
1186
1768
|
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1769
|
+
def close(self, /):
|
|
1770
|
+
self.file.close()
|
|
1771
|
+
self._closed = True
|
|
1190
1772
|
|
|
1191
|
-
|
|
1773
|
+
async def read(self, size: None | int = -1, /) -> bytes: # type: ignore
|
|
1774
|
+
if size is None or size < 0:
|
|
1775
|
+
size = self.length - self._pos
|
|
1776
|
+
else:
|
|
1777
|
+
size = min(self.length - self._pos, size)
|
|
1778
|
+
if size <= 0:
|
|
1779
|
+
return b""
|
|
1780
|
+
buf = bytearray(size)
|
|
1781
|
+
await self.readinto(buf)
|
|
1782
|
+
return bytes(buf)
|
|
1783
|
+
|
|
1784
|
+
async def readinto(self, buffer: Buffer, /) -> int: # type: ignore
|
|
1785
|
+
pos = self._pos
|
|
1786
|
+
remaining = min(self.length - pos, buffer_length(buffer))
|
|
1787
|
+
if remaining <= 0:
|
|
1788
|
+
return 0
|
|
1789
|
+
if "file" not in self.__dict__ or self.closed or self.response_closed:
|
|
1790
|
+
await self.reconnect(pos)
|
|
1791
|
+
try:
|
|
1792
|
+
readinto = self.file.readinto
|
|
1793
|
+
view = memoryview(buffer).cast("B")[:remaining]
|
|
1794
|
+
size = 0
|
|
1795
|
+
while size < remaining:
|
|
1796
|
+
start_pos = pos + size
|
|
1797
|
+
await self.seek(start_pos)
|
|
1798
|
+
size += await readinto(view[size:])
|
|
1799
|
+
return size
|
|
1800
|
+
except:
|
|
1801
|
+
self._pos = pos
|
|
1802
|
+
await self.aclose()
|
|
1803
|
+
raise
|
|
1192
1804
|
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
urlopen=urlopen_wrapper,
|
|
1220
|
-
)
|
|
1805
|
+
async def readline(self, size: None | int = -1, /) -> bytes: # type: ignore
|
|
1806
|
+
pos = self._pos
|
|
1807
|
+
if size is None or size < 0:
|
|
1808
|
+
size = self.length - pos
|
|
1809
|
+
else:
|
|
1810
|
+
size = min(self.length - pos, size)
|
|
1811
|
+
if size <= 0:
|
|
1812
|
+
return b""
|
|
1813
|
+
if "file" not in self.__dict__ or self.closed or self.response_closed:
|
|
1814
|
+
await self.reconnect(pos)
|
|
1815
|
+
try:
|
|
1816
|
+
readline = self.file.readline
|
|
1817
|
+
buf = bytearray()
|
|
1818
|
+
while size > 0:
|
|
1819
|
+
if line := await readline(size):
|
|
1820
|
+
buf += line
|
|
1821
|
+
if buf.endswith(b"\n"):
|
|
1822
|
+
break
|
|
1823
|
+
else:
|
|
1824
|
+
await self.seek(pos + len(buf))
|
|
1825
|
+
size -= len(line)
|
|
1826
|
+
return bytes(buf)
|
|
1827
|
+
except:
|
|
1828
|
+
self._pos = pos
|
|
1829
|
+
await self.aclose()
|
|
1830
|
+
raise
|
|
1221
1831
|
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1832
|
+
async def readlines(self, hint: int = -1, /) -> list[bytes]: # type: ignore
|
|
1833
|
+
pos = self._pos
|
|
1834
|
+
readline = self.readline
|
|
1835
|
+
ls: list[bytes]
|
|
1836
|
+
add = ls.append
|
|
1837
|
+
try:
|
|
1838
|
+
if hint <= 0:
|
|
1839
|
+
while l := await readline():
|
|
1840
|
+
add(l)
|
|
1841
|
+
else:
|
|
1842
|
+
while hint >=0. and (l := await readline()):
|
|
1843
|
+
add(l)
|
|
1844
|
+
hint -= len(l)
|
|
1845
|
+
return ls
|
|
1846
|
+
except:
|
|
1847
|
+
self._pos = pos
|
|
1848
|
+
await self.aclose()
|
|
1849
|
+
raise
|
|
1850
|
+
|
|
1851
|
+
async def read_range(self, size: None | int = -1, /, offset: int = 0) -> bytes:
|
|
1852
|
+
if offset < 0:
|
|
1853
|
+
offset += self.length
|
|
1854
|
+
if offset < 0:
|
|
1855
|
+
offset = 0
|
|
1856
|
+
if size is None or size < 0:
|
|
1857
|
+
size = self.length - offset
|
|
1858
|
+
else:
|
|
1859
|
+
size = min(self.length - offset, size)
|
|
1860
|
+
if size <= 0:
|
|
1861
|
+
return b""
|
|
1862
|
+
buf = bytearray(size)
|
|
1863
|
+
await self.readinto_range(buf, offset)
|
|
1864
|
+
return bytes(buf)
|
|
1865
|
+
|
|
1866
|
+
async def readinto_range(self, buffer: Buffer, /, offset: int = 0):
|
|
1867
|
+
from urllib3_future_request import request_async
|
|
1868
|
+
|
|
1869
|
+
if offset < 0:
|
|
1870
|
+
offset += self.length
|
|
1871
|
+
if offset < 0:
|
|
1872
|
+
offset = 0
|
|
1873
|
+
remaining = min(self.length - offset, buffer_length(buffer))
|
|
1874
|
+
if remaining <= 0:
|
|
1875
|
+
return 0
|
|
1876
|
+
request_kwargs = dict(self.request_kwargs)
|
|
1877
|
+
headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
|
|
1878
|
+
headers["accept-encoding"] = "identity"
|
|
1879
|
+
view = memoryview(buffer).cast("B")[:remaining]
|
|
1880
|
+
size = 0
|
|
1881
|
+
while size < remaining:
|
|
1882
|
+
for vol in self._parts:
|
|
1883
|
+
if vol["start"] <= offset < vol["stop"]:
|
|
1884
|
+
vol_offset = offset + size - vol["start"]
|
|
1885
|
+
headers["range"] = f"bytes={vol_offset}-"
|
|
1886
|
+
resp = await request_async(vol["url"], **request_kwargs)
|
|
1887
|
+
try:
|
|
1888
|
+
if vol_offset and resp.headers.get("accept-ranges") != "bytes":
|
|
1889
|
+
raise OSError(29, "non-seekable stream")
|
|
1890
|
+
size += await resp.readinto(view[size:])
|
|
1891
|
+
finally:
|
|
1892
|
+
await resp.close()
|
|
1893
|
+
return size
|
|
1225
1894
|
|
|
1226
|
-
|
|
1227
|
-
|
|
1895
|
+
async def reconnect(self, /, start: None | int = None) -> int:
|
|
1896
|
+
if start is None:
|
|
1897
|
+
start = self._pos
|
|
1898
|
+
elif start < 0:
|
|
1899
|
+
start += self.length
|
|
1900
|
+
if start < 0:
|
|
1901
|
+
start = 0
|
|
1902
|
+
if start >= self.length:
|
|
1903
|
+
return start
|
|
1904
|
+
for vol in self._parts:
|
|
1905
|
+
if vol["start"] <= start < vol["stop"]:
|
|
1906
|
+
await self.aclose()
|
|
1907
|
+
file = self.file = await AsyncHTTPFileReader.new(
|
|
1908
|
+
vol["url"],
|
|
1909
|
+
start=vol["start"] - start,
|
|
1910
|
+
seek_threshold=self.seek_threshold,
|
|
1911
|
+
request=self.request,
|
|
1912
|
+
get_file=self.get_file,
|
|
1913
|
+
**self.request_kwargs,
|
|
1914
|
+
)
|
|
1915
|
+
self.response = file.response
|
|
1916
|
+
self._pos = start
|
|
1917
|
+
self._closed = False
|
|
1918
|
+
return start
|
|
1919
|
+
return start
|
|
1920
|
+
|
|
1921
|
+
async def seek(self, pos: int, whence: int = 0, /) -> int: # type: ignore
|
|
1922
|
+
old_pos = self._pos
|
|
1923
|
+
match whence:
|
|
1924
|
+
case 1:
|
|
1925
|
+
pos = old_pos + pos
|
|
1926
|
+
case 2:
|
|
1927
|
+
pos += self.length
|
|
1928
|
+
if pos < 0:
|
|
1929
|
+
pos = 0
|
|
1930
|
+
if pos == old_pos:
|
|
1931
|
+
return pos
|
|
1932
|
+
elif pos >= self.length:
|
|
1933
|
+
await self.aclose()
|
|
1934
|
+
self._pos = pos
|
|
1935
|
+
return pos
|
|
1936
|
+
if hasattr(self, "file"):
|
|
1937
|
+
file = self.file
|
|
1938
|
+
start = old_pos - file._pos
|
|
1939
|
+
stop = start + file.length
|
|
1940
|
+
if start <= pos < stop:
|
|
1941
|
+
await file.seek(pos-start)
|
|
1942
|
+
self._pos = pos
|
|
1943
|
+
return pos
|
|
1944
|
+
return await self.reconnect(pos)
|
|
1228
1945
|
|
|
1229
|
-
# TODO: 实现任意请求模块的 HTTPFileReader,只要它支持流式读取即可(分成 2 种类型:要么支持 read,要么支持迭代器)
|
|
1230
1946
|
# TODO: 设计实现一个 HTTPFileWriter,用于实现上传,关闭后视为上传完成
|
|
1231
|
-
# TODO: httpx 重新实现 stream 和 async_stream 方法,确保不需要用上下文管理器,如此才能真的简化
|