python-httpfile 0.0.5.4__py3-none-any.whl → 0.0.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
httpfile/__init__.py CHANGED
@@ -4,156 +4,113 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  __author__ = "ChenyangGao <https://chenyanggao.github.io>"
7
- __version__ = (0, 0, 5)
8
- __all__ = ["HTTPFileReader", "AsyncHTTPFileReader"]
7
+ __version__ = (0, 0, 6)
8
+ __all__ = ["HTTPFileReader", "AsyncHTTPFileReader", "MultipartHTTPFileReader", "AsyncMultipartHTTPFileReader"]
9
9
 
10
- import errno
11
-
12
- from collections.abc import Awaitable, Callable, Mapping
13
- from functools import cached_property, partial
14
- from importlib.util import find_spec
10
+ from collections.abc import AsyncIterator, Awaitable, Buffer, Callable, Iterable, Iterator
11
+ from functools import cached_property
15
12
  from inspect import isawaitable
16
13
  from io import (
17
- BufferedReader, RawIOBase, TextIOWrapper, UnsupportedOperation, DEFAULT_BUFFER_SIZE,
14
+ BufferedReader, BytesIO, RawIOBase, TextIOWrapper, UnsupportedOperation,
15
+ DEFAULT_BUFFER_SIZE,
18
16
  )
19
- from os import fstat, stat, PathLike
20
- from shutil import COPY_BUFSIZE # type: ignore
17
+ try:
18
+ from shutil import COPY_BUFSIZE # type: ignore
19
+ except ImportError:
20
+ COPY_BUFSIZE = 1 << 16
21
21
  from typing import cast, overload, Any, BinaryIO, Literal, Self
22
- from types import MappingProxyType
23
22
  from warnings import warn
24
23
 
24
+ from argtools import argcount
25
25
  from asynctools import ensure_async, run_async
26
- from filewrap import AsyncBufferedReader, AsyncTextIOWrapper
27
- from http_response import get_filename, get_length, get_range, get_total_length, is_chunked, is_range_request
28
- from property import funcproperty
29
-
30
-
31
- def get_filesize(
32
- file,
33
- /,
34
- dont_read: bool = True,
35
- ) -> int:
36
- if isinstance(file, (bytes, str, PathLike)):
37
- return stat(file).st_size
38
- curpos = 0
39
- try:
40
- curpos = file.seek(0, 1)
41
- seekable = True
42
- except Exception:
43
- seekable = False
44
- if not seekable:
45
- try:
46
- curpos = file.tell()
47
- except Exception:
48
- pass
49
- try:
50
- return len(file) - curpos
51
- except TypeError:
52
- pass
53
- if hasattr(file, "fileno"):
54
- try:
55
- return fstat(file.fileno()).st_size - curpos
56
- except Exception:
57
- pass
58
- if hasattr(file, "headers"):
59
- l = get_length(file)
60
- if l is not None:
61
- return l - curpos
62
- if seekable:
63
- try:
64
- return file.seek(0, 2) - curpos
65
- finally:
66
- file.seek(curpos)
67
- if dont_read:
68
- return -1
69
- total = 0
70
- if hasattr(file, "readinto"):
71
- readinto = file.readinto
72
- buf = bytearray(COPY_BUFSIZE)
73
- while (size := readinto(buf)):
74
- total += size
75
- elif hasattr(file, "read"):
76
- read = file.read
77
- while (chunk := read(COPY_BUFSIZE)):
78
- total += len(chunk)
79
- else:
80
- return -1
81
- return total
82
-
83
-
84
- class HTTPFileReader(RawIOBase, BinaryIO):
85
- url: str | Callable[[], str]
86
- response: Any
87
- length: int
88
- chunked: bool
89
- start: int
90
- urlopen: Callable
91
- headers: Mapping
92
- seek_threshold: int
93
- _seekable: bool
26
+ from errno2 import errno
27
+ from filewrap import (
28
+ buffer_length, bytes_iter_to_reader, bytes_iter_to_async_reader,
29
+ to_bytes_view, AsyncBufferedReader, AsyncTextIOWrapper,
30
+ )
31
+ from http_request import SupportsGeturl
32
+ from http_response import (
33
+ get_filename, get_range, get_status_code, get_total_length, is_range_request,
34
+ )
35
+ from yarl import URL
36
+
37
+
38
+ def coalesce(*vals, base=None):
39
+ for v in vals:
40
+ if v is not base:
41
+ return v
42
+ return base
43
+
44
+
45
+ def geturl(url, /) -> str:
46
+ if callable(url):
47
+ url = url()
48
+ if isinstance(url, SupportsGeturl):
49
+ return url.geturl()
50
+ return str(url)
51
+
52
+
53
+ async def ageturl(url, /) -> str:
54
+ if callable(url):
55
+ url = url()
56
+ if isawaitable(url):
57
+ url = await url
58
+ if isinstance(url, SupportsGeturl):
59
+ url = url.geturl()
60
+ if isawaitable(url):
61
+ url = await url
62
+ return url
63
+ return str(url)
64
+
65
+
66
+ class HTTPFileReader[Response](RawIOBase, BinaryIO):
94
67
 
95
68
  def __init__(
96
69
  self,
97
70
  /,
98
- url: str | Callable[[], str],
99
- headers: None | Mapping = None,
71
+ url: (
72
+ str | SupportsGeturl | URL |
73
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
74
+ ),
100
75
  start: int = 0,
101
- # NOTE: If the offset of the forward seek is not higher than this value,
102
- # it will be directly read and discarded, default to 1 MB
103
76
  seek_threshold: int = 1 << 20,
104
- urlopen = None,
77
+ request: None | Callable[..., Response] = None,
78
+ get_file: None | str | Callable[[Response], Any] = None,
79
+ **request_kwargs,
105
80
  ):
106
- if headers:
107
- headers = {**headers, "accept-encoding": "identity"}
108
- else:
109
- headers = {"accept-encoding": "identity"}
110
- if start > 0:
111
- headers["range"] = f"bytes={start}-"
112
- elif start < 0:
113
- headers["range"] = f"bytes={start}"
114
- if urlopen is None:
115
- from http_request.extension.request import urlopen
116
- if callable(url):
117
- geturl = url
118
- def url():
119
- url = geturl()
120
- headers_extra = getattr(url, "headers")
121
- if headers_extra:
122
- headers.update(headers_extra)
123
- return url
124
- elif hasattr(url, "headers"):
125
- headers_extra = getattr(url, "headers")
126
- if headers_extra:
127
- headers.update(headers_extra)
128
- response = urlopen(url() if callable(url) else url, headers=headers)
129
- if start:
130
- rng = get_range(response)
131
- if not rng:
132
- raise OSError(errno.ESPIPE, "non-seekable")
133
- start = rng[0]
134
- self.__dict__.update(
135
- url = url,
136
- response = response,
137
- length = get_total_length(response) or 0,
138
- chunked = is_chunked(response),
139
- start = start,
140
- closed = False,
141
- urlopen = urlopen,
142
- headers = MappingProxyType(headers),
143
- seek_threshold = max(seek_threshold, 0),
144
- _seekable = is_range_request(response),
145
- )
81
+ if start < 0:
82
+ raise ValueError("`start` cannot be < 0")
83
+ self._closed = False
84
+ self._pos = start
85
+ self._url = url
86
+ self.seek_threshold = max(seek_threshold, 0)
87
+ if request is None:
88
+ from urllib3_future_request import request_sync as request # type: ignore
89
+ self.request = cast(Callable[..., Response], request)
90
+ if isinstance(get_file, str):
91
+ if get_file:
92
+ is_method = get_file.endswith("()")
93
+ attrs = get_file.removesuffix("()").split(".")
94
+ def get_file(file, /):
95
+ for attr in attrs:
96
+ file = getattr(file, attr)
97
+ if is_method:
98
+ file = file()
99
+ return file
100
+ else:
101
+ get_file = lambda file, /: file
102
+ self.get_file = get_file
103
+ self.request_kwargs = request_kwargs
104
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
105
+ headers["accept-encoding"] = "identity"
146
106
 
147
107
  def __del__(self, /):
148
- try:
149
- self.close()
150
- except:
151
- pass
108
+ self.close()
152
109
 
153
- def __enter__(self, /):
110
+ def __enter__(self, /) -> Self:
154
111
  return self
155
112
 
156
- def __exit__(self, /, *exc_info):
113
+ def __exit__(self, /, *_) -> None:
157
114
  self.close()
158
115
 
159
116
  def __iter__(self, /) -> Self:
@@ -163,215 +120,378 @@ class HTTPFileReader(RawIOBase, BinaryIO):
163
120
  return self.length
164
121
 
165
122
  def __next__(self, /) -> bytes:
166
- line = self.readline()
167
- if line:
123
+ if line := self.readline():
168
124
  return line
169
- else:
170
- raise StopIteration
125
+ raise StopIteration
171
126
 
172
127
  def __repr__(self, /) -> str:
173
128
  cls = type(self)
174
- module = cls.__module__
175
- name = cls.__qualname__
176
- if module != "__main__":
177
- name = module + "." + name
178
- return f"{name}(url={self.url!r}, headers={self.headers!r}, start={self.tell()}, seek_threshold={self.seek_threshold}, urlopen={self.urlopen!r})"
129
+ kwargs = {
130
+ "url": self._url,
131
+ "start": self._pos,
132
+ "seek_threshold": self.seek_threshold,
133
+ "request": self.request,
134
+ "get_file": self.get_file,
135
+ **self.request_kwargs,
136
+ }
137
+ return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
179
138
 
180
- def __setattr__(self, attr, val, /):
181
- raise TypeError("can't set attribute")
139
+ @property
140
+ def closed(self, /) -> bool:
141
+ return self._closed
182
142
 
183
- def _add_start(self, delta: int, /):
184
- if not self.file_can_tell:
185
- self.__dict__["start"] += delta
143
+ @property
144
+ def response_closed(self, /) -> bool:
145
+ file = self.__dict__.get("response")
146
+ while file:
147
+ if hasattr(file, "closed"):
148
+ return file.closed
149
+ elif hasattr(file, "is_closed"):
150
+ closed = file.is_closed
151
+ if callable(closed):
152
+ closed = closed()
153
+ return closed
154
+ file = getattr(file, "raw", None)
155
+ return self.closed
156
+
157
+ def _init_info(self, /, key: str = "length"):
158
+ response = self.__dict__.get("response")
159
+ if response is None:
160
+ from urllib3_future_request import request_sync as request
161
+ request_kwargs = dict(self.request_kwargs, parse=...)
162
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
163
+ headers["range"] = "bytes=0-0"
164
+ response = request(self._geturl(), **request_kwargs)
165
+ self._seekable = not is_range_request(response)
166
+ self.length = coalesce(get_total_length(response), -1)
167
+ match key:
168
+ case "length":
169
+ return self.length
170
+ case "seekable":
171
+ return self._seekable
186
172
 
187
173
  @cached_property
188
- def file_can_tell(self, /) -> bool:
189
- try:
190
- self.file.tell()
191
- return True
192
- except (AttributeError, TypeError, OSError):
193
- return False
174
+ def _seekable(self, /) -> bool:
175
+ return self._init_info("seekable")
194
176
 
195
- @funcproperty
196
- def file_closed(self, /) -> bool:
197
- return self.file.closed
177
+ @cached_property
178
+ def length(self, /) -> int:
179
+ return self._init_info("length")
180
+
181
+ @cached_property
182
+ def response(self, /):
183
+ self.reconnect()
184
+ return self.__dict__.get("response")
198
185
 
199
186
  @property
200
- def closed(self, /) -> bool:
201
- return self.__dict__["closed"]
187
+ def mode(self, /) -> str:
188
+ return "rb"
189
+
190
+ @property
191
+ def name(self, /) -> str:
192
+ return get_filename(self.response)
193
+
194
+ def _get_file(self, /):
195
+ response = self.__dict__.get("response")
196
+ if not response:
197
+ self.reconnect()
198
+ response = self.__dict__.get("response")
199
+ if not response:
200
+ return None
201
+ if get_file := self.get_file or (getattr(response, "get_file", None)):
202
+ file = get_file(response)
203
+ if hasattr(file, "read"):
204
+ return file
205
+ elif isinstance(file, Iterator):
206
+ return bytes_iter_to_reader(file)
207
+ else:
208
+ if file := getattr(response, "file", None):
209
+ return file
210
+ if hasattr(response, "read"):
211
+ read = response.read
212
+ if argcount(read):
213
+ return response
214
+ for attr in (
215
+ "iter_bytes", "iter_chunks", "iter_chunked", "iter_stream",
216
+ "iter_body", "iter_raw", "iter_content",
217
+ ):
218
+ method = getattr(response, attr, None)
219
+ if method is not None:
220
+ file = method()
221
+ if isinstance(file, Iterator):
222
+ return bytes_iter_to_reader(file)
223
+ for attr in ("stream", "body", "raw", "content"):
224
+ file = getattr(response, attr, None)
225
+ if file is not None:
226
+ if hasattr(file, "read") and argcount(getattr(file, "read")):
227
+ return file
228
+ if callable(file):
229
+ file = file()
230
+ if isinstance(file, Buffer):
231
+ return BytesIO(file)
232
+ elif isinstance(file, Iterator):
233
+ return bytes_iter_to_reader(file)
234
+ raise TypeError("can't determine how to `read`")
235
+
236
+ def _geturl(self, /) -> str:
237
+ return geturl(self._url)
202
238
 
203
239
  def close(self, /):
204
240
  if not self.closed:
205
- self.response.close()
206
- self.__dict__["closed"] = True
207
-
208
- @funcproperty
209
- def file(self, /) -> BinaryIO:
210
- return self.response
241
+ if response := self.__dict__.get("response"):
242
+ try:
243
+ response.close()
244
+ except (AttributeError, TypeError):
245
+ pass
246
+ self._closed = True
211
247
 
212
248
  def fileno(self, /) -> int:
213
- return self.file.fileno()
249
+ file = self.__dict__.get("response")
250
+ while file:
251
+ if hasattr(file, "fileno"):
252
+ return file.fileno()
253
+ file = getattr(file, "raw", None)
254
+ return 0
214
255
 
215
256
  def flush(self, /):
216
- if self.closed:
217
- raise ValueError("I/O operation on closed file.")
218
- return self.file.flush()
257
+ if not self.closed:
258
+ file = self.__dict__.get("response")
259
+ while file:
260
+ if hasattr(file, "flush"):
261
+ return file.flush()
262
+ file = getattr(file, "raw", None)
219
263
 
220
264
  def isatty(self, /) -> bool:
221
265
  return False
222
266
 
223
- @cached_property
224
- def mode(self, /) -> str:
225
- return "rb"
226
-
227
- @cached_property
228
- def name(self, /) -> str:
229
- return get_filename(self.response)
267
+ def _readinto(self, buffer: Buffer, /) -> int:
268
+ read = self.file.read
269
+ m = to_bytes_view(buffer)
270
+ size = len(m)
271
+ start = 0
272
+ last_intv = size - size % COPY_BUFSIZE
273
+ while start < size:
274
+ if start >= last_intv:
275
+ delta = size - start
276
+ else:
277
+ delta = COPY_BUFSIZE
278
+ data = read(delta)
279
+ data_len = len(data)
280
+ m[:data_len] = data
281
+ start += data_len
282
+ if data_len < delta:
283
+ break
284
+ return start
230
285
 
231
- def read(self, size: int = -1, /) -> bytes:
232
- if self.closed:
233
- raise ValueError("I/O operation on closed file.")
234
- if size == 0 or not self.chunked and self.tell() >= self.length:
235
- return b""
236
- if self.file_closed:
237
- self.reconnect()
286
+ def _readline(self, size: None | int = -1, /) -> bytes:
287
+ read = self.file.read
288
+ buf = bytearray()
238
289
  if size is None or size < 0:
239
- data = self.file.read()
290
+ while b := read(1):
291
+ buf += b
292
+ if b == b"\n":
293
+ break
294
+ else:
295
+ while size and (b := read(1)):
296
+ buf += b
297
+ if b == b"\n":
298
+ break
299
+ size -= 1
300
+ return bytes(buf)
301
+
302
+ def _readlines(self, hint: int = -1, /) -> list[bytes]:
303
+ try:
304
+ readline = self.file.readline
305
+ except AttributeError:
306
+ readline = self._readline
307
+ ls: list[bytes]
308
+ add = ls.append
309
+ if hint <= 0:
310
+ while l := readline():
311
+ add(l)
240
312
  else:
241
- data = self.file.read(size)
242
- if data:
243
- self._add_start(len(data))
244
- return data
313
+ while hint >= 0 and (l := readline()):
314
+ add(l)
315
+ hint -= len(l)
316
+ return ls
245
317
 
246
318
  def readable(self, /) -> bool:
247
319
  return True
248
320
 
249
- def readinto(self, buffer, /) -> int:
250
- if self.closed:
251
- raise ValueError("I/O operation on closed file.")
252
- if not buffer or not self.chunked and self.tell() >= self.length:
253
- return 0
254
- if self.file_closed:
255
- self.reconnect()
256
- try:
257
- readinto = self.file.readinto
258
- except AttributeError:
259
- read = self.file.read
260
- def readinto(buffer, /):
261
- data = read(len(buffer))
262
- size = len(data)
263
- buffer[:size] = data
321
+ def read(self, size: int = -1, /) -> bytes:
322
+ pos = self._pos
323
+ if size == 0:
324
+ return b""
325
+ length = self.__dict__.get("length")
326
+ if length is not None:
327
+ if length == 0 or length > 0 and pos >= length:
328
+ return b""
329
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
330
+ self.reconnect(pos)
331
+ if file := self.file:
332
+ try:
333
+ if size is None or size < 0:
334
+ data = file.read()
335
+ else:
336
+ data = file.read(size)
337
+ if data:
338
+ self._pos = pos + len(data)
264
339
  return data
265
- size = readinto(buffer)
266
- if size:
267
- self._add_start(size)
268
- return size
340
+ except:
341
+ self._pos = pos
342
+ self.close()
343
+ raise
344
+ return b""
345
+
346
+ def readinto(self, buffer: Buffer, /) -> int:
347
+ pos = self._pos
348
+ if not buffer_length(buffer):
349
+ return 0
350
+ length = self.__dict__.get("length")
351
+ if length is not None:
352
+ if length == 0 or length > 0 and pos >= length:
353
+ return 0
354
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
355
+ self.reconnect(pos)
356
+ if file := self.file:
357
+ try:
358
+ readinto = file.readinto
359
+ except AttributeError:
360
+ readinto = self._readinto
361
+ try:
362
+ size = readinto(buffer)
363
+ if size:
364
+ self._pos = pos + size
365
+ return size
366
+ except:
367
+ self._pos = pos
368
+ self.close()
369
+ raise
370
+ return 0
269
371
 
270
372
  def readline(self, size: None | int = -1, /) -> bytes:
271
- if self.closed:
272
- raise ValueError("I/O operation on closed file.")
273
- if size == 0 or not self.chunked and self.tell() >= self.length:
373
+ pos = self._pos
374
+ if size == 0:
274
375
  return b""
275
- if self.file_closed:
276
- self.reconnect()
277
- # TODO: if no readline method, give a default impl
278
- if size is None or size < 0:
279
- data = self.file.readline()
280
- else:
281
- data = self.file.readline(size)
282
- if data:
283
- self._add_start(len(data))
284
- return data
376
+ length = self.__dict__.get("length")
377
+ if length is not None:
378
+ if length == 0 or length > 0 and pos >= length:
379
+ return b""
380
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
381
+ self.reconnect(pos)
382
+ if file := self.file:
383
+ try:
384
+ readline = file.readline
385
+ except AttributeError:
386
+ readline = self._readline
387
+ try:
388
+ if size is None or size < 0:
389
+ data = readline()
390
+ else:
391
+ data = readline(size)
392
+ if data:
393
+ self._pos = pos + len(data)
394
+ return data
395
+ except:
396
+ self._pos = pos
397
+ self.close()
398
+ raise
399
+ return b""
285
400
 
286
401
  def readlines(self, hint: int = -1, /) -> list[bytes]:
287
- if self.closed:
288
- raise ValueError("I/O operation on closed file.")
289
- if not self.chunked and self.tell() >= self.length:
290
- return []
291
- if self.file_closed:
292
- self.reconnect()
293
- # TODO: if no readlines method, give a default impl
294
- ls = self.file.readlines(hint)
295
- if ls:
296
- self._add_start(sum(map(len, ls)))
297
- return ls
402
+ pos = self._pos
403
+ length = self.__dict__.get("length")
404
+ if length is not None:
405
+ if length == 0 or length > 0 and pos >= length:
406
+ return []
407
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
408
+ self.reconnect(pos)
409
+ if file := self.file:
410
+ try:
411
+ readlines = file.readlines
412
+ except AttributeError:
413
+ readlines = self._readlines
414
+ try:
415
+ ls = readlines(hint)
416
+ if ls:
417
+ self._pos = pos + sum(map(len, ls))
418
+ return ls
419
+ except:
420
+ self._pos = pos
421
+ self.close()
422
+ raise
423
+ return []
298
424
 
299
425
  def reconnect(self, /, start: None | int = None) -> int:
300
- if not self._seekable:
301
- if start is None and self.tell() or start:
302
- raise OSError(errno.EOPNOTSUPP, "Unsupport for reconnection of non-seekable streams.")
303
- start = 0
304
426
  if start is None:
305
- start = self.tell()
427
+ start = self._pos
428
+ if start and not self.seekable():
429
+ raise OSError(errno.EOPNOTSUPP, "unsupport for reconnection of non-seekable streams")
430
+ request_kwargs = self.request_kwargs
431
+ headers = request_kwargs["headers"] = request_kwargs["headers"]
432
+ if start >= 0:
433
+ headers["range"] = f"bytes={start}-"
306
434
  elif start < 0:
307
- start = self.length + start
308
- if start < 0:
309
- start = 0
310
- if start >= self.length:
311
- self.__dict__.update(start=start)
312
- return start
313
- self.response.close()
314
- url = self.url
315
- response = self.urlopen(
316
- url() if callable(url) else url,
317
- headers={**self.headers, "range": f"bytes={start}-"}
318
- )
319
- length_new = get_total_length(response)
320
- if self.length != length_new:
321
- raise OSError(errno.EIO, f"file size changed: {self.length} -> {length_new}")
322
- self.__dict__.update(
323
- response=response,
324
- start=start,
325
- closed=False,
326
- )
435
+ headers["range"] = f"bytes={start}"
436
+ self.close()
437
+ response = self.request(self._geturl(), **request_kwargs)
438
+ status_code = get_status_code(response)
439
+ if not 200 <= status_code < 300:
440
+ raise OSError(
441
+ errno.EIO,
442
+ {
443
+ "code": status_code,
444
+ "response": response,
445
+ "reason": "status code must be in the `range(200, 300)`",
446
+ },
447
+ )
448
+ if start:
449
+ rng = get_range(response)
450
+ if not rng:
451
+ raise OSError(errno.ESPIPE, "non-seekable")
452
+ start = self._pos = rng[0]
453
+ self.response = response
454
+ self.file = self._get_file()
455
+ self._closed = False
327
456
  return start
328
457
 
329
458
  def seek(self, pos: int, whence: int = 0, /) -> int:
330
- if self.closed:
331
- raise ValueError("I/O operation on closed file.")
332
- if not self._seekable:
459
+ if not self.seekable():
333
460
  raise OSError(errno.EINVAL, "not a seekable stream")
334
- if whence == 0:
335
- if pos < 0:
336
- raise OSError(errno.EINVAL, f"negative seek start: {pos!r}")
337
- old_pos = self.tell()
338
- if old_pos == pos:
339
- return pos
340
- if pos > old_pos and (size := pos - old_pos) <= self.seek_threshold:
341
- if size <= COPY_BUFSIZE:
342
- self.read(size)
343
- else:
344
- buf = bytearray(COPY_BUFSIZE)
345
- readinto = self.readinto
346
- while size > COPY_BUFSIZE:
347
- readinto(buf)
348
- size -= COPY_BUFSIZE
349
- self.read(size)
350
- else:
351
- self.reconnect(pos)
461
+ old_pos = self._pos
462
+ match whence:
463
+ case 1:
464
+ pos = old_pos + pos
465
+ case 2:
466
+ length = self.__dict__.get("length")
467
+ if length is None:
468
+ length = self._init_info()
469
+ pos += length
470
+ if pos < 0:
471
+ pos = 0
472
+ if pos == old_pos:
352
473
  return pos
353
- elif whence == 1:
354
- if pos == 0:
355
- return self.tell()
356
- return self.seek(self.tell() + pos)
357
- elif whence == 2:
358
- return self.seek(self.length + pos)
359
- else:
360
- raise OSError(errno.EINVAL, f"whence value unsupported: {whence!r}")
474
+ if (self.__dict__.get("response") and
475
+ self.file and
476
+ not self.closed and
477
+ not self.response_closed and
478
+ pos > old_pos and
479
+ (size := pos - old_pos) <= self.seek_threshold
480
+ ):
481
+ read = self.read
482
+ while size > COPY_BUFSIZE:
483
+ read(COPY_BUFSIZE)
484
+ size -= COPY_BUFSIZE
485
+ read(size)
486
+ return self.reconnect(pos)
361
487
 
362
488
  def seekable(self, /) -> bool:
363
489
  return self._seekable
364
490
 
365
491
  def tell(self, /) -> int:
366
- if self.file_can_tell:
367
- start = self.start
368
- if start >= self.length:
369
- return start
370
- return start + self.file.tell()
371
- else:
372
- return self.start
492
+ return self._pos
373
493
 
374
- def truncate(self, size: None | int = None, /):
494
+ def truncate(self, size: None | int = None, /) -> int:
375
495
  raise UnsupportedOperation(errno.ENOTSUP, "truncate")
376
496
 
377
497
  def writable(self, /) -> bool:
@@ -388,17 +508,21 @@ class HTTPFileReader(RawIOBase, BinaryIO):
388
508
  def open(
389
509
  cls,
390
510
  /,
391
- url: str | Callable[[], str],
511
+ url: (
512
+ str | SupportsGeturl | URL |
513
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
514
+ ),
392
515
  mode: Literal["br", "rb"],
393
516
  *,
394
517
  buffering: None | int = None,
395
518
  encoding: None | str = None,
396
519
  errors: None | str = None,
397
520
  newline: None | str = None,
398
- headers: None | Mapping = None,
399
521
  start: int = 0,
400
522
  seek_threshold: int = 1 << 20,
401
- urlopen = None,
523
+ request: None | Callable[..., Response] = None,
524
+ get_file: None | str | Callable[[Response], Any] = None,
525
+ **request_kwargs,
402
526
  ) -> Self | BufferedReader:
403
527
  ...
404
528
  @overload
@@ -406,46 +530,55 @@ class HTTPFileReader(RawIOBase, BinaryIO):
406
530
  def open(
407
531
  cls,
408
532
  /,
409
- url: str | Callable[[], str],
533
+ url: (
534
+ str | SupportsGeturl | URL |
535
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
536
+ ),
410
537
  mode: Literal["r", "rt", "tr"] = "r",
411
538
  *,
412
539
  buffering: None | int = None,
413
540
  encoding: None | str = None,
414
541
  errors: None | str = None,
415
542
  newline: None | str = None,
416
- headers: None | Mapping = None,
417
543
  start: int = 0,
418
544
  seek_threshold: int = 1 << 20,
419
- urlopen = None,
545
+ request: None | Callable[..., Response] = None,
546
+ get_file: None | str | Callable[[Response], Any] = None,
547
+ **request_kwargs,
420
548
  ) -> TextIOWrapper:
421
549
  ...
422
550
  @classmethod
423
551
  def open(
424
552
  cls,
425
553
  /,
426
- url: str | Callable[[], str],
554
+ url: (
555
+ str | SupportsGeturl | URL |
556
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL]
557
+ ),
427
558
  mode: Literal["r", "rt", "tr", "br", "rb"] = "r",
428
559
  *,
429
560
  buffering: None | int = None,
430
561
  encoding: None | str = None,
431
562
  errors: None | str = None,
432
563
  newline: None | str = None,
433
- headers: None | Mapping = None,
434
564
  start: int = 0,
435
565
  seek_threshold: int = 1 << 20,
436
- urlopen = None,
566
+ request: None | Callable[..., Response] = None,
567
+ get_file: None | str | Callable[[Response], Any] = None,
568
+ **request_kwargs,
437
569
  ) -> Self | BufferedReader | TextIOWrapper:
438
570
  file = cls(
439
571
  url=url,
440
- headers=headers,
441
572
  start=start,
442
573
  seek_threshold=seek_threshold,
443
- urlopen=urlopen,
574
+ request=request,
575
+ get_file=get_file,
576
+ **request_kwargs,
444
577
  )
445
578
  if mode not in ("r", "rt", "tr", "rb", "br"):
446
579
  raise OSError(errno.EINVAL, f"invalid (or unsupported) mode: {mode!r}")
447
580
  return file.wrap(
448
- text_mode="b" not in mode, # type: ignore
581
+ text_mode="b" not in cast(str, mode), # type: ignore
449
582
  buffering=buffering,
450
583
  encoding=encoding,
451
584
  errors=errors,
@@ -507,6 +640,8 @@ class HTTPFileReader(RawIOBase, BinaryIO):
507
640
  line_buffering = True
508
641
  else:
509
642
  buffer_size = buffering
643
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
644
+ self.reconnect()
510
645
  raw = self
511
646
  buffer: BufferedReader = BufferedReader(raw, buffer_size)
512
647
  if text_mode:
@@ -521,34 +656,81 @@ class HTTPFileReader(RawIOBase, BinaryIO):
521
656
  return buffer
522
657
 
523
658
 
524
- class AsyncHTTPFileReader(HTTPFileReader):
525
- url: str | Callable[[], Awaitable[str]] # type: ignore
659
+ class AsyncHTTPFileReader[Response](RawIOBase, BinaryIO):
526
660
 
527
661
  def __init__(
528
662
  self,
529
663
  /,
530
- url: str | Callable[[], str] | Callable[[], Awaitable[str]],
531
- headers: None | Mapping = None,
664
+ url: (
665
+ str | SupportsGeturl | URL |
666
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
667
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
668
+ ),
532
669
  start: int = 0,
533
670
  seek_threshold: int = 1 << 20,
534
- urlopen = None,
671
+ request: None | Callable[..., Awaitable[Response]] = None,
672
+ get_file: None | str | Callable[[Response], Any] = None,
673
+ **request_kwargs,
535
674
  ):
536
- self.__dict__.update(closed=False, start=0, _seekable=True)
537
- self.__dict__["_initing"] = run_async(self.__ainit__(
538
- url=url,
539
- headers=headers,
675
+ if start < 0:
676
+ raise ValueError("`start` cannot be < 0")
677
+ self._closed = False
678
+ self._pos = start
679
+ self._url = url
680
+ self.seek_threshold = max(seek_threshold, 0)
681
+ if request is None:
682
+ from urllib3_future_request import request_async as request # type: ignore
683
+ self.request = cast(Callable[..., Awaitable[Response]], request)
684
+ if isinstance(get_file, str):
685
+ if get_file:
686
+ is_method = get_file.endswith("()")
687
+ attrs = get_file.removesuffix("()").split(".")
688
+ def get_file(file, /):
689
+ for attr in attrs:
690
+ file = getattr(file, attr)
691
+ if is_method:
692
+ file = file()
693
+ return file
694
+ else:
695
+ get_file = lambda file, /: file
696
+ self.get_file = get_file
697
+ self.request_kwargs = request_kwargs
698
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
699
+ headers["accept-encoding"] = "identity"
700
+
701
+ @classmethod
702
+ async def new(
703
+ cls,
704
+ /,
705
+ url: (
706
+ str | SupportsGeturl | URL |
707
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
708
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
709
+ ),
710
+ start: int = 0,
711
+ seek_threshold: int = 1 << 20,
712
+ request: None | Callable[..., Awaitable[Response]] = None,
713
+ get_file: None | str | Callable[[Response], Any] = None,
714
+ **request_kwargs,
715
+ ) -> Self:
716
+ self = cls(
717
+ url,
540
718
  start=start,
541
719
  seek_threshold=seek_threshold,
542
- urlopen=urlopen,
543
- ))
720
+ request=request,
721
+ get_file=get_file,
722
+ **request_kwargs,
723
+ )
724
+ await self.reconnect()
725
+ return self
726
+
727
+ def __del__(self, /):
728
+ self.close()
544
729
 
545
730
  async def __aenter__(self, /) -> Self:
546
- if _initing := self.__dict__.get("_initing"):
547
- await _initing
548
- self.__dict__.pop("_initing", None)
549
731
  return self
550
732
 
551
- async def __aexit__(self, /, *exc_info):
733
+ async def __aexit__(self, /, *_):
552
734
  await self.aclose()
553
735
 
554
736
  def __aiter__(self, /) -> Self:
@@ -557,282 +739,456 @@ class AsyncHTTPFileReader(HTTPFileReader):
557
739
  async def __anext__(self, /) -> bytes:
558
740
  if line := await self.readline():
559
741
  return line
560
- else:
561
- raise StopAsyncIteration
742
+ raise StopAsyncIteration
562
743
 
563
- async def __ainit__(
564
- self,
565
- /,
566
- url: str | Callable[[], str] | Callable[[], Awaitable[str]],
567
- headers: None | Mapping = None,
568
- start: int = 0,
569
- seek_threshold: int = 1 << 20,
570
- urlopen = None,
571
- ):
572
- if headers:
573
- headers = {**headers, "accept-encoding": "identity"}
574
- else:
575
- headers = {"accept-encoding": "identity"}
576
- if start > 0:
577
- headers["range"] = f"bytes={start}-"
578
- elif start < 0:
579
- headers["range"] = f"bytes={start}"
580
- if urlopen is None:
581
- from http_request.extension.request import urlopen
582
- if callable(url):
583
- geturl = url
584
- async def url() -> str:
585
- url = geturl()
586
- if isawaitable(url):
587
- url = await url
588
- headers_extra = getattr(url, "headers")
589
- if headers_extra:
590
- headers.update(headers_extra)
591
- return cast(str, url)
592
- elif hasattr(url, "headers"):
593
- headers_extra = getattr(url, "headers")
594
- if headers_extra:
595
- headers.update(headers_extra)
596
- urlopen = ensure_async(urlopen, threaded=True)
597
- response = await urlopen((await url()) if callable(url) else url, headers=headers)
598
- if start:
599
- rng = get_range(response)
600
- if not rng:
601
- raise OSError(errno.ESPIPE, "non-seekable")
602
- start = rng[0]
603
- self.__dict__.update(
604
- url = url,
605
- response = response,
606
- length = get_total_length(response) or 0,
607
- chunked = is_chunked(response),
608
- start = start,
609
- urlopen = urlopen,
610
- headers = MappingProxyType(headers),
611
- seek_threshold = max(seek_threshold, 0),
612
- _seekable = is_range_request(response),
613
- )
614
- self.__dict__.pop("_initing", None)
744
+ def __len__(self, /) -> int:
745
+ return self.length
615
746
 
616
- async def aclose(self, /):
617
- if not self.closed:
618
- self.__dict__["closed"] = True
619
- await self.close_response()
747
+ def __repr__(self, /) -> str:
748
+ cls = type(self)
749
+ kwargs = {
750
+ "url": self._url,
751
+ "start": self._pos,
752
+ "seek_threshold": self.seek_threshold,
753
+ "request": self.request,
754
+ "get_file": self.get_file,
755
+ **self.request_kwargs,
756
+ }
757
+ return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
620
758
 
621
- def close(self, /):
622
- if not self.closed:
623
- self.__dict__["closed"] = True
759
+ @property
760
+ def closed(self, /) -> bool:
761
+ return self._closed
762
+
763
+ @property
764
+ def response_closed(self, /) -> bool:
765
+ file = self.__dict__.get("response")
766
+ while file:
767
+ if hasattr(file, "closed"):
768
+ return file.closed
769
+ elif hasattr(file, "is_closed"):
770
+ closed = file.is_closed
771
+ if callable(closed):
772
+ closed = closed()
773
+ return closed
774
+ file = getattr(file, "raw", None)
775
+ return self.closed
776
+
777
+ async def _init_info(self, /, key: str = "length"):
778
+ response = self.__dict__.get("response")
779
+ if response is None:
780
+ from urllib3_future_request import request_async as request
781
+ request_kwargs = dict(self.request_kwargs, parse=...)
782
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
783
+ headers["range"] = "bytes=0-0"
784
+ response = await request(await self._geturl(), **request_kwargs)
785
+ self._seekable = not is_range_request(response)
786
+ self.length = coalesce(get_total_length(response), -1)
787
+ match key:
788
+ case "length":
789
+ return self.length
790
+ case "seekable":
791
+ return self._seekable
792
+
793
+ @cached_property
794
+ def _seekable(self, /) -> bool:
795
+ return run_async(self._init_info("seekable"))
796
+
797
+ @cached_property
798
+ def length(self, /) -> int:
799
+ return run_async(self._init_info("length"))
800
+
801
+ @cached_property
802
+ def response(self, /):
803
+ run_async(self.reconnect())
804
+ return self.__dict__.get("response")
805
+
806
+ @property
807
+ def mode(self, /) -> str:
808
+ return "rb"
809
+
810
+ @property
811
+ def name(self, /) -> str:
812
+ return get_filename(self.response)
813
+
814
+ async def _get_file(self, /):
815
+ response = self.__dict__.get("response")
816
+ if not response:
817
+ await self.reconnect()
818
+ response = self.__dict__.get("response")
819
+ if not response:
820
+ return None
821
+ if get_file := self.get_file or (getattr(response, "get_file", None)):
822
+ file = get_file(response)
823
+ if isawaitable(file):
824
+ file = await file
825
+ if hasattr(file, "aread") or hasattr(file, "read"):
826
+ return file
827
+ elif isinstance(file, (Iterator, AsyncIterator)):
828
+ return bytes_iter_to_async_reader(file, threaded=True)
829
+ else:
830
+ if file := getattr(response, "file", None):
831
+ return file
832
+ for attr in ("aread", "read"):
833
+ method = getattr(response, attr, None)
834
+ if method is not None and argcount(method):
835
+ return response
836
+ for attr in (
837
+ "aiter_bytes", "aiter_chunks", "aiter_chunked", "aiter_stream",
838
+ "aiter_body", "aiter_raw", "aiter_content",
839
+ "iter_bytes", "iter_chunks", "iter_chunked", "iter_stream",
840
+ "iter_body", "iter_raw", "iter_content",
841
+ ):
842
+ method = getattr(response, attr, None)
843
+ if method is not None:
844
+ file = method()
845
+ if isinstance(file, (Iterator, AsyncIterator)):
846
+ return bytes_iter_to_async_reader(file, threaded=True)
847
+ for attr in (
848
+ "astream", "abody", "araw", "acontent",
849
+ "stream", "body", "raw", "content",
850
+ ):
851
+ file = getattr(response, attr, None)
852
+ if file is not None:
853
+ if isawaitable(file):
854
+ file = await file
855
+ for attr in ("aread", "read"):
856
+ method = getattr(file, attr, None)
857
+ if method is not None and argcount(method):
858
+ return file
859
+ if callable(file):
860
+ file = file()
861
+ if isawaitable(file):
862
+ file = await file
863
+ if isinstance(file, Buffer):
864
+ return BytesIO(file)
865
+ elif isinstance(file, (Iterator, AsyncIterator)):
866
+ return bytes_iter_to_async_reader(file, threaded=True)
867
+ raise TypeError("can't determine how to `read`")
868
+
869
+ async def _geturl(self, /) -> str:
870
+ return await ageturl(self._url)
871
+
872
+ async def close_response(self, /):
873
+ if response := self.__dict__.get("response"):
624
874
  try:
625
- ret = self.response.aclose()
875
+ ret = response.aclose()
626
876
  except (AttributeError, TypeError):
627
877
  try:
628
- ret = self.response.close()
878
+ ret = response.close()
629
879
  except (AttributeError, TypeError):
630
880
  return
631
881
  if isawaitable(ret):
632
- run_async(ret)
882
+ await ret
633
883
 
634
- async def close_response(self, /):
884
+ async def aclose(self, /):
885
+ if not self.closed:
886
+ await self.close_response()
887
+ self._closed = True
888
+
889
+ def close(self, /):
890
+ from asynctools import run_async
891
+ run_async(self.aclose())
892
+
893
+ def fileno(self, /) -> int:
894
+ file = self.__dict__.get("response")
895
+ while file:
896
+ if hasattr(file, "fileno"):
897
+ return file.fileno()
898
+ file = getattr(file, "raw", None)
899
+ return 0
900
+
901
+ async def flush(self, /):
902
+ if not self.closed:
903
+ file = self.__dict__.get("response")
904
+ while file:
905
+ if hasattr(file, "flush"):
906
+ ret = file.flush()
907
+ if isawaitable(ret):
908
+ ret = await ret
909
+ return ret
910
+ file = getattr(file, "raw", None)
911
+
912
+ def isatty(self, /) -> bool:
913
+ return False
914
+
915
+ def _get_read(self, /):
916
+ try:
917
+ return self.file.aread
918
+ except AttributeError:
919
+ return ensure_async(self.file.read, threaded=True)
920
+
921
+ async def _readinto(self, buffer: Buffer, /) -> int:
922
+ read = self._get_read()
923
+ m = to_bytes_view(buffer)
924
+ size = len(m)
925
+ start = 0
926
+ last_intv = size - size % COPY_BUFSIZE
927
+ while start < size:
928
+ if start >= last_intv:
929
+ delta = size - start
930
+ else:
931
+ delta = COPY_BUFSIZE
932
+ data = await read(delta)
933
+ data_len = len(data)
934
+ m[:data_len] = data
935
+ start += data_len
936
+ if data_len < delta:
937
+ break
938
+ return start
939
+
940
+ async def _readline(self, size: None | int = -1, /) -> bytes:
941
+ read = self._get_read()
942
+ buf = bytearray()
943
+ if size is None or size < 0:
944
+ while b := await read(1):
945
+ buf += b
946
+ if b == b"\n":
947
+ break
948
+ else:
949
+ while size and (b := await read(1)):
950
+ buf += b
951
+ if b == b"\n":
952
+ break
953
+ size -= 1
954
+ return bytes(buf)
955
+
956
+ async def _readlines(self, hint: int = -1, /) -> list[bytes]:
635
957
  try:
636
- ret = self.response.aclose()
637
- except (AttributeError, TypeError):
958
+ readline = self.file.areadline
959
+ except AttributeError:
638
960
  try:
639
- ret = self.response.close()
640
- except (AttributeError, TypeError):
641
- return
642
- if isawaitable(ret):
643
- await ret
961
+ readline = ensure_async(self.file.readline, threaded=True)
962
+ except AttributeError:
963
+ readline = self._readline
964
+ ls: list[bytes]
965
+ add = ls.append
966
+ if hint <= 0:
967
+ while l := await readline():
968
+ add(l)
969
+ else:
970
+ while hint >= 0 and (l := await readline()):
971
+ add(l)
972
+ hint -= len(l)
973
+ return ls
644
974
 
645
- async def flush(self, /):
646
- if self.closed:
647
- raise ValueError("I/O operation on closed file.")
648
- if _initing := self.__dict__.get("_initing"):
649
- await _initing
650
- self.__dict__.pop("_initing", None)
651
- return await ensure_async(self.file.flush, threaded=True)()
975
+ def readable(self, /) -> bool:
976
+ return True
652
977
 
653
978
  async def read(self, size: int = -1, /) -> bytes: # type: ignore
654
- if self.closed:
655
- raise ValueError("I/O operation on closed file.")
656
- if _initing := self.__dict__.get("_initing"):
657
- await _initing
658
- self.__dict__.pop("_initing", None)
659
- if size == 0 or not self.chunked and self.tell() >= self.length:
979
+ pos = self._pos
980
+ if size == 0:
660
981
  return b""
661
- if self.file_closed:
662
- await self.reconnect()
663
- read = ensure_async(self.file.read, threaded=True)
664
- if size is None or size < 0:
665
- data = await read(-1)
666
- else:
667
- data = await read(size)
668
- if data:
669
- self._add_start(len(data))
670
- return data
982
+ length = self.__dict__.get("length")
983
+ if length is not None:
984
+ if length == 0 or length > 0 and pos >= length:
985
+ return b""
986
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
987
+ await self.reconnect(pos)
988
+ if self.file:
989
+ read = self._get_read()
990
+ try:
991
+ if size is None or size < 0:
992
+ data = await read()
993
+ else:
994
+ data = await read(size)
995
+ if data:
996
+ self._pos = pos + len(data)
997
+ return data
998
+ except:
999
+ self._pos = pos
1000
+ await self.aclose()
1001
+ raise
1002
+ return b""
671
1003
 
672
1004
  async def readinto(self, buffer, /) -> int: # type: ignore
673
- if self.closed:
674
- raise ValueError("I/O operation on closed file.")
675
- if _initing := self.__dict__.get("_initing"):
676
- await _initing
677
- self.__dict__.pop("_initing", None)
678
- if not buffer or not self.chunked and self.tell() >= self.length:
1005
+ pos = self._pos
1006
+ if not buffer_length(buffer):
679
1007
  return 0
680
- if self.file_closed:
681
- await self.reconnect()
682
- try:
683
- readinto = ensure_async(self.file.readinto, threaded=True)
684
- except AttributeError:
685
- read = ensure_async(self.file.read, threaded=True)
686
- async def readinto(buffer, /):
687
- data = await read(len(buffer))
688
- size = len(data)
689
- buffer[:size] = data
1008
+ length = self.__dict__.get("length")
1009
+ if length is not None:
1010
+ if length == 0 or length > 0 and pos >= length:
1011
+ return 0
1012
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
1013
+ await self.reconnect(pos)
1014
+ if file := self.file:
1015
+ try:
1016
+ readinto = file.areadinto
1017
+ except AttributeError:
1018
+ try:
1019
+ readinto = ensure_async(file.readinto, threaded=True)
1020
+ except AttributeError:
1021
+ readinto = self._readinto
1022
+ try:
1023
+ size = await readinto(buffer)
1024
+ if size:
1025
+ self._pos = pos + size
690
1026
  return size
691
- size = await readinto(buffer)
692
- if size:
693
- self._add_start(size)
694
- return size
1027
+ except:
1028
+ self._pos = pos
1029
+ await self.aclose()
1030
+ raise
1031
+ return 0
695
1032
 
696
1033
  async def readline(self, size: None | int = -1, /) -> bytes: # type: ignore
697
- if self.closed:
698
- raise ValueError("I/O operation on closed file.")
699
- if _initing := self.__dict__.get("_initing"):
700
- await _initing
701
- self.__dict__.pop("_initing", None)
702
- if size == 0 or not self.chunked and self.tell() >= self.length:
1034
+ pos = self._pos
1035
+ if size == 0:
703
1036
  return b""
704
- if self.file_closed:
705
- await self.reconnect()
706
- # TODO: if no readline method, give a default impl
707
- if size is None or size < 0:
708
- data = await ensure_async(self.file.readline, threaded=True)()
709
- else:
710
- data = await ensure_async(self.file.readline, threaded=True)(size)
711
- if data:
712
- self._add_start(len(data))
713
- return data
1037
+ length = self.__dict__.get("length")
1038
+ if length is not None:
1039
+ if length == 0 or length > 0 and pos >= length:
1040
+ return b""
1041
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
1042
+ await self.reconnect(pos)
1043
+ if file := self.file:
1044
+ try:
1045
+ readline = file.areadline
1046
+ except AttributeError:
1047
+ try:
1048
+ readline = ensure_async(file.readline, threaded=True)
1049
+ except AttributeError:
1050
+ readline = self._readline
1051
+ try:
1052
+ if size is None or size < 0:
1053
+ data = await readline()
1054
+ else:
1055
+ data = await readline(size)
1056
+ if data:
1057
+ self._pos = pos + len(data)
1058
+ return data
1059
+ except:
1060
+ self._pos = pos
1061
+ await self.aclose()
1062
+ raise
1063
+ return b""
714
1064
 
715
1065
  async def readlines(self, hint: int = -1, /) -> list[bytes]: # type: ignore
716
- if self.closed:
717
- raise ValueError("I/O operation on closed file.")
718
- if _initing := self.__dict__.get("_initing"):
719
- await _initing
720
- self.__dict__.pop("_initing", None)
721
- if not self.chunked and self.tell() >= self.length:
722
- return []
723
- if self.file_closed:
724
- await self.reconnect()
725
- # TODO: if no readlines method, give a default impl
726
- ls = await ensure_async(self.file.readlines, threaded=True)(hint)
727
- if ls:
728
- self._add_start(sum(map(len, ls)))
729
- return ls
730
-
731
- async def reconnect(self, /, start: None | int = None) -> int: # type: ignore
732
- if _initing := self.__dict__.get("_initing"):
733
- await _initing
734
- self.__dict__.pop("_initing", None)
735
- if not self._seekable:
736
- if start is None and self.tell() or start:
737
- raise OSError(errno.EOPNOTSUPP, "Unsupport for reconnection of non-seekable streams.")
738
- start = 0
1066
+ pos = self._pos
1067
+ length = self.__dict__.get("length")
1068
+ if length is not None:
1069
+ if length == 0 or length > 0 and pos >= length:
1070
+ return []
1071
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
1072
+ await self.reconnect(pos)
1073
+ if file := self.file:
1074
+ try:
1075
+ readlines = file.areadlines
1076
+ except AttributeError:
1077
+ try:
1078
+ readlines = ensure_async(file.readlines, threaded=True)
1079
+ except AttributeError:
1080
+ readlines = self._readlines
1081
+ try:
1082
+ ls = await readlines(hint)
1083
+ if ls:
1084
+ self._pos = pos + sum(map(len, ls))
1085
+ return ls
1086
+ except:
1087
+ self._pos = pos
1088
+ await self.aclose()
1089
+ raise
1090
+ return []
1091
+
1092
+ async def reconnect(self, /, start: None | int = None) -> int:
739
1093
  if start is None:
740
- start = self.tell()
1094
+ start = self._pos
1095
+ if start and not self.seekable():
1096
+ raise OSError(errno.EOPNOTSUPP, "unsupport for reconnection of non-seekable streams")
1097
+ request_kwargs = self.request_kwargs
1098
+ headers = request_kwargs["headers"] = request_kwargs["headers"]
1099
+ if start >= 0:
1100
+ headers["range"] = f"bytes={start}-"
741
1101
  elif start < 0:
742
- start = self.length + start
743
- if start < 0:
744
- start = 0
745
- if start >= self.length:
746
- self.__dict__.update(start=start)
747
- return start
748
- await self.close_response()
749
- url = self.url
750
- response = await self.urlopen(
751
- (await url()) if callable(url) else url,
752
- headers={**self.headers, "range": f"bytes={start}-"}
753
- )
754
- length_new = get_total_length(response)
755
- if self.length != length_new:
756
- raise OSError(errno.EIO, f"file size changed: {self.length} -> {length_new}")
757
- self.__dict__.update(
758
- response=response,
759
- start=start,
760
- closed=False,
761
- )
1102
+ headers["range"] = f"bytes={start}"
1103
+ await self.aclose()
1104
+ response = await self.request(await self._geturl(), **request_kwargs)
1105
+ status_code = get_status_code(response)
1106
+ if not 200 <= status_code < 300:
1107
+ raise OSError(
1108
+ errno.EIO,
1109
+ {
1110
+ "code": status_code,
1111
+ "response": response,
1112
+ "reason": "status code must be in the `range(200, 300)`",
1113
+ },
1114
+ )
1115
+ if start:
1116
+ rng = get_range(response)
1117
+ if not rng:
1118
+ raise OSError(errno.ESPIPE, "non-seekable")
1119
+ start = self._pos = rng[0]
1120
+ self.response = response
1121
+ self.file = await self._get_file()
1122
+ self._closed = False
762
1123
  return start
763
1124
 
764
1125
  async def seek(self, pos: int, whence: int = 0, /) -> int: # type: ignore
765
- if self.closed:
766
- raise ValueError("I/O operation on closed file.")
767
- if _initing := self.__dict__.get("_initing"):
768
- await _initing
769
- self.__dict__.pop("_initing", None)
770
- if not self._seekable:
1126
+ if not self.seekable():
771
1127
  raise OSError(errno.EINVAL, "not a seekable stream")
772
- if whence == 0:
773
- if pos < 0:
774
- raise OSError(errno.EINVAL, f"negative seek start: {pos!r}")
775
- old_pos = self.tell()
776
- if old_pos == pos:
777
- return pos
778
- if pos > old_pos and (size := pos - old_pos) <= self.seek_threshold:
779
- if size <= COPY_BUFSIZE:
780
- await self.read(size)
781
- else:
782
- buf = bytearray(COPY_BUFSIZE)
783
- readinto = self.readinto
784
- while size > COPY_BUFSIZE:
785
- await readinto(buf)
786
- size -= COPY_BUFSIZE
787
- await self.read(size)
788
- else:
789
- await self.reconnect(pos)
790
- return pos
791
- elif whence == 1:
792
- if pos == 0:
793
- return self.tell()
794
- return await self.seek(self.tell() + pos)
795
- elif whence == 2:
796
- return await self.seek(self.length + pos)
797
- else:
798
- raise OSError(errno.EINVAL, f"whence value unsupported: {whence!r}")
1128
+ old_pos = self._pos
1129
+ match whence:
1130
+ case 1:
1131
+ pos = old_pos + pos
1132
+ case 2:
1133
+ length = self.__dict__.get("length")
1134
+ if length is None:
1135
+ length = await self._init_info()
1136
+ pos += length
1137
+ if pos < 0:
1138
+ pos = 0
1139
+ if (self.__dict__.get("response") and
1140
+ self.file and
1141
+ not self.closed and
1142
+ not self.response_closed and
1143
+ pos > old_pos and
1144
+ (size := pos - old_pos) <= self.seek_threshold
1145
+ ):
1146
+ read = self._get_read()
1147
+ while size > COPY_BUFSIZE:
1148
+ await read(COPY_BUFSIZE)
1149
+ size -= COPY_BUFSIZE
1150
+ await read(size)
1151
+ return await self.reconnect(pos)
799
1152
 
800
- @classmethod
801
- async def new(
802
- cls,
803
- /,
804
- url: str | Callable[[], str],
805
- headers: None | Mapping = None,
806
- start: int = 0,
807
- seek_threshold: int = 1 << 20,
808
- urlopen = None,
809
- ) -> Self: # type: ignore
810
- self = cls.__new__(cls)
811
- await self.__ainit__(
812
- url=url,
813
- headers=headers,
814
- start=start,
815
- seek_threshold=seek_threshold,
816
- urlopen=urlopen,
817
- )
818
- return self
1153
+ def seekable(self, /) -> bool:
1154
+ return self._seekable
1155
+
1156
+ def tell(self, /) -> int:
1157
+ return self._pos
819
1158
 
820
- @overload # type: ignore
1159
+ async def truncate(self, size: None | int = None, /) -> int: # type: ignore
1160
+ raise UnsupportedOperation(errno.ENOTSUP, "truncate")
1161
+
1162
+ async def writable(self, /) -> bool: # type: ignore
1163
+ return False
1164
+
1165
+ async def write(self, b, /) -> int: # type: ignore
1166
+ raise UnsupportedOperation(errno.ENOTSUP, "write")
1167
+
1168
+ async def writelines(self, lines, /): # type: ignore
1169
+ raise UnsupportedOperation(errno.ENOTSUP, "writelines")
1170
+
1171
+ @overload
821
1172
  @classmethod
822
1173
  async def open(
823
1174
  cls,
824
1175
  /,
825
- url: str | Callable[[], str],
1176
+ url: (
1177
+ str | SupportsGeturl | URL |
1178
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1179
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
1180
+ ),
826
1181
  mode: Literal["br", "rb"],
827
1182
  *,
828
1183
  buffering: None | int = None,
829
1184
  encoding: None | str = None,
830
1185
  errors: None | str = None,
831
1186
  newline: None | str = None,
832
- headers: None | Mapping = None,
833
1187
  start: int = 0,
834
1188
  seek_threshold: int = 1 << 20,
835
- urlopen = None,
1189
+ request: None | Callable[..., Awaitable[Response]] = None,
1190
+ get_file: None | str | Callable[[Response], Any] = None,
1191
+ **request_kwargs,
836
1192
  ) -> Self | AsyncBufferedReader:
837
1193
  ...
838
1194
  @overload
@@ -840,45 +1196,57 @@ class AsyncHTTPFileReader(HTTPFileReader):
840
1196
  async def open(
841
1197
  cls,
842
1198
  /,
843
- url: str | Callable[[], str],
1199
+ url: (
1200
+ str | SupportsGeturl | URL |
1201
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1202
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
1203
+ ),
844
1204
  mode: Literal["r", "rt", "tr"] = "r",
845
1205
  *,
846
1206
  buffering: None | int = None,
847
1207
  encoding: None | str = None,
848
1208
  errors: None | str = None,
849
1209
  newline: None | str = None,
850
- headers: None | Mapping = None,
851
1210
  start: int = 0,
852
1211
  seek_threshold: int = 1 << 20,
853
- urlopen = None,
1212
+ request: None | Callable[..., Awaitable[Response]] = None,
1213
+ get_file: None | str | Callable[[Response], Any] = None,
1214
+ **request_kwargs,
854
1215
  ) -> AsyncTextIOWrapper:
855
1216
  ...
856
1217
  @classmethod
857
1218
  async def open(
858
1219
  cls,
859
1220
  /,
860
- url: str | Callable[[], str],
1221
+ url: (
1222
+ str | SupportsGeturl | URL |
1223
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1224
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]]
1225
+ ),
861
1226
  mode: Literal["r", "rt", "tr", "br", "rb"] = "r",
1227
+ *,
862
1228
  buffering: None | int = None,
863
1229
  encoding: None | str = None,
864
1230
  errors: None | str = None,
865
1231
  newline: None | str = None,
866
- headers: None | Mapping = None,
867
1232
  start: int = 0,
868
1233
  seek_threshold: int = 1 << 20,
869
- urlopen = None,
1234
+ request: None | Callable[..., Awaitable[Response]] = None,
1235
+ get_file: None | str | Callable[[Response], Any] = None,
1236
+ **request_kwargs,
870
1237
  ) -> Self | AsyncBufferedReader | AsyncTextIOWrapper:
871
- file = await cls.new(
872
- url=url,
873
- headers=headers,
1238
+ file = cls(
1239
+ url,
874
1240
  start=start,
875
1241
  seek_threshold=seek_threshold,
876
- urlopen=urlopen,
1242
+ request=request,
1243
+ get_file=get_file,
1244
+ **request_kwargs,
877
1245
  )
878
1246
  if mode not in ("r", "rt", "tr", "rb", "br"):
879
1247
  raise OSError(errno.EINVAL, f"invalid (or unsupported) mode: {mode!r}")
880
1248
  return file.wrap(
881
- text_mode="b" not in mode, # type: ignore
1249
+ text_mode="b" not in cast(str, mode), # type: ignore
882
1250
  buffering=buffering,
883
1251
  encoding=encoding,
884
1252
  errors=errors,
@@ -886,7 +1254,7 @@ class AsyncHTTPFileReader(HTTPFileReader):
886
1254
  )
887
1255
 
888
1256
  @overload
889
- def wrap(
1257
+ async def wrap(
890
1258
  self,
891
1259
  /,
892
1260
  text_mode: Literal[False] = False,
@@ -898,7 +1266,7 @@ class AsyncHTTPFileReader(HTTPFileReader):
898
1266
  ) -> Self | AsyncBufferedReader:
899
1267
  ...
900
1268
  @overload
901
- def wrap(
1269
+ async def wrap(
902
1270
  self,
903
1271
  /,
904
1272
  text_mode: Literal[True],
@@ -909,7 +1277,7 @@ class AsyncHTTPFileReader(HTTPFileReader):
909
1277
  newline: None | str = None,
910
1278
  ) -> AsyncTextIOWrapper:
911
1279
  ...
912
- def wrap(
1280
+ async def wrap(
913
1281
  self,
914
1282
  /,
915
1283
  text_mode: bool = False,
@@ -940,6 +1308,8 @@ class AsyncHTTPFileReader(HTTPFileReader):
940
1308
  line_buffering = True
941
1309
  else:
942
1310
  buffer_size = buffering
1311
+ if "response" not in self.__dict__ or self.closed or self.response_closed:
1312
+ await self.reconnect()
943
1313
  raw = self
944
1314
  buffer = AsyncBufferedReader(raw, buffer_size)
945
1315
  if text_mode:
@@ -954,278 +1324,623 @@ class AsyncHTTPFileReader(HTTPFileReader):
954
1324
  return buffer
955
1325
 
956
1326
 
957
- if find_spec("urllib3"):
958
- from urllib.error import HTTPError
1327
+ class MultipartHTTPFileReader[Response](HTTPFileReader[Response]):
959
1328
 
960
- _urllib3_urlopen = None
1329
+ def __init__(
1330
+ self,
1331
+ /,
1332
+ urls: (
1333
+ str | SupportsGeturl | URL |
1334
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1335
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1336
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1337
+ Iterable[
1338
+ str | SupportsGeturl | URL |
1339
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1340
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1341
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]]
1342
+ ]
1343
+ ),
1344
+ start: int = 0,
1345
+ seek_threshold: int = 1 << 20,
1346
+ request: None | Callable[..., Response] = None,
1347
+ get_file: None | str | Callable[[Response], Any] = None,
1348
+ **request_kwargs,
1349
+ ):
1350
+ self.seek_threshold = seek_threshold
1351
+ if request is None:
1352
+ from urllib3_future_request import request_sync as request # type: ignore
1353
+ self.request = cast(Callable[..., Response], request)
1354
+ self.get_file = get_file # type: ignore
1355
+ self.request_kwargs = request_kwargs
1356
+ if (isinstance(urls, (str, SupportsGeturl, URL)) or
1357
+ callable(urls) or
1358
+ isinstance(urls, tuple) and urls and isinstance(urls[0], int)
1359
+ ):
1360
+ urls = [urls]
1361
+ else:
1362
+ urls = list(urls) # type: ignore
961
1363
 
962
- class Urllib3FileReader(HTTPFileReader):
1364
+ from urllib3_future_request import request_sync
963
1365
 
964
- def __init__(
965
- self,
966
- /,
967
- url: str | Callable[[], str],
968
- headers: None | Mapping = None,
969
- start: int = 0,
970
- seek_threshold: int = 1 << 20,
971
- urlopen = None,
972
- ):
973
- global _urllib3_urlopen
974
- from urllib3 import request
975
- from urllib3.poolmanager import PoolManager
976
-
977
- if isinstance(urlopen, PoolManager):
978
- urlopen = partial(urlopen.request, "GET", preload_content=False)
979
- elif urlopen is None:
980
- if _urllib3_urlopen is None:
981
- urlopen = _urllib3_urlopen = partial(cast(Callable, request), "GET", preload_content=False)
982
- else:
983
- urlopen = _urllib3_urlopen
1366
+ self._parts: list[dict] = []
1367
+ add_part = self._parts.append
1368
+ seekable = True
1369
+ length = 0
1370
+ request_kwargs = dict(request_kwargs, parse=...)
1371
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
1372
+ headers["range"] = "bytes=0-0"
1373
+ for url in urls:
1374
+ if isinstance(url, tuple):
1375
+ size, url = url
984
1376
  else:
985
- urlopen = partial(urlopen, preload_content=False)
986
-
987
- def urlopen_wrapper(url: str, headers: None | Mapping = headers):
988
- resp = urlopen(url, headers=headers)
989
- if resp.status >= 400:
990
- raise HTTPError(resp.url, resp.status, resp.reason, resp.headers, resp)
991
- return resp
992
-
993
- super().__init__(
994
- url,
995
- headers=headers,
996
- start=start,
997
- seek_threshold=seek_threshold,
998
- urlopen=urlopen_wrapper,
999
- )
1377
+ url_ = url
1378
+ if callable(url_):
1379
+ url_ = url_()
1380
+ if isinstance(url_, SupportsGeturl):
1381
+ url_ = url_.geturl()
1382
+ else:
1383
+ url_ = str(url_)
1384
+ resp = request_sync(url_, **request_kwargs)
1385
+ if not is_range_request(resp):
1386
+ seekable = False
1387
+ size = get_total_length(resp) or 0
1388
+ if size:
1389
+ start = length
1390
+ length += size
1391
+ stop = length
1392
+ add_part({
1393
+ "url": url,
1394
+ "length": size,
1395
+ "start": start,
1396
+ "stop": stop,
1397
+ })
1398
+ self.length = length
1399
+ if start < 0:
1400
+ start += length
1401
+ if start < 0:
1402
+ start = 0
1403
+ self._pos = start
1404
+ self._seekable = seekable
1405
+ self._closed = False
1000
1406
 
1001
- __all__.append("Urllib3FileReader")
1407
+ def __repr__(self):
1408
+ cls = type(self)
1409
+ kwargs = {
1410
+ "urls": [(d["length"], d["url"]) for d in self._parts],
1411
+ "start": self._pos,
1412
+ "seek_threshold": self.seek_threshold,
1413
+ "request": self.request,
1414
+ "get_file": self.get_file,
1415
+ **self.request_kwargs,
1416
+ }
1417
+ return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
1002
1418
 
1419
+ @cached_property
1420
+ def response(self, /):
1421
+ return self.file.response
1003
1422
 
1004
- if find_spec("requests"):
1005
- _requests_urlopen = None
1423
+ def close(self, /):
1424
+ self.file.close()
1425
+ self._closed = True
1006
1426
 
1007
- class RequestsFileReader(HTTPFileReader):
1427
+ def read(self, size: None | int = -1, /) -> bytes:
1428
+ if size is None or size < 0:
1429
+ size = self.length - self._pos
1430
+ else:
1431
+ size = min(self.length - self._pos, size)
1432
+ if size <= 0:
1433
+ return b""
1434
+ buf = bytearray(size)
1435
+ self.readinto(buf)
1436
+ return bytes(buf)
1437
+
1438
+ def readinto(self, buffer: Buffer, /) -> int:
1439
+ pos = self._pos
1440
+ remaining = min(self.length - pos, buffer_length(buffer))
1441
+ if remaining <= 0:
1442
+ return 0
1443
+ if "file" not in self.__dict__ or self.closed or self.response_closed:
1444
+ self.reconnect(pos)
1445
+ try:
1446
+ readinto = self.file.readinto
1447
+ view = to_bytes_view(buffer)[:remaining]
1448
+ size = 0
1449
+ while size < remaining:
1450
+ start_pos = pos + size
1451
+ self.seek(start_pos)
1452
+ size += readinto(view[size:])
1453
+ return size
1454
+ except:
1455
+ self._pos = pos
1456
+ self.close()
1457
+ raise
1008
1458
 
1009
- def __init__(
1010
- self,
1011
- /,
1012
- url: str | Callable[[], str],
1013
- headers: None | Mapping = None,
1014
- start: int = 0,
1015
- seek_threshold: int = 1 << 20,
1016
- urlopen = None,
1017
- ):
1018
- global _requests_urlopen
1019
- from requests import request, Session
1020
- if isinstance(urlopen, Session):
1021
- urlopen = partial(urlopen.request, "GET", stream=True)
1022
- elif urlopen is None:
1023
- if _requests_urlopen is None:
1024
- if "__del__" not in Session.__dict__:
1025
- setattr(Session, "__del__", lambda self, /: self.close())
1026
- urlopen = _requests_urlopen = partial(request, "GET", stream=True)
1459
+ def readline(self, size: None | int = -1, /) -> bytes:
1460
+ pos = self._pos
1461
+ if size is None or size < 0:
1462
+ size = self.length - pos
1463
+ else:
1464
+ size = min(self.length - pos, size)
1465
+ if size <= 0:
1466
+ return b""
1467
+ if "file" not in self.__dict__ or self.closed or self.response_closed:
1468
+ self.reconnect(pos)
1469
+ try:
1470
+ readline = self.file.readline
1471
+ buf = bytearray()
1472
+ while size > 0:
1473
+ if line := readline(size):
1474
+ buf += line
1475
+ if buf.endswith(b"\n"):
1476
+ break
1027
1477
  else:
1028
- urlopen = _requests_urlopen
1478
+ self.seek(pos + len(buf))
1479
+ size -= len(line)
1480
+ return bytes(buf)
1481
+ except:
1482
+ self._pos = pos
1483
+ self.close()
1484
+ raise
1485
+
1486
+ def readlines(self, hint: int = -1, /) -> list[bytes]:
1487
+ pos = self._pos
1488
+ readline = self.readline
1489
+ ls: list[bytes]
1490
+ add = ls.append
1491
+ try:
1492
+ if hint <= 0:
1493
+ while l := readline():
1494
+ add(l)
1029
1495
  else:
1030
- urlopen = partial(urlopen, stream=True)
1031
- def urlopen_wrapper(url: str, headers: None | Mapping = headers):
1032
- resp = urlopen(url, headers=headers)
1033
- resp.raise_for_status()
1034
- return resp
1035
- super().__init__(
1036
- url,
1037
- headers=headers,
1038
- start=start,
1039
- seek_threshold=seek_threshold,
1040
- urlopen=urlopen_wrapper,
1041
- )
1496
+ while hint >=0. and (l := readline()):
1497
+ add(l)
1498
+ hint -= len(l)
1499
+ return ls
1500
+ except:
1501
+ self._pos = pos
1502
+ self.close()
1503
+ raise
1504
+
1505
+ def read_range(self, size: None | int = -1, /, offset: int = 0) -> bytes:
1506
+ if offset < 0:
1507
+ offset += self.length
1508
+ if offset < 0:
1509
+ offset = 0
1510
+ if size is None or size < 0:
1511
+ size = self.length - offset
1512
+ else:
1513
+ size = min(self.length - offset, size)
1514
+ if size <= 0:
1515
+ return b""
1516
+ buf = bytearray(size)
1517
+ self.readinto_range(buf, offset)
1518
+ return bytes(buf)
1519
+
1520
+ def readinto_range(self, buffer: Buffer, /, offset: int = 0):
1521
+ from urllib3_future_request import request_sync as request
1522
+
1523
+ if offset < 0:
1524
+ offset += self.length
1525
+ if offset < 0:
1526
+ offset = 0
1527
+ remaining = min(self.length - offset, buffer_length(buffer))
1528
+ if remaining <= 0:
1529
+ return 0
1530
+ request_kwargs = dict(self.request_kwargs)
1531
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
1532
+ headers["accept-encoding"] = "identity"
1533
+ view = to_bytes_view(buffer)[:remaining]
1534
+ size = 0
1535
+ while size < remaining:
1536
+ for vol in self._parts:
1537
+ if vol["start"] <= offset < vol["stop"]:
1538
+ vol_offset = offset + size - vol["start"]
1539
+ headers["range"] = f"bytes={vol_offset}-"
1540
+ resp = request(vol["url"], **request_kwargs)
1541
+ try:
1542
+ if vol_offset and resp.headers.get("accept-ranges") != "bytes":
1543
+ raise OSError(29, "non-seekable stream")
1544
+ size += resp.readinto(view[size:])
1545
+ finally:
1546
+ resp.close()
1547
+ return size
1042
1548
 
1043
- @funcproperty
1044
- def file(self, /) -> BinaryIO:
1045
- return self.response.raw
1549
+ def reconnect(self, /, start: None | int = None) -> int:
1550
+ if start is None:
1551
+ start = self._pos
1552
+ elif start < 0:
1553
+ start += self.length
1554
+ if start < 0:
1555
+ start = 0
1556
+ if start >= self.length:
1557
+ return start
1558
+ for vol in self._parts:
1559
+ if vol["start"] <= start < vol["stop"]:
1560
+ self.close()
1561
+ file = self.file = HTTPFileReader(
1562
+ vol["url"],
1563
+ start=vol["start"] - start,
1564
+ seek_threshold=self.seek_threshold,
1565
+ request=self.request,
1566
+ get_file=self.get_file,
1567
+ **self.request_kwargs,
1568
+ )
1569
+ self.response = file.response
1570
+ self._pos = start
1571
+ self._closed = False
1572
+ return start
1573
+ return start
1046
1574
 
1047
- __all__.append("RequestsFileReader")
1575
+ def seek(self, pos: int, whence: int = 0, /) -> int:
1576
+ old_pos = self._pos
1577
+ match whence:
1578
+ case 1:
1579
+ pos = old_pos + pos
1580
+ case 2:
1581
+ pos += self.length
1582
+ if pos < 0:
1583
+ pos = 0
1584
+ if pos == old_pos:
1585
+ return pos
1586
+ elif pos >= self.length:
1587
+ self.close()
1588
+ self._pos = pos
1589
+ return pos
1590
+ if hasattr(self, "file"):
1591
+ file = self.file
1592
+ start = old_pos - file._pos
1593
+ stop = start + file.length
1594
+ if start <= pos < stop:
1595
+ file.seek(pos-start)
1596
+ self._pos = pos
1597
+ return pos
1598
+ return self.reconnect(pos)
1048
1599
 
1049
1600
 
1050
- if find_spec("aiohttp"):
1051
- _aiohttp_urlopen = None
1601
+ class AsyncMultipartHTTPFileReader[Response](AsyncHTTPFileReader[Response]):
1052
1602
 
1053
- class AiohttpFileReader(AsyncHTTPFileReader):
1603
+ def __init__(
1604
+ self,
1605
+ /,
1606
+ urls: (
1607
+ str | SupportsGeturl | URL |
1608
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1609
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
1610
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1611
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1612
+ tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]] |
1613
+ Iterable[
1614
+ str | SupportsGeturl | URL |
1615
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1616
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
1617
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1618
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1619
+ tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]]
1620
+ ]
1621
+ ),
1622
+ start: int = 0,
1623
+ seek_threshold: int = 1 << 20,
1624
+ request: None | Callable[..., Awaitable[Response]] = None,
1625
+ get_file: None | str | Callable[[Response], Any] = None,
1626
+ **request_kwargs,
1627
+ ):
1628
+ run_async(self.__ainit__(
1629
+ urls,
1630
+ start=start,
1631
+ seek_threshold=seek_threshold,
1632
+ request=request,
1633
+ get_file=get_file,
1634
+ **request_kwargs,
1635
+ ))
1054
1636
 
1055
- async def __ainit__(
1056
- self,
1057
- /,
1058
- url: str | Callable[[], str] | Callable[[], Awaitable[str]],
1059
- headers: None | Mapping = None,
1060
- start: int = 0,
1061
- seek_threshold: int = 1 << 20,
1062
- urlopen = None,
1637
+ async def __ainit__(
1638
+ self,
1639
+ /,
1640
+ urls: (
1641
+ str | SupportsGeturl | URL |
1642
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1643
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
1644
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1645
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1646
+ tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]] |
1647
+ Iterable[
1648
+ str | SupportsGeturl | URL |
1649
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1650
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
1651
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1652
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1653
+ tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]]
1654
+ ]
1655
+ ),
1656
+ start: int = 0,
1657
+ seek_threshold: int = 1 << 20,
1658
+ request: None | Callable[..., Awaitable[Response]] = None,
1659
+ get_file: None | str | Callable[[Response], Any] = None,
1660
+ **request_kwargs,
1661
+ ):
1662
+ self.seek_threshold = seek_threshold
1663
+ if request is None:
1664
+ from urllib3_future_request import request_async as request # type: ignore
1665
+ self.request = cast(Callable[..., Awaitable[Response]], request)
1666
+ self.get_file = get_file # type: ignore
1667
+ self.request_kwargs = request_kwargs
1668
+ if (isinstance(urls, (str, SupportsGeturl, URL)) or
1669
+ callable(urls) or
1670
+ isinstance(urls, tuple) and urls and isinstance(urls[0], int)
1063
1671
  ):
1064
- global _aiohttp_urlopen
1065
- from aiohttp import request, ClientSession
1066
-
1067
- if isinstance(urlopen, ClientSession):
1068
- urlopen = urlopen.get
1069
- elif urlopen is None:
1070
- if _aiohttp_urlopen is None:
1071
- urlopen = _aiohttp_urlopen = partial(request, "GET")
1072
- else:
1073
- urlopen = _aiohttp_urlopen
1074
- async def urlopen_wrapper(url: str, headers: None | Mapping = headers):
1075
- ctx = urlopen(url, headers=headers)
1076
- resp = await ctx.__aenter__()
1077
- resp.aclose = lambda: ctx.__aexit__(None, None, None)
1078
- resp.raise_for_status()
1079
- return resp
1080
- await super().__ainit__(
1081
- url=url,
1082
- headers=headers,
1083
- start=start,
1084
- seek_threshold=seek_threshold,
1085
- urlopen=urlopen_wrapper,
1086
- )
1672
+ urls = [urls]
1673
+ else:
1674
+ urls = list(urls) # type: ignore
1087
1675
 
1088
- @funcproperty
1089
- def file(self, /) -> BinaryIO:
1090
- return self.response.content
1676
+ from urllib3_future_request import request_async
1091
1677
 
1092
- @funcproperty
1093
- def file_closed(self, /) -> bool:
1094
- return self.response.closed
1678
+ self._parts: list[dict] = []
1679
+ add_part = self._parts.append
1680
+ seekable = True
1681
+ length = 0
1682
+ request_kwargs = dict(request_kwargs, parse=...)
1683
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
1684
+ headers["range"] = "bytes=0-0"
1685
+ for url in urls:
1686
+ if isinstance(url, tuple):
1687
+ size, url = url
1688
+ else:
1689
+ resp = await request_async(await ageturl(url), **request_kwargs)
1690
+ if not is_range_request(resp):
1691
+ seekable = False
1692
+ size = get_total_length(resp) or 0
1693
+ if size:
1694
+ start = length
1695
+ length += size
1696
+ stop = length
1697
+ add_part({
1698
+ "url": url,
1699
+ "length": size,
1700
+ "start": start,
1701
+ "stop": stop,
1702
+ })
1703
+ self.length = length
1704
+ if start < 0:
1705
+ start += length
1706
+ if start < 0:
1707
+ start = 0
1708
+ self._pos = start
1709
+ self._seekable = seekable
1710
+ self._closed = False
1095
1711
 
1096
- __all__.append("AiohttpFileReader")
1712
+ @classmethod
1713
+ async def new( # type: ignore
1714
+ cls,
1715
+ /,
1716
+ urls: (
1717
+ str | SupportsGeturl | URL |
1718
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1719
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
1720
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1721
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1722
+ tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]] |
1723
+ Iterable[
1724
+ str | SupportsGeturl | URL |
1725
+ Callable[[], str] | Callable[[], SupportsGeturl] | Callable[[], URL] |
1726
+ Callable[[], Awaitable[str]] | Callable[[], Awaitable[SupportsGeturl]] | Callable[[], Awaitable[URL]] |
1727
+ tuple[int, str] | tuple[int, SupportsGeturl] | tuple[int, URL] |
1728
+ tuple[int, Callable[[], str]] | tuple[int, Callable[[], SupportsGeturl]] | tuple[int, Callable[[], URL]] |
1729
+ tuple[int, Callable[[], Awaitable[str]]] | tuple[int, Callable[[], Awaitable[SupportsGeturl]]] | tuple[int, Callable[[], Awaitable[URL]]]
1730
+ ]
1731
+ ),
1732
+ start: int = 0,
1733
+ seek_threshold: int = 1 << 20,
1734
+ request: None | Callable[..., Awaitable[Response]] = None,
1735
+ get_file: None | str | Callable[[Response], Any] = None,
1736
+ **request_kwargs,
1737
+ ) -> Self:
1738
+ self = cls.__new__(cls)
1739
+ await self.__ainit__(
1740
+ urls,
1741
+ start=start,
1742
+ seek_threshold=seek_threshold,
1743
+ request=request,
1744
+ get_file=get_file,
1745
+ **request_kwargs,
1746
+ )
1747
+ return self
1097
1748
 
1749
+ def __repr__(self):
1750
+ cls = type(self)
1751
+ kwargs = {
1752
+ "urls": [(d["length"], d["url"]) for d in self._parts],
1753
+ "start": self._pos,
1754
+ "seek_threshold": self.seek_threshold,
1755
+ "request": self.request,
1756
+ "get_file": self.get_file,
1757
+ **self.request_kwargs,
1758
+ }
1759
+ return f"{cls.__module__}.{cls.__qualname__}({', '.join(map('%s=%r'.__mod__, kwargs.items()))})"
1098
1760
 
1099
- if find_spec("httpx"):
1100
- from filewrap import bytes_iter_to_reader, bytes_iter_to_async_reader
1761
+ @cached_property
1762
+ def response(self, /):
1763
+ return self.file.response
1101
1764
 
1102
- def httpx_request(
1103
- url,
1104
- method: str = "GET",
1105
- auth = None,
1106
- follow_redirects: bool = True,
1107
- stream: bool = True,
1108
- session = None,
1109
- **request_kwargs,
1110
- ):
1111
- if session is None:
1112
- from httpx import Client
1113
- if "__del__" not in Client.__dict__:
1114
- setattr(Client, "__del__", lambda self: self.close())
1115
- session = Client()
1116
- resp = session.send(
1117
- request=session.build_request(
1118
- method=method,
1119
- url=url,
1120
- **request_kwargs,
1121
- ),
1122
- auth=auth,
1123
- follow_redirects=follow_redirects,
1124
- stream=stream,
1125
- )
1126
- setattr(resp, "session", session)
1127
- return resp
1128
-
1129
- async def httpx_request_async(
1130
- url,
1131
- method: str = "GET",
1132
- auth = None,
1133
- follow_redirects: bool = True,
1134
- stream: bool = True,
1135
- session = None,
1136
- **request_kwargs,
1137
- ):
1138
- if session is None:
1139
- from httpx import AsyncClient
1140
- if "__del__" not in AsyncClient.__dict__:
1141
- setattr(AsyncClient, "__del__", lambda self: run_async(self.aclose()))
1142
- session = AsyncClient()
1143
- resp = await session.send(
1144
- request=session.build_request(
1145
- method=method,
1146
- url=url,
1147
- **request_kwargs,
1148
- ),
1149
- auth=auth,
1150
- follow_redirects=follow_redirects,
1151
- stream=stream,
1152
- )
1153
- setattr(resp, "session", session)
1154
- return resp
1155
-
1156
- class HttpxFileReader(HTTPFileReader):
1157
-
1158
- def __init__(
1159
- self,
1160
- /,
1161
- url: str | Callable[[], str],
1162
- headers: None | Mapping = None,
1163
- start: int = 0,
1164
- seek_threshold: int = 1 << 20,
1165
- urlopen = None,
1166
- ):
1167
- from httpx import Client
1168
-
1169
- if urlopen is None or isinstance(urlopen, Client):
1170
- urlopen = partial(httpx_request, session=urlopen)
1171
-
1172
- def urlopen_wrapper(url: str, headers: None | Mapping = headers):
1173
- resp = urlopen(url, headers=headers)
1174
- resp.raise_for_status()
1175
- file = bytes_iter_to_reader(resp.iter_raw())
1176
- self.__dict__["file"] = file
1177
- return resp
1178
-
1179
- super().__init__(
1180
- url,
1181
- headers=headers,
1182
- start=start,
1183
- seek_threshold=seek_threshold,
1184
- urlopen=urlopen_wrapper,
1185
- )
1765
+ async def aclose(self, /):
1766
+ await self.file.aclose()
1767
+ self._closed = True
1186
1768
 
1187
- @funcproperty
1188
- def file_closed(self, /) -> bool:
1189
- return self.response.is_closed
1769
+ def close(self, /):
1770
+ self.file.close()
1771
+ self._closed = True
1190
1772
 
1191
- class AsyncHttpxFileReader(AsyncHTTPFileReader):
1773
+ async def read(self, size: None | int = -1, /) -> bytes: # type: ignore
1774
+ if size is None or size < 0:
1775
+ size = self.length - self._pos
1776
+ else:
1777
+ size = min(self.length - self._pos, size)
1778
+ if size <= 0:
1779
+ return b""
1780
+ buf = bytearray(size)
1781
+ await self.readinto(buf)
1782
+ return bytes(buf)
1783
+
1784
+ async def readinto(self, buffer: Buffer, /) -> int: # type: ignore
1785
+ pos = self._pos
1786
+ remaining = min(self.length - pos, buffer_length(buffer))
1787
+ if remaining <= 0:
1788
+ return 0
1789
+ if "file" not in self.__dict__ or self.closed or self.response_closed:
1790
+ await self.reconnect(pos)
1791
+ try:
1792
+ readinto = self.file.readinto
1793
+ view = to_bytes_view(buffer)[:remaining]
1794
+ size = 0
1795
+ while size < remaining:
1796
+ start_pos = pos + size
1797
+ await self.seek(start_pos)
1798
+ size += await readinto(view[size:])
1799
+ return size
1800
+ except:
1801
+ self._pos = pos
1802
+ await self.aclose()
1803
+ raise
1192
1804
 
1193
- async def __ainit__(
1194
- self,
1195
- /,
1196
- url: str | Callable[[], str] | Callable[[], Awaitable[str]],
1197
- headers: None | Mapping = None,
1198
- start: int = 0,
1199
- seek_threshold: int = 1 << 20,
1200
- urlopen = None,
1201
- ):
1202
- from httpx import AsyncClient
1203
-
1204
- if urlopen is None or isinstance(urlopen, AsyncClient):
1205
- urlopen = partial(httpx_request_async, session=urlopen)
1206
-
1207
- async def urlopen_wrapper(url: str, headers: None | Mapping = headers):
1208
- resp = await urlopen(url, headers=headers)
1209
- resp.raise_for_status()
1210
- file = bytes_iter_to_async_reader(resp.aiter_raw())
1211
- self.__dict__["file"] = file
1212
- return resp
1213
-
1214
- await super().__ainit__(
1215
- url=url,
1216
- headers=headers,
1217
- start=start,
1218
- seek_threshold=seek_threshold,
1219
- urlopen=urlopen_wrapper,
1220
- )
1805
+ async def readline(self, size: None | int = -1, /) -> bytes: # type: ignore
1806
+ pos = self._pos
1807
+ if size is None or size < 0:
1808
+ size = self.length - pos
1809
+ else:
1810
+ size = min(self.length - pos, size)
1811
+ if size <= 0:
1812
+ return b""
1813
+ if "file" not in self.__dict__ or self.closed or self.response_closed:
1814
+ await self.reconnect(pos)
1815
+ try:
1816
+ readline = self.file.readline
1817
+ buf = bytearray()
1818
+ while size > 0:
1819
+ if line := await readline(size):
1820
+ buf += line
1821
+ if buf.endswith(b"\n"):
1822
+ break
1823
+ else:
1824
+ await self.seek(pos + len(buf))
1825
+ size -= len(line)
1826
+ return bytes(buf)
1827
+ except:
1828
+ self._pos = pos
1829
+ await self.aclose()
1830
+ raise
1221
1831
 
1222
- @funcproperty
1223
- def file_closed(self, /) -> bool:
1224
- return self.response.is_closed
1832
+ async def readlines(self, hint: int = -1, /) -> list[bytes]: # type: ignore
1833
+ pos = self._pos
1834
+ readline = self.readline
1835
+ ls: list[bytes]
1836
+ add = ls.append
1837
+ try:
1838
+ if hint <= 0:
1839
+ while l := await readline():
1840
+ add(l)
1841
+ else:
1842
+ while hint >=0. and (l := await readline()):
1843
+ add(l)
1844
+ hint -= len(l)
1845
+ return ls
1846
+ except:
1847
+ self._pos = pos
1848
+ await self.aclose()
1849
+ raise
1850
+
1851
+ async def read_range(self, size: None | int = -1, /, offset: int = 0) -> bytes:
1852
+ if offset < 0:
1853
+ offset += self.length
1854
+ if offset < 0:
1855
+ offset = 0
1856
+ if size is None or size < 0:
1857
+ size = self.length - offset
1858
+ else:
1859
+ size = min(self.length - offset, size)
1860
+ if size <= 0:
1861
+ return b""
1862
+ buf = bytearray(size)
1863
+ await self.readinto_range(buf, offset)
1864
+ return bytes(buf)
1865
+
1866
+ async def readinto_range(self, buffer: Buffer, /, offset: int = 0):
1867
+ from urllib3_future_request import request_async
1868
+
1869
+ if offset < 0:
1870
+ offset += self.length
1871
+ if offset < 0:
1872
+ offset = 0
1873
+ remaining = min(self.length - offset, buffer_length(buffer))
1874
+ if remaining <= 0:
1875
+ return 0
1876
+ request_kwargs = dict(self.request_kwargs)
1877
+ headers = request_kwargs["headers"] = dict(request_kwargs.get("headers") or ())
1878
+ headers["accept-encoding"] = "identity"
1879
+ view = to_bytes_view(buffer)[:remaining]
1880
+ size = 0
1881
+ while size < remaining:
1882
+ for vol in self._parts:
1883
+ if vol["start"] <= offset < vol["stop"]:
1884
+ vol_offset = offset + size - vol["start"]
1885
+ headers["range"] = f"bytes={vol_offset}-"
1886
+ resp = await request_async(vol["url"], **request_kwargs)
1887
+ try:
1888
+ if vol_offset and resp.headers.get("accept-ranges") != "bytes":
1889
+ raise OSError(29, "non-seekable stream")
1890
+ size += await resp.readinto(view[size:])
1891
+ finally:
1892
+ await resp.close()
1893
+ return size
1225
1894
 
1226
- __all__.append("HttpxFileReader")
1227
- __all__.append("AsyncHttpxFileReader")
1895
+ async def reconnect(self, /, start: None | int = None) -> int:
1896
+ if start is None:
1897
+ start = self._pos
1898
+ elif start < 0:
1899
+ start += self.length
1900
+ if start < 0:
1901
+ start = 0
1902
+ if start >= self.length:
1903
+ return start
1904
+ for vol in self._parts:
1905
+ if vol["start"] <= start < vol["stop"]:
1906
+ await self.aclose()
1907
+ file = self.file = await AsyncHTTPFileReader.new(
1908
+ vol["url"],
1909
+ start=vol["start"] - start,
1910
+ seek_threshold=self.seek_threshold,
1911
+ request=self.request,
1912
+ get_file=self.get_file,
1913
+ **self.request_kwargs,
1914
+ )
1915
+ self.response = file.response
1916
+ self._pos = start
1917
+ self._closed = False
1918
+ return start
1919
+ return start
1920
+
1921
+ async def seek(self, pos: int, whence: int = 0, /) -> int: # type: ignore
1922
+ old_pos = self._pos
1923
+ match whence:
1924
+ case 1:
1925
+ pos = old_pos + pos
1926
+ case 2:
1927
+ pos += self.length
1928
+ if pos < 0:
1929
+ pos = 0
1930
+ if pos == old_pos:
1931
+ return pos
1932
+ elif pos >= self.length:
1933
+ await self.aclose()
1934
+ self._pos = pos
1935
+ return pos
1936
+ if hasattr(self, "file"):
1937
+ file = self.file
1938
+ start = old_pos - file._pos
1939
+ stop = start + file.length
1940
+ if start <= pos < stop:
1941
+ await file.seek(pos-start)
1942
+ self._pos = pos
1943
+ return pos
1944
+ return await self.reconnect(pos)
1228
1945
 
1229
- # TODO: 实现任意请求模块的 HTTPFileReader,只要它支持流式读取即可(分成 2 种类型:要么支持 read,要么支持迭代器)
1230
1946
  # TODO: 设计实现一个 HTTPFileWriter,用于实现上传,关闭后视为上传完成
1231
- # TODO: httpx 重新实现 stream 和 async_stream 方法,确保不需要用上下文管理器,如此才能真的简化