omlish 0.0.0.dev368__py3-none-any.whl → 0.0.0.dev369__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omlish/__about__.py +2 -2
- omlish/http/coro/client/__init__.py +0 -0
- omlish/http/coro/client/client.py +795 -0
- omlish/http/coro/client/errors.py +57 -0
- omlish/http/coro/client/headers.py +99 -0
- omlish/http/coro/client/io.py +50 -0
- omlish/http/coro/client/response.py +445 -0
- omlish/http/coro/client/status.py +86 -0
- omlish/http/coro/client/validation.py +109 -0
- omlish/http/coro/server/__init__.py +0 -0
- omlish/http/coro/{fdio.py → server/fdio.py} +6 -6
- omlish/http/coro/{server.py → server/server.py} +13 -13
- omlish/http/coro/{simple.py → server/simple.py} +16 -16
- omlish/http/coro/{sockets.py → server/sockets.py} +4 -4
- {omlish-0.0.0.dev368.dist-info → omlish-0.0.0.dev369.dist-info}/METADATA +1 -1
- {omlish-0.0.0.dev368.dist-info → omlish-0.0.0.dev369.dist-info}/RECORD +20 -11
- {omlish-0.0.0.dev368.dist-info → omlish-0.0.0.dev369.dist-info}/WHEEL +0 -0
- {omlish-0.0.0.dev368.dist-info → omlish-0.0.0.dev369.dist-info}/entry_points.txt +0 -0
- {omlish-0.0.0.dev368.dist-info → omlish-0.0.0.dev369.dist-info}/licenses/LICENSE +0 -0
- {omlish-0.0.0.dev368.dist-info → omlish-0.0.0.dev369.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,57 @@
|
|
1
|
+
# @omlish-lite
|
2
|
+
# ruff: noqa: UP006 UP007 UP043 UP045
|
3
|
+
import dataclasses as dc
|
4
|
+
import enum
|
5
|
+
import typing as ta
|
6
|
+
|
7
|
+
|
8
|
+
##
|
9
|
+
|
10
|
+
|
11
|
+
class CoroHttpClientErrors:
|
12
|
+
class ClientError(Exception):
|
13
|
+
pass
|
14
|
+
|
15
|
+
class NotConnectedError(ClientError):
|
16
|
+
pass
|
17
|
+
|
18
|
+
class InvalidUrlError(ClientError):
|
19
|
+
pass
|
20
|
+
|
21
|
+
@dc.dataclass()
|
22
|
+
class UnknownProtocolError(ClientError):
|
23
|
+
version: str
|
24
|
+
|
25
|
+
@dc.dataclass()
|
26
|
+
class IncompleteReadError(ClientError):
|
27
|
+
partial: bytes
|
28
|
+
expected: ta.Optional[int] = None
|
29
|
+
|
30
|
+
class ImproperConnectionStateError(ClientError):
|
31
|
+
pass
|
32
|
+
|
33
|
+
class CannotSendRequestError(ImproperConnectionStateError):
|
34
|
+
pass
|
35
|
+
|
36
|
+
class CannotSendHeaderError(ImproperConnectionStateError):
|
37
|
+
pass
|
38
|
+
|
39
|
+
class ResponseNotReadyError(ImproperConnectionStateError):
|
40
|
+
pass
|
41
|
+
|
42
|
+
@dc.dataclass()
|
43
|
+
class BadStatusLineError(ClientError):
|
44
|
+
line: str
|
45
|
+
|
46
|
+
class RemoteDisconnectedError(BadStatusLineError, ConnectionResetError):
|
47
|
+
pass
|
48
|
+
|
49
|
+
@dc.dataclass()
|
50
|
+
class LineTooLongError(ClientError):
|
51
|
+
class LineType(enum.Enum):
|
52
|
+
STATUS = enum.auto()
|
53
|
+
HEADER = enum.auto()
|
54
|
+
CHUNK_SIZE = enum.auto()
|
55
|
+
TRAILER = enum.auto()
|
56
|
+
|
57
|
+
line_type: LineType
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# @omlish-lite
|
2
|
+
# ruff: noqa: UP006 UP007 UP043 UP045
|
3
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
4
|
+
# --------------------------------------------
|
5
|
+
#
|
6
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
7
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
8
|
+
# documentation.
|
9
|
+
#
|
10
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
11
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
12
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
13
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
14
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
|
15
|
+
# alone or in any derivative version prepared by Licensee.
|
16
|
+
#
|
17
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
18
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
19
|
+
# any such work a brief summary of the changes made to Python.
|
20
|
+
#
|
21
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
22
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
23
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
24
|
+
# RIGHTS.
|
25
|
+
#
|
26
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
27
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
28
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
29
|
+
#
|
30
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
31
|
+
#
|
32
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
33
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
34
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
35
|
+
#
|
36
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
37
|
+
# License Agreement.
|
38
|
+
import email.message
|
39
|
+
import email.parser
|
40
|
+
import typing as ta
|
41
|
+
|
42
|
+
from ....lite.check import check
|
43
|
+
from .errors import CoroHttpClientErrors
|
44
|
+
from .io import CoroHttpClientIo
|
45
|
+
|
46
|
+
|
47
|
+
##
|
48
|
+
|
49
|
+
|
50
|
+
class CoroHttpClientHeaders:
|
51
|
+
def __new__(cls, *args, **kwargs): # noqa
|
52
|
+
raise TypeError
|
53
|
+
|
54
|
+
#
|
55
|
+
|
56
|
+
MAX_HEADERS: ta.ClassVar[int] = 100
|
57
|
+
|
58
|
+
@classmethod
|
59
|
+
def read_headers(cls) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], ta.List[bytes]]:
|
60
|
+
"""
|
61
|
+
Reads potential header lines into a list from a file pointer.
|
62
|
+
|
63
|
+
Length of line is limited by MAX_LINE, and number of headers is limited by MAX_HEADERS.
|
64
|
+
"""
|
65
|
+
|
66
|
+
headers = []
|
67
|
+
while True:
|
68
|
+
line = check.isinstance((yield CoroHttpClientIo.ReadLineIo(CoroHttpClientIo.MAX_LINE + 1)), bytes)
|
69
|
+
if len(line) > CoroHttpClientIo.MAX_LINE:
|
70
|
+
raise CoroHttpClientErrors.LineTooLongError(CoroHttpClientErrors.LineTooLongError.LineType.HEADER)
|
71
|
+
|
72
|
+
headers.append(line)
|
73
|
+
if len(headers) > cls.MAX_HEADERS:
|
74
|
+
raise CoroHttpClientErrors.ClientError(f'got more than {cls.MAX_HEADERS} headers')
|
75
|
+
|
76
|
+
if line in (b'\r\n', b'\n', b''):
|
77
|
+
break
|
78
|
+
|
79
|
+
return headers
|
80
|
+
|
81
|
+
@classmethod
|
82
|
+
def parse_header_lines(cls, header_lines: ta.Sequence[bytes]) -> email.message.Message:
|
83
|
+
"""
|
84
|
+
Parses only RFC2822 headers from header lines.
|
85
|
+
|
86
|
+
email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many
|
87
|
+
bytes from the stream, bytes which we later need to read as bytes. So we read the correct bytes here, as bytes,
|
88
|
+
for email Parser to parse.
|
89
|
+
"""
|
90
|
+
|
91
|
+
text = b''.join(header_lines).decode('iso-8859-1')
|
92
|
+
return email.parser.Parser().parsestr(text)
|
93
|
+
|
94
|
+
@classmethod
|
95
|
+
def parse_headers(cls) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], email.message.Message]:
|
96
|
+
"""Parses only RFC2822 headers from a file pointer."""
|
97
|
+
|
98
|
+
headers = yield from cls.read_headers()
|
99
|
+
return cls.parse_header_lines(headers)
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# @omlish-lite
|
2
|
+
# ruff: noqa: UP006 UP007 UP043 UP045
|
3
|
+
import abc
|
4
|
+
import dataclasses as dc
|
5
|
+
import typing as ta
|
6
|
+
|
7
|
+
|
8
|
+
##
|
9
|
+
|
10
|
+
|
11
|
+
class CoroHttpClientIo:
|
12
|
+
MAX_LINE: ta.ClassVar[int] = 65536
|
13
|
+
|
14
|
+
#
|
15
|
+
|
16
|
+
class Io(abc.ABC): # noqa
|
17
|
+
pass
|
18
|
+
|
19
|
+
#
|
20
|
+
|
21
|
+
@dc.dataclass(frozen=True)
|
22
|
+
class ConnectIo(Io):
|
23
|
+
args: ta.Tuple[ta.Any, ...]
|
24
|
+
kwargs: ta.Optional[ta.Dict[str, ta.Any]] = None
|
25
|
+
|
26
|
+
class CloseIo(Io):
|
27
|
+
pass
|
28
|
+
|
29
|
+
#
|
30
|
+
|
31
|
+
class AnyReadIo(Io): # noqa
|
32
|
+
pass
|
33
|
+
|
34
|
+
@dc.dataclass(frozen=True)
|
35
|
+
class ReadIo(AnyReadIo):
|
36
|
+
sz: ta.Optional[int]
|
37
|
+
|
38
|
+
@dc.dataclass(frozen=True)
|
39
|
+
class ReadLineIo(AnyReadIo):
|
40
|
+
sz: int
|
41
|
+
|
42
|
+
@dc.dataclass(frozen=True)
|
43
|
+
class PeekIo(AnyReadIo):
|
44
|
+
sz: int
|
45
|
+
|
46
|
+
#
|
47
|
+
|
48
|
+
@dc.dataclass(frozen=True)
|
49
|
+
class WriteIo(Io):
|
50
|
+
data: bytes
|
@@ -0,0 +1,445 @@
|
|
1
|
+
# @omlish-lite
|
2
|
+
# ruff: noqa: UP006 UP007 UP043 UP045
|
3
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
4
|
+
# --------------------------------------------
|
5
|
+
#
|
6
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
7
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
8
|
+
# documentation.
|
9
|
+
#
|
10
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
11
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
12
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
13
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
14
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
|
15
|
+
# alone or in any derivative version prepared by Licensee.
|
16
|
+
#
|
17
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
18
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
19
|
+
# any such work a brief summary of the changes made to Python.
|
20
|
+
#
|
21
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
22
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
23
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
24
|
+
# RIGHTS.
|
25
|
+
#
|
26
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
27
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
28
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
29
|
+
#
|
30
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
31
|
+
#
|
32
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
33
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
34
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
35
|
+
#
|
36
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
37
|
+
# License Agreement.
|
38
|
+
import email.parser
|
39
|
+
import http
|
40
|
+
import typing as ta
|
41
|
+
|
42
|
+
from ....lite.check import check
|
43
|
+
from .errors import CoroHttpClientErrors
|
44
|
+
from .headers import CoroHttpClientHeaders
|
45
|
+
from .io import CoroHttpClientIo
|
46
|
+
from .status import CoroHttpClientStatusLine
|
47
|
+
|
48
|
+
|
49
|
+
##
|
50
|
+
|
51
|
+
|
52
|
+
class CoroHttpClientResponse(
|
53
|
+
CoroHttpClientErrors,
|
54
|
+
CoroHttpClientIo,
|
55
|
+
):
|
56
|
+
# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
|
57
|
+
|
58
|
+
# The bytes from the socket object are iso-8859-1 strings. See RFC 2616 sec 2.2 which notes an exception for
|
59
|
+
# MIME-encoded text following RFC 2047. The basic status line parsing only accepts iso-8859-1.
|
60
|
+
|
61
|
+
def __init__(self, method: str) -> None:
|
62
|
+
super().__init__()
|
63
|
+
|
64
|
+
state = self._State()
|
65
|
+
state.method = method
|
66
|
+
self._state = state
|
67
|
+
|
68
|
+
class _State:
|
69
|
+
closed: bool = False
|
70
|
+
|
71
|
+
# If the response includes a content-length header, we need to make sure that the client doesn't read more than
|
72
|
+
# the specified number of bytes. If it does, it will block until the server times out and closes the connection.
|
73
|
+
# This will happen if a read is done (without a size) whether self.fp is buffered or not. So, no read by clients
|
74
|
+
# unless they know what they are doing.
|
75
|
+
method: str
|
76
|
+
|
77
|
+
# The HttpResponse object is returned via urllib. The clients of http and urllib expect different attributes for
|
78
|
+
# the headers. headers is used here and supports urllib. msg is provided as a backwards compatibility layer for
|
79
|
+
# http clients.
|
80
|
+
headers: email.message.Message
|
81
|
+
|
82
|
+
# From the Status-Line of the response
|
83
|
+
version: int # HTTP-Version
|
84
|
+
status: int # Status-Code
|
85
|
+
reason: str # Reason-Phrase
|
86
|
+
|
87
|
+
chunked: bool # Is "chunked" being used?
|
88
|
+
chunk_left: ta.Optional[int] # Bytes left to read in current chunk
|
89
|
+
length: ta.Optional[int] # Number of bytes left in response
|
90
|
+
will_close: bool # Conn will close at end of response
|
91
|
+
|
92
|
+
#
|
93
|
+
|
94
|
+
def _read_status(self) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], CoroHttpClientStatusLine]:
|
95
|
+
try:
|
96
|
+
return (yield from CoroHttpClientStatusLine.read())
|
97
|
+
except self.BadStatusLineError:
|
98
|
+
self._close_conn()
|
99
|
+
raise
|
100
|
+
|
101
|
+
def _begin(self) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], None]:
|
102
|
+
state = self._state
|
103
|
+
|
104
|
+
check.state(not hasattr(state, 'headers'))
|
105
|
+
|
106
|
+
# Read until we get a non-100 response
|
107
|
+
while True:
|
108
|
+
version, status, reason = yield from self._read_status()
|
109
|
+
if status != http.HTTPStatus.CONTINUE:
|
110
|
+
break
|
111
|
+
|
112
|
+
# Skip the header from the 100 response
|
113
|
+
skipped_headers = yield from CoroHttpClientHeaders.read_headers() # noqa
|
114
|
+
|
115
|
+
del skipped_headers
|
116
|
+
|
117
|
+
state.status = status
|
118
|
+
state.reason = reason.strip()
|
119
|
+
if version in ('HTTP/1.0', 'HTTP/0.9'):
|
120
|
+
# Some servers might still return '0.9', treat it as 1.0 anyway
|
121
|
+
state.version = 10
|
122
|
+
elif version.startswith('HTTP/1.'):
|
123
|
+
# Use HTTP/1.1 code for HTTP/1.x where x>=1
|
124
|
+
state.version = 11
|
125
|
+
else:
|
126
|
+
raise self.UnknownProtocolError(version)
|
127
|
+
|
128
|
+
state.headers = yield from CoroHttpClientHeaders.parse_headers()
|
129
|
+
|
130
|
+
# Are we using the chunked-style of transfer encoding?
|
131
|
+
tr_enc = state.headers.get('transfer-encoding')
|
132
|
+
if tr_enc and tr_enc.lower() == 'chunked':
|
133
|
+
state.chunked = True
|
134
|
+
state.chunk_left = None
|
135
|
+
else:
|
136
|
+
state.chunked = False
|
137
|
+
|
138
|
+
# Will the connection close at the end of the response?
|
139
|
+
state.will_close = self._check_close()
|
140
|
+
|
141
|
+
# Do we have a Content-Length?
|
142
|
+
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is 'chunked'
|
143
|
+
state.length = None
|
144
|
+
length = state.headers.get('content-length')
|
145
|
+
if length and not state.chunked:
|
146
|
+
try:
|
147
|
+
state.length = int(length)
|
148
|
+
except ValueError:
|
149
|
+
state.length = None
|
150
|
+
else:
|
151
|
+
if state.length < 0: # Ignore nonsensical negative lengths
|
152
|
+
state.length = None
|
153
|
+
else:
|
154
|
+
state.length = None
|
155
|
+
|
156
|
+
# Does the body have a fixed length? (of zero)
|
157
|
+
if (
|
158
|
+
status in (http.HTTPStatus.NO_CONTENT, http.HTTPStatus.NOT_MODIFIED) or
|
159
|
+
100 <= status < 200 or # 1xx codes
|
160
|
+
state.method == 'HEAD'
|
161
|
+
):
|
162
|
+
state.length = 0
|
163
|
+
|
164
|
+
# If the connection remains open, and we aren't using chunked, and a content-length was not provided, then
|
165
|
+
# assume that the connection WILL close.
|
166
|
+
if (
|
167
|
+
not state.will_close and
|
168
|
+
not state.chunked and
|
169
|
+
state.length is None
|
170
|
+
):
|
171
|
+
state.will_close = True
|
172
|
+
|
173
|
+
#
|
174
|
+
|
175
|
+
def _check_close(self) -> bool:
|
176
|
+
conn = self._state.headers.get('connection')
|
177
|
+
if getattr(self._state, 'version', None) == 11:
|
178
|
+
# An HTTP/1.1 proxy is assumed to stay open unless explicitly closed.
|
179
|
+
if conn and 'close' in conn.lower():
|
180
|
+
return True
|
181
|
+
return False
|
182
|
+
|
183
|
+
# Some HTTP/1.0 implementations have support for persistent connections, using rules different than HTTP/1.1.
|
184
|
+
|
185
|
+
# For older HTTP, Keep-Alive indicates persistent connection.
|
186
|
+
if self._state.headers.get('keep-alive'):
|
187
|
+
return False
|
188
|
+
|
189
|
+
# At least Akamai returns a 'Connection: Keep-Alive' header, which was supposed to be sent by the client.
|
190
|
+
if conn and 'keep-alive' in conn.lower():
|
191
|
+
return False
|
192
|
+
|
193
|
+
# Proxy-Connection is a netscape hack.
|
194
|
+
pconn = self._state.headers.get('proxy-connection')
|
195
|
+
if pconn and 'keep-alive' in pconn.lower():
|
196
|
+
return False
|
197
|
+
|
198
|
+
# Otherwise, assume it will close
|
199
|
+
return True
|
200
|
+
|
201
|
+
def _close_conn(self) -> None:
|
202
|
+
self._state.closed = True
|
203
|
+
|
204
|
+
def close(self) -> None:
|
205
|
+
if not self._state.closed:
|
206
|
+
self._close_conn()
|
207
|
+
|
208
|
+
def is_closed(self) -> bool:
|
209
|
+
"""True if the connection is closed."""
|
210
|
+
|
211
|
+
# NOTE: it is possible that we will not ever call self.close(). This case occurs when will_close is TRUE, length
|
212
|
+
# is None, and we read up to the last byte, but NOT past it.
|
213
|
+
#
|
214
|
+
# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be called, meaning self.is_closed() is
|
215
|
+
# meaningful.
|
216
|
+
return self._state.closed
|
217
|
+
|
218
|
+
#
|
219
|
+
|
220
|
+
def read(self, amt: ta.Optional[int] = None) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
221
|
+
"""Read and return the response body, or up to the next amt bytes."""
|
222
|
+
|
223
|
+
if self._state.closed:
|
224
|
+
return b''
|
225
|
+
|
226
|
+
if self._state.method == 'HEAD':
|
227
|
+
self._close_conn()
|
228
|
+
return b''
|
229
|
+
|
230
|
+
if self._state.chunked:
|
231
|
+
return (yield from self._read_chunked(amt))
|
232
|
+
|
233
|
+
if amt is not None:
|
234
|
+
if self._state.length is not None and amt > self._state.length:
|
235
|
+
# Clip the read to the "end of response"
|
236
|
+
amt = self._state.length
|
237
|
+
|
238
|
+
s = check.isinstance((yield self.ReadIo(amt)), bytes)
|
239
|
+
|
240
|
+
if not s and amt:
|
241
|
+
# Ideally, we would raise IncompleteRead if the content-length wasn't satisfied, but it might break
|
242
|
+
# compatibility.
|
243
|
+
self._close_conn()
|
244
|
+
|
245
|
+
elif self._state.length is not None:
|
246
|
+
self._state.length -= len(s)
|
247
|
+
if not self._state.length:
|
248
|
+
self._close_conn()
|
249
|
+
|
250
|
+
return s
|
251
|
+
|
252
|
+
else:
|
253
|
+
# Amount is not given (unbounded read) so we must check self.length
|
254
|
+
if self._state.length is None:
|
255
|
+
s = check.isinstance((yield self.ReadIo(None)), bytes)
|
256
|
+
|
257
|
+
else:
|
258
|
+
try:
|
259
|
+
s = yield from self._safe_read(self._state.length)
|
260
|
+
except self.IncompleteReadError:
|
261
|
+
self._close_conn()
|
262
|
+
raise
|
263
|
+
|
264
|
+
self._state.length = 0
|
265
|
+
|
266
|
+
self._close_conn() # We read everything
|
267
|
+
return s
|
268
|
+
|
269
|
+
def _read_next_chunk_size(self) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], int]:
|
270
|
+
# Read the next chunk size from the file
|
271
|
+
line = check.isinstance((yield self.ReadLineIo(self.MAX_LINE + 1)), bytes)
|
272
|
+
if len(line) > self.MAX_LINE:
|
273
|
+
raise self.LineTooLongError(self.LineTooLongError.LineType.CHUNK_SIZE)
|
274
|
+
|
275
|
+
i = line.find(b';')
|
276
|
+
if i >= 0:
|
277
|
+
line = line[:i] # Strip chunk-extensions
|
278
|
+
|
279
|
+
try:
|
280
|
+
return int(line, 16)
|
281
|
+
except ValueError:
|
282
|
+
# Close the connection as protocol synchronisation is probably lost
|
283
|
+
self._close_conn()
|
284
|
+
raise
|
285
|
+
|
286
|
+
def _read_and_discard_trailer(self) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], None]:
|
287
|
+
# Read and discard trailer up to the CRLF terminator
|
288
|
+
# NOTE: we shouldn't have any trailers!
|
289
|
+
while True:
|
290
|
+
line = check.isinstance((yield self.ReadLineIo(self.MAX_LINE + 1)), bytes)
|
291
|
+
if len(line) > self.MAX_LINE:
|
292
|
+
raise self.LineTooLongError(self.LineTooLongError.LineType.TRAILER)
|
293
|
+
|
294
|
+
if not line:
|
295
|
+
# A vanishingly small number of sites EOF without sending the trailer
|
296
|
+
break
|
297
|
+
|
298
|
+
if line in (b'\r\n', b'\n', b''):
|
299
|
+
break
|
300
|
+
|
301
|
+
def _get_chunk_left(self) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], ta.Optional[int]]:
|
302
|
+
# Return self.chunk_left, reading a new chunk if necessary. chunk_left == 0: at the end of the current chunk,
|
303
|
+
# need to close it chunk_left == None: No current chunk, should read next. This function returns non-zero or
|
304
|
+
# None if the last chunk has been read.
|
305
|
+
chunk_left = self._state.chunk_left
|
306
|
+
if not chunk_left: # Can be 0 or None
|
307
|
+
if chunk_left is not None:
|
308
|
+
# We are at the end of chunk, discard chunk end
|
309
|
+
yield from self._safe_read(2) # Toss the CRLF at the end of the chunk
|
310
|
+
|
311
|
+
try:
|
312
|
+
chunk_left = yield from self._read_next_chunk_size()
|
313
|
+
except ValueError:
|
314
|
+
raise self.IncompleteReadError(b'') from None
|
315
|
+
|
316
|
+
if chunk_left == 0:
|
317
|
+
# Last chunk: 1*('0') [ chunk-extension ] CRLF
|
318
|
+
yield from self._read_and_discard_trailer()
|
319
|
+
|
320
|
+
# We read everything; close the 'file'
|
321
|
+
self._close_conn()
|
322
|
+
|
323
|
+
chunk_left = None
|
324
|
+
|
325
|
+
self._state.chunk_left = chunk_left
|
326
|
+
|
327
|
+
return chunk_left
|
328
|
+
|
329
|
+
def _read_chunked(
|
330
|
+
self,
|
331
|
+
amt: ta.Optional[int] = None,
|
332
|
+
) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
333
|
+
check.state(hasattr(self._state, 'chunked'))
|
334
|
+
value = []
|
335
|
+
try:
|
336
|
+
while (chunk_left := (yield from self._get_chunk_left())) is not None:
|
337
|
+
if amt is not None and amt <= chunk_left:
|
338
|
+
value.append((yield from self._safe_read(amt)))
|
339
|
+
self._state.chunk_left = chunk_left - amt
|
340
|
+
break
|
341
|
+
|
342
|
+
value.append((yield from self._safe_read(chunk_left)))
|
343
|
+
if amt is not None:
|
344
|
+
amt -= chunk_left
|
345
|
+
self._state.chunk_left = 0
|
346
|
+
|
347
|
+
return b''.join(value)
|
348
|
+
|
349
|
+
except self.IncompleteReadError as exc:
|
350
|
+
raise self.IncompleteReadError(b''.join(value)) from exc
|
351
|
+
|
352
|
+
def _safe_read(self, amt: int) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
353
|
+
"""
|
354
|
+
Read the number of bytes requested.
|
355
|
+
|
356
|
+
This function should be used when <amt> bytes "should" be present for reading. If the bytes are truly not
|
357
|
+
available (due to EOF), then the IncompleteRead exception can be used to detect the problem.
|
358
|
+
"""
|
359
|
+
|
360
|
+
data = check.isinstance((yield self.ReadIo(amt)), bytes)
|
361
|
+
if len(data) < amt:
|
362
|
+
raise self.IncompleteReadError(data, amt - len(data))
|
363
|
+
return data
|
364
|
+
|
365
|
+
def peek(self, n: int = -1) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
366
|
+
# Having this enables IOBase.readline() to read more than one byte at a time
|
367
|
+
if self._state.closed or self._state.method == 'HEAD':
|
368
|
+
return b''
|
369
|
+
|
370
|
+
if self._state.chunked:
|
371
|
+
return (yield from self._peek_chunked(n))
|
372
|
+
|
373
|
+
return check.isinstance((yield self.PeekIo(n)), bytes)
|
374
|
+
|
375
|
+
def _readline(
|
376
|
+
self,
|
377
|
+
size: ta.Optional[int] = -1,
|
378
|
+
) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
379
|
+
if size is None:
|
380
|
+
size = -1
|
381
|
+
else:
|
382
|
+
try:
|
383
|
+
size_index = size.__index__
|
384
|
+
except AttributeError:
|
385
|
+
raise TypeError(f'{size!r} is not an integer') from None
|
386
|
+
else:
|
387
|
+
size = size_index()
|
388
|
+
|
389
|
+
# For backwards compatibility, a (slowish) readline().
|
390
|
+
def nreadahead():
|
391
|
+
readahead = yield from self.peek(1)
|
392
|
+
if not readahead:
|
393
|
+
return 1
|
394
|
+
n = (readahead.find(b'\n') + 1) or len(readahead)
|
395
|
+
if size >= 0:
|
396
|
+
n = min(n, size)
|
397
|
+
return n
|
398
|
+
|
399
|
+
res = bytearray()
|
400
|
+
while size < 0 or len(res) < size:
|
401
|
+
b = (yield from self.read((yield from nreadahead())))
|
402
|
+
if not b:
|
403
|
+
break
|
404
|
+
res += b
|
405
|
+
if res.endswith(b'\n'):
|
406
|
+
break
|
407
|
+
|
408
|
+
return bytes(res)
|
409
|
+
|
410
|
+
def readline(self, limit: int = -1) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
411
|
+
if self._state.closed or self._state.method == 'HEAD':
|
412
|
+
return b''
|
413
|
+
|
414
|
+
if self._state.chunked:
|
415
|
+
# Fallback to IOBase readline which uses peek() and read()
|
416
|
+
return (yield from self._readline(limit))
|
417
|
+
|
418
|
+
if self._state.length is not None and (limit < 0 or limit > self._state.length):
|
419
|
+
limit = self._state.length
|
420
|
+
|
421
|
+
result = check.isinstance((yield self.ReadLineIo(limit)), bytes)
|
422
|
+
|
423
|
+
if not result and limit:
|
424
|
+
self._close_conn()
|
425
|
+
|
426
|
+
elif self._state.length is not None:
|
427
|
+
self._state.length -= len(result)
|
428
|
+
if not self._state.length:
|
429
|
+
self._close_conn()
|
430
|
+
|
431
|
+
return result
|
432
|
+
|
433
|
+
def _peek_chunked(self, n: int) -> ta.Generator[CoroHttpClientIo.Io, ta.Optional[bytes], bytes]:
|
434
|
+
# Strictly speaking, _get_chunk_left() may cause more than one read, but that is ok, since that is to satisfy
|
435
|
+
# the chunked protocol.
|
436
|
+
try:
|
437
|
+
chunk_left = yield from self._get_chunk_left()
|
438
|
+
except self.IncompleteReadError:
|
439
|
+
return b'' # Peek doesn't worry about protocol
|
440
|
+
|
441
|
+
if chunk_left is None:
|
442
|
+
return b'' # Eof
|
443
|
+
|
444
|
+
# Peek is allowed to return more than requested. Just request the entire chunk, and truncate what we get.
|
445
|
+
return check.isinstance((yield self.PeekIo(chunk_left)), bytes)[:chunk_left]
|