hip 0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hip/__init__.py +56 -0
- hip/__version__.py +2 -0
- hip/_body.py +73 -0
- hip/_client.py +180 -0
- hip/_connection.py +181 -0
- hip/_content.py +557 -0
- hip/_headers.py +251 -0
- hip/_network.py +252 -0
- hip/_parsers.py +531 -0
- hip/_pool.py +102 -0
- hip/_request.py +124 -0
- hip/_response.py +207 -0
- hip/_server.py +123 -0
- hip/_streams.py +199 -0
- hip/_urlencode.py +76 -0
- hip/_urlparse.py +533 -0
- hip/_urls.py +541 -0
- hip-0.2.dist-info/METADATA +111 -0
- hip-0.2.dist-info/RECORD +20 -0
- hip-0.2.dist-info/WHEEL +4 -0
hip/_parsers.py
ADDED
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
import enum
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
from ._streams import Stream
|
|
5
|
+
|
|
6
|
+
__all__ = ["HTTPParser", "Mode", "ProtocolError"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# TODO...
|
|
10
|
+
|
|
11
|
+
# * Upgrade
|
|
12
|
+
# * CONNECT
|
|
13
|
+
|
|
14
|
+
# * Support 'Expect: 100 Continue'
|
|
15
|
+
# * Add 'Error' state transitions
|
|
16
|
+
# * Add tests to trickle data
|
|
17
|
+
# * Add type annotations
|
|
18
|
+
|
|
19
|
+
# * Optional... HTTP/1.0 support
|
|
20
|
+
# * Read trailing headers on Transfer-Encoding: chunked. Not just '\r\n'.
|
|
21
|
+
# * When writing Transfer-Encoding: chunked, split large writes into buffer size.
|
|
22
|
+
# * When reading Transfer-Encoding: chunked, handle incomplete reads from large chunk sizes.
|
|
23
|
+
# * .read() doesn't document if will always return maximum available.
|
|
24
|
+
|
|
25
|
+
# * validate method, target, protocol in request line
|
|
26
|
+
# * validate protocol, status_code, reason_phrase in response line
|
|
27
|
+
# * validate name, value on headers
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class State(enum.Enum):
|
|
31
|
+
WAIT = 0
|
|
32
|
+
SEND_METHOD_LINE = 1
|
|
33
|
+
SEND_STATUS_LINE = 2
|
|
34
|
+
SEND_HEADERS = 3
|
|
35
|
+
SEND_BODY = 4
|
|
36
|
+
RECV_METHOD_LINE = 5
|
|
37
|
+
RECV_STATUS_LINE = 6
|
|
38
|
+
RECV_HEADERS = 7
|
|
39
|
+
RECV_BODY = 8
|
|
40
|
+
DONE = 9
|
|
41
|
+
CLOSED = 10
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Mode(enum.Enum):
|
|
45
|
+
CLIENT = 0
|
|
46
|
+
SERVER = 1
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# The usual transitions will be...
|
|
50
|
+
|
|
51
|
+
# IDLE, IDLE
|
|
52
|
+
# SEND_HEADERS, IDLE
|
|
53
|
+
# SEND_BODY, IDLE
|
|
54
|
+
# DONE, IDLE
|
|
55
|
+
# DONE, SEND_HEADERS
|
|
56
|
+
# DONE, SEND_BODY
|
|
57
|
+
# DONE, DONE
|
|
58
|
+
|
|
59
|
+
# Then either back to IDLE, IDLE
|
|
60
|
+
# or move to CLOSED, CLOSED
|
|
61
|
+
|
|
62
|
+
# 1. It is also valid for the server to start
|
|
63
|
+
# sending the response without waiting for the
|
|
64
|
+
# complete request.
|
|
65
|
+
# 2. 1xx status codes are interim states, and
|
|
66
|
+
# transition from SEND_HEADERS back to IDLE
|
|
67
|
+
# 3. ...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ProtocolError(Exception):
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class HTTPParser:
|
|
75
|
+
"""
|
|
76
|
+
Usage...
|
|
77
|
+
|
|
78
|
+
client = HTTPParser(writer, reader)
|
|
79
|
+
client.send_method_line()
|
|
80
|
+
client.send_headers()
|
|
81
|
+
client.send_body()
|
|
82
|
+
client.recv_status_line()
|
|
83
|
+
client.recv_headers()
|
|
84
|
+
client.recv_body()
|
|
85
|
+
client.complete()
|
|
86
|
+
client.close()
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def __init__(self, stream: Stream, mode: str) -> None:
|
|
90
|
+
self.stream = stream
|
|
91
|
+
self.parser = ReadAheadParser(stream)
|
|
92
|
+
self.mode = {"CLIENT": Mode.CLIENT, "SERVER": Mode.SERVER}[mode]
|
|
93
|
+
self.keepalive_duration = 5.0
|
|
94
|
+
|
|
95
|
+
# Track state...
|
|
96
|
+
if self.mode == Mode.CLIENT:
|
|
97
|
+
self.send_state: State = State.SEND_METHOD_LINE
|
|
98
|
+
self.recv_state: State = State.WAIT
|
|
99
|
+
else:
|
|
100
|
+
self.recv_state = State.RECV_METHOD_LINE
|
|
101
|
+
self.send_state = State.WAIT
|
|
102
|
+
|
|
103
|
+
# Track message framing...
|
|
104
|
+
self.send_content_length: int | None = 0
|
|
105
|
+
self.recv_content_length: int | None = 0
|
|
106
|
+
self.send_seen_length = 0
|
|
107
|
+
self.recv_seen_length = 0
|
|
108
|
+
|
|
109
|
+
# Track connection keep alive...
|
|
110
|
+
self.send_keep_alive = True
|
|
111
|
+
self.recv_keep_alive = True
|
|
112
|
+
self.keepalive_until: float | None = None
|
|
113
|
+
|
|
114
|
+
# Special states...
|
|
115
|
+
self.processing_1xx = False
|
|
116
|
+
|
|
117
|
+
def send_method_line(self, method: bytes, target: bytes, protocol: bytes) -> None:
|
|
118
|
+
"""
|
|
119
|
+
Send the initial request line:
|
|
120
|
+
|
|
121
|
+
>>> p.send_method_line(b'GET', b'/', b'HTTP/1.1')
|
|
122
|
+
|
|
123
|
+
Sending state will switch to SEND_HEADERS state.
|
|
124
|
+
"""
|
|
125
|
+
# Scrub connection keepalive
|
|
126
|
+
self.keepalive_until = None
|
|
127
|
+
|
|
128
|
+
if self.send_state != State.SEND_METHOD_LINE:
|
|
129
|
+
msg = f"Called 'send_method_line' in invalid state {self.send_state}"
|
|
130
|
+
raise ProtocolError(msg)
|
|
131
|
+
|
|
132
|
+
# Send initial request line, eg. "GET / HTTP/1.1"
|
|
133
|
+
if protocol != b"HTTP/1.1":
|
|
134
|
+
raise ProtocolError("Sent unsupported protocol version")
|
|
135
|
+
data = b" ".join([method, target, protocol]) + b"\r\n"
|
|
136
|
+
self.stream.write(data)
|
|
137
|
+
|
|
138
|
+
self.send_state = State.SEND_HEADERS
|
|
139
|
+
self.recv_state = State.RECV_STATUS_LINE
|
|
140
|
+
|
|
141
|
+
def send_status_line(
|
|
142
|
+
self, protocol: bytes, status_code: int, reason: bytes
|
|
143
|
+
) -> None:
|
|
144
|
+
"""
|
|
145
|
+
Send the initial response line:
|
|
146
|
+
|
|
147
|
+
>>> p.send_method_line(b'HTTP/1.1', 200, b'OK')
|
|
148
|
+
|
|
149
|
+
Sending state will switch to SEND_HEADERS state.
|
|
150
|
+
"""
|
|
151
|
+
if self.send_state != State.SEND_STATUS_LINE:
|
|
152
|
+
msg = f"Called 'send_status_line' in invalid state {self.send_state}"
|
|
153
|
+
raise ProtocolError(msg)
|
|
154
|
+
|
|
155
|
+
# Send initial request line, eg. "GET / HTTP/1.1"
|
|
156
|
+
if protocol != b"HTTP/1.1":
|
|
157
|
+
raise ProtocolError("Sent unsupported protocol version")
|
|
158
|
+
status_code_bytes = str(status_code).encode("ascii")
|
|
159
|
+
data = b" ".join([protocol, status_code_bytes, reason]) + b"\r\n"
|
|
160
|
+
self.stream.write(data)
|
|
161
|
+
|
|
162
|
+
self.send_state = State.SEND_HEADERS
|
|
163
|
+
|
|
164
|
+
def send_headers(self, headers: list[tuple[bytes, bytes]]) -> None:
|
|
165
|
+
"""
|
|
166
|
+
Send the request headers:
|
|
167
|
+
|
|
168
|
+
>>> p.send_headers([(b'Host', b'www.example.com')])
|
|
169
|
+
|
|
170
|
+
Sending state will switch to SEND_BODY state.
|
|
171
|
+
"""
|
|
172
|
+
if self.send_state != State.SEND_HEADERS:
|
|
173
|
+
msg = f"Called 'send_headers' in invalid state {self.send_state}"
|
|
174
|
+
raise ProtocolError(msg)
|
|
175
|
+
|
|
176
|
+
# Update header state
|
|
177
|
+
seen_host = False
|
|
178
|
+
for name, value in headers:
|
|
179
|
+
lname = name.lower()
|
|
180
|
+
if lname == b"host":
|
|
181
|
+
seen_host = True
|
|
182
|
+
elif lname == b"content-length":
|
|
183
|
+
self.send_content_length = bounded_int(
|
|
184
|
+
value, max_digits=20, exc_text="Sent invalid Content-Length"
|
|
185
|
+
)
|
|
186
|
+
elif lname == b"connection" and value == b"close":
|
|
187
|
+
self.send_keep_alive = False
|
|
188
|
+
elif lname == b"transfer-encoding" and value == b"chunked":
|
|
189
|
+
self.send_content_length = None
|
|
190
|
+
|
|
191
|
+
if self.mode == Mode.CLIENT and not seen_host:
|
|
192
|
+
raise ProtocolError("Request missing 'Host' header")
|
|
193
|
+
|
|
194
|
+
# Send request headers
|
|
195
|
+
lines = [name + b": " + value + b"\r\n" for name, value in headers]
|
|
196
|
+
data = b"".join(lines) + b"\r\n"
|
|
197
|
+
self.stream.write(data)
|
|
198
|
+
|
|
199
|
+
self.send_state = State.SEND_BODY
|
|
200
|
+
|
|
201
|
+
def send_body(self, body: bytes) -> None:
|
|
202
|
+
"""
|
|
203
|
+
Send the request body. An empty bytes argument indicates the end of the stream:
|
|
204
|
+
|
|
205
|
+
>>> p.send_body(b'')
|
|
206
|
+
|
|
207
|
+
Sending state will switch to DONE.
|
|
208
|
+
"""
|
|
209
|
+
if self.send_state != State.SEND_BODY:
|
|
210
|
+
msg = f"Called 'send_body' in invalid state {self.send_state}"
|
|
211
|
+
raise ProtocolError(msg)
|
|
212
|
+
|
|
213
|
+
if self.send_content_length is None:
|
|
214
|
+
# Transfer-Encoding: chunked
|
|
215
|
+
self.send_seen_length += len(body)
|
|
216
|
+
marker = f"{len(body):x}\r\n".encode("ascii")
|
|
217
|
+
self.stream.write(marker + body + b"\r\n")
|
|
218
|
+
|
|
219
|
+
else:
|
|
220
|
+
# Content-Length: xxx
|
|
221
|
+
self.send_seen_length += len(body)
|
|
222
|
+
if self.send_seen_length > self.send_content_length:
|
|
223
|
+
msg = "Too much data sent for declared Content-Length"
|
|
224
|
+
raise ProtocolError(msg)
|
|
225
|
+
if self.send_seen_length < self.send_content_length and body == b"":
|
|
226
|
+
msg = "Not enough data sent for declared Content-Length"
|
|
227
|
+
raise ProtocolError(msg)
|
|
228
|
+
if body:
|
|
229
|
+
self.stream.write(body)
|
|
230
|
+
|
|
231
|
+
if body == b"":
|
|
232
|
+
# Handle body close
|
|
233
|
+
self.send_state = State.DONE
|
|
234
|
+
|
|
235
|
+
def recv_method_line(self) -> tuple[bytes, bytes, bytes]:
|
|
236
|
+
"""
|
|
237
|
+
Receive the initial request method line:
|
|
238
|
+
|
|
239
|
+
>>> method, target, protocol = p.recv_status_line()
|
|
240
|
+
|
|
241
|
+
Receive state will switch to RECV_HEADERS.
|
|
242
|
+
"""
|
|
243
|
+
# Scrub connection keepalive
|
|
244
|
+
self.keepalive_until = None
|
|
245
|
+
|
|
246
|
+
if self.recv_state != State.RECV_METHOD_LINE:
|
|
247
|
+
msg = f"Called 'recv_method_line' in invalid state {self.recv_state}"
|
|
248
|
+
raise ProtocolError(msg)
|
|
249
|
+
|
|
250
|
+
# Read initial response line, eg. "GET / HTTP/1.1"
|
|
251
|
+
exc_text = "reading request method line"
|
|
252
|
+
line = self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text)
|
|
253
|
+
method, target, protocol = line.split(b" ", 2)
|
|
254
|
+
if protocol != b"HTTP/1.1":
|
|
255
|
+
raise ProtocolError("Received unsupported protocol version")
|
|
256
|
+
|
|
257
|
+
self.recv_state = State.RECV_HEADERS
|
|
258
|
+
self.send_state = State.SEND_STATUS_LINE
|
|
259
|
+
return method, target, protocol
|
|
260
|
+
|
|
261
|
+
def recv_status_line(self) -> tuple[bytes, int, bytes]:
|
|
262
|
+
"""
|
|
263
|
+
Receive the initial response status line:
|
|
264
|
+
|
|
265
|
+
>>> protocol, status_code, reason_phrase = p.recv_status_line()
|
|
266
|
+
|
|
267
|
+
Receive state will switch to RECV_HEADERS.
|
|
268
|
+
"""
|
|
269
|
+
if self.recv_state != State.RECV_STATUS_LINE:
|
|
270
|
+
msg = f"Called 'recv_status_line' in invalid state {self.recv_state}"
|
|
271
|
+
raise ProtocolError(msg)
|
|
272
|
+
|
|
273
|
+
# Read initial response line, eg. "HTTP/1.1 200 OK"
|
|
274
|
+
exc_text = "reading response status line"
|
|
275
|
+
line = self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text)
|
|
276
|
+
protocol, status_code_str, reason_phrase = line.split(b" ", 2)
|
|
277
|
+
if protocol != b"HTTP/1.1":
|
|
278
|
+
raise ProtocolError("Received unsupported protocol version")
|
|
279
|
+
|
|
280
|
+
status_code = bounded_int(
|
|
281
|
+
status_code_str, max_digits=3, exc_text="Received invalid status code"
|
|
282
|
+
)
|
|
283
|
+
if status_code < 100:
|
|
284
|
+
raise ProtocolError("Received invalid status code")
|
|
285
|
+
# 1xx status codes preceed the final response status code
|
|
286
|
+
self.processing_1xx = status_code < 200
|
|
287
|
+
|
|
288
|
+
self.recv_state = State.RECV_HEADERS
|
|
289
|
+
return protocol, status_code, reason_phrase
|
|
290
|
+
|
|
291
|
+
def recv_headers(self) -> list[tuple[bytes, bytes]]:
|
|
292
|
+
"""
|
|
293
|
+
Receive the response headers:
|
|
294
|
+
|
|
295
|
+
>>> headers = p.recv_status_line()
|
|
296
|
+
|
|
297
|
+
Receive state will switch to RECV_BODY by default.
|
|
298
|
+
Receive state will revert to RECV_STATUS_CODE for interim 1xx responses.
|
|
299
|
+
"""
|
|
300
|
+
if self.recv_state != State.RECV_HEADERS:
|
|
301
|
+
msg = f"Called 'recv_headers' in invalid state {self.recv_state}"
|
|
302
|
+
raise ProtocolError(msg)
|
|
303
|
+
|
|
304
|
+
# Read response headers
|
|
305
|
+
headers = []
|
|
306
|
+
exc_text = "reading response headers"
|
|
307
|
+
while line := self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text):
|
|
308
|
+
name, value = line.split(b":", 1)
|
|
309
|
+
value = value.strip(b" ")
|
|
310
|
+
headers.append((name, value))
|
|
311
|
+
|
|
312
|
+
# Update header state
|
|
313
|
+
seen_host = False
|
|
314
|
+
for name, value in headers:
|
|
315
|
+
lname = name.lower()
|
|
316
|
+
if lname == b"host":
|
|
317
|
+
seen_host = True
|
|
318
|
+
elif lname == b"content-length":
|
|
319
|
+
self.recv_content_length = bounded_int(
|
|
320
|
+
value, max_digits=20, exc_text="Received invalid Content-Length"
|
|
321
|
+
)
|
|
322
|
+
elif lname == b"connection" and value == b"close":
|
|
323
|
+
self.recv_keep_alive = False
|
|
324
|
+
elif lname == b"transfer-encoding" and value == b"chunked":
|
|
325
|
+
self.recv_content_length = None
|
|
326
|
+
|
|
327
|
+
if self.mode == Mode.SERVER and not seen_host:
|
|
328
|
+
raise ProtocolError("Request missing 'Host' header")
|
|
329
|
+
|
|
330
|
+
if self.processing_1xx:
|
|
331
|
+
# 1xx status codes preceed the final response status code
|
|
332
|
+
self.processing_1xx = False
|
|
333
|
+
self.recv_state = State.RECV_STATUS_LINE
|
|
334
|
+
else:
|
|
335
|
+
self.recv_state = State.RECV_BODY
|
|
336
|
+
return headers
|
|
337
|
+
|
|
338
|
+
def recv_body(self) -> bytes:
|
|
339
|
+
"""
|
|
340
|
+
Receive the response body. An empty byte string indicates the end of the stream:
|
|
341
|
+
|
|
342
|
+
>>> buffer = bytearray()
|
|
343
|
+
>>> while body := p.recv_body()
|
|
344
|
+
>>> buffer.extend(body)
|
|
345
|
+
|
|
346
|
+
The server will switch to DONE.
|
|
347
|
+
"""
|
|
348
|
+
if self.recv_state != State.RECV_BODY:
|
|
349
|
+
msg = f"Called 'recv_body' in invalid state {self.recv_state}"
|
|
350
|
+
raise ProtocolError(msg)
|
|
351
|
+
|
|
352
|
+
if self.recv_content_length is None:
|
|
353
|
+
# Transfer-Encoding: chunked
|
|
354
|
+
exc_text = "reading chunk size"
|
|
355
|
+
line = self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text)
|
|
356
|
+
sizestr, _, _ = line.partition(b";")
|
|
357
|
+
|
|
358
|
+
exc_text = "Received invalid chunk size"
|
|
359
|
+
size = bounded_hex(sizestr, max_digits=8, exc_text=exc_text)
|
|
360
|
+
if size > 0:
|
|
361
|
+
body = self.parser.read(size=size)
|
|
362
|
+
exc_text = "reading chunk data"
|
|
363
|
+
self.parser.read_until(b"\r\n", max_size=2, exc_text=exc_text)
|
|
364
|
+
self.recv_seen_length += len(body)
|
|
365
|
+
else:
|
|
366
|
+
body = b""
|
|
367
|
+
exc_text = "reading chunk termination"
|
|
368
|
+
self.parser.read_until(b"\r\n", max_size=2, exc_text=exc_text)
|
|
369
|
+
|
|
370
|
+
else:
|
|
371
|
+
# Content-Length: xxx
|
|
372
|
+
remaining = self.recv_content_length - self.recv_seen_length
|
|
373
|
+
size = min(remaining, 4096)
|
|
374
|
+
body = self.parser.read(size=size)
|
|
375
|
+
self.recv_seen_length += len(body)
|
|
376
|
+
if self.recv_seen_length < self.recv_content_length and body == b"":
|
|
377
|
+
msg = "Not enough data received for declared Content-Length"
|
|
378
|
+
raise ProtocolError(msg)
|
|
379
|
+
|
|
380
|
+
if body == b"":
|
|
381
|
+
# Handle body close
|
|
382
|
+
self.recv_state = State.DONE
|
|
383
|
+
return body
|
|
384
|
+
|
|
385
|
+
def complete(self):
|
|
386
|
+
is_fully_complete = (
|
|
387
|
+
self.send_state == State.DONE and self.recv_state == State.DONE
|
|
388
|
+
)
|
|
389
|
+
is_keepalive = self.send_keep_alive and self.recv_keep_alive
|
|
390
|
+
|
|
391
|
+
if not (is_fully_complete and is_keepalive):
|
|
392
|
+
self.close()
|
|
393
|
+
return
|
|
394
|
+
|
|
395
|
+
if self.mode == Mode.CLIENT:
|
|
396
|
+
self.send_state = State.SEND_METHOD_LINE
|
|
397
|
+
self.recv_state = State.WAIT
|
|
398
|
+
else:
|
|
399
|
+
self.recv_state = State.RECV_METHOD_LINE
|
|
400
|
+
self.send_state = State.WAIT
|
|
401
|
+
|
|
402
|
+
self.send_content_length = 0
|
|
403
|
+
self.recv_content_length = 0
|
|
404
|
+
self.send_seen_length = 0
|
|
405
|
+
self.recv_seen_length = 0
|
|
406
|
+
self.send_keep_alive = True
|
|
407
|
+
self.recv_keep_alive = True
|
|
408
|
+
self.processing_1xx = False
|
|
409
|
+
self.keepalive_until = time.monotonic() + self.keepalive_duration
|
|
410
|
+
|
|
411
|
+
def close(self):
|
|
412
|
+
if self.send_state != State.CLOSED:
|
|
413
|
+
self.send_state = State.CLOSED
|
|
414
|
+
self.recv_state = State.CLOSED
|
|
415
|
+
self.stream.close()
|
|
416
|
+
|
|
417
|
+
def is_idle(self) -> bool:
|
|
418
|
+
return (
|
|
419
|
+
self.send_state == State.SEND_METHOD_LINE
|
|
420
|
+
or self.recv_state == State.RECV_METHOD_LINE
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
def is_closed(self) -> bool:
|
|
424
|
+
return self.send_state == State.CLOSED
|
|
425
|
+
|
|
426
|
+
def keepalive_expired(self) -> bool:
|
|
427
|
+
return (self.keepalive_until is not None) and (
|
|
428
|
+
time.monotonic() > self.keepalive_until
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
def description(self) -> str:
|
|
432
|
+
return {
|
|
433
|
+
State.SEND_METHOD_LINE: "idle",
|
|
434
|
+
State.CLOSED: "closed",
|
|
435
|
+
}.get(self.send_state, "active")
|
|
436
|
+
|
|
437
|
+
def __repr__(self) -> str:
|
|
438
|
+
cl_state = self.send_state.name
|
|
439
|
+
sr_state = self.recv_state.name
|
|
440
|
+
detail = f"client {cl_state}, server {sr_state}"
|
|
441
|
+
return f"<HTTPParser [{detail}]>"
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class ReadAheadParser:
|
|
445
|
+
"""
|
|
446
|
+
A buffered I/O stream, with methods for read-ahead parsing.
|
|
447
|
+
"""
|
|
448
|
+
|
|
449
|
+
def __init__(self, stream: Stream) -> None:
|
|
450
|
+
self._buffer = b""
|
|
451
|
+
self._stream = stream
|
|
452
|
+
self._chunk_size = 4096
|
|
453
|
+
|
|
454
|
+
def _read_some(self) -> bytes:
|
|
455
|
+
if self._buffer:
|
|
456
|
+
ret, self._buffer = self._buffer, b""
|
|
457
|
+
return ret
|
|
458
|
+
return self._stream.read(self._chunk_size)
|
|
459
|
+
|
|
460
|
+
def _push_back(self, buffer):
|
|
461
|
+
assert self._buffer == b""
|
|
462
|
+
self._buffer = buffer
|
|
463
|
+
|
|
464
|
+
def read(self, size: int) -> bytes:
|
|
465
|
+
"""
|
|
466
|
+
Read and return up to 'size' bytes from the stream, with I/O buffering provided.
|
|
467
|
+
|
|
468
|
+
* Returns b'' to indicate connection close.
|
|
469
|
+
"""
|
|
470
|
+
buffer = bytearray()
|
|
471
|
+
while len(buffer) < size:
|
|
472
|
+
chunk = self._read_some()
|
|
473
|
+
if not chunk:
|
|
474
|
+
break
|
|
475
|
+
buffer.extend(chunk)
|
|
476
|
+
|
|
477
|
+
if len(buffer) > size:
|
|
478
|
+
buffer, push_back = buffer[:size], buffer[size:]
|
|
479
|
+
self._push_back(bytes(push_back))
|
|
480
|
+
return bytes(buffer)
|
|
481
|
+
|
|
482
|
+
def read_until(self, marker: bytes, max_size: int, exc_text: str) -> bytes:
|
|
483
|
+
"""
|
|
484
|
+
Read and return bytes from the stream, delimited by marker.
|
|
485
|
+
|
|
486
|
+
* The marker is not included in the return bytes.
|
|
487
|
+
* The marker is consumed from the I/O stream.
|
|
488
|
+
* Raises `ProtocolError` if the stream closes before a marker occurance.
|
|
489
|
+
* Raises `ProtocolError` if marker did not occur within 'max_size + len(marker)' bytes.
|
|
490
|
+
"""
|
|
491
|
+
buffer = bytearray()
|
|
492
|
+
while len(buffer) <= max_size:
|
|
493
|
+
chunk = self._read_some()
|
|
494
|
+
if not chunk:
|
|
495
|
+
# stream closed before marker found.
|
|
496
|
+
raise ProtocolError(f"Stream closed early {exc_text}")
|
|
497
|
+
start_search = max(len(buffer) - len(marker), 0)
|
|
498
|
+
buffer.extend(chunk)
|
|
499
|
+
index = buffer.find(marker, start_search)
|
|
500
|
+
|
|
501
|
+
if index > max_size:
|
|
502
|
+
# marker was found, though 'max_size' exceeded.
|
|
503
|
+
raise ProtocolError(f"Exceeded maximum size {exc_text}")
|
|
504
|
+
elif index >= 0:
|
|
505
|
+
endindex = index + len(marker)
|
|
506
|
+
self._push_back(bytes(buffer[endindex:]))
|
|
507
|
+
return bytes(buffer[:index])
|
|
508
|
+
|
|
509
|
+
raise ProtocolError(f"Exceeded maximum size {exc_text}")
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def bounded_int(intstr: bytes, max_digits: int, exc_text: str):
|
|
513
|
+
if len(intstr) > max_digits:
|
|
514
|
+
# Length of bytestring exceeds maximum.
|
|
515
|
+
raise ProtocolError(exc_text)
|
|
516
|
+
if len(intstr.strip(b"0123456789")) != 0:
|
|
517
|
+
# Contains invalid characters.
|
|
518
|
+
raise ProtocolError(exc_text)
|
|
519
|
+
|
|
520
|
+
return int(intstr)
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def bounded_hex(hexstr: bytes, max_digits: int, exc_text: str):
|
|
524
|
+
if len(hexstr) > max_digits:
|
|
525
|
+
# Length of bytestring exceeds maximum.
|
|
526
|
+
raise ProtocolError(exc_text)
|
|
527
|
+
if len(hexstr.strip(b"0123456789abcdefABCDEF")) != 0:
|
|
528
|
+
# Contains invalid characters.
|
|
529
|
+
raise ProtocolError(exc_text)
|
|
530
|
+
|
|
531
|
+
return int(hexstr, base=16)
|
hip/_pool.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import types
|
|
3
|
+
import typing
|
|
4
|
+
|
|
5
|
+
if typing.TYPE_CHECKING:
|
|
6
|
+
# Once we drop 3.10 support, we can use 'typing.Self'.
|
|
7
|
+
from typing_extensions import Self
|
|
8
|
+
|
|
9
|
+
from ._connection import Connection, Transport, open_connection
|
|
10
|
+
from ._network import NetworkBackend, Semaphore
|
|
11
|
+
from ._request import Request
|
|
12
|
+
from ._response import Response
|
|
13
|
+
from ._urls import URL
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"ConnectionPool",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ConnectionPool(Transport):
|
|
21
|
+
def __init__(self, backend: NetworkBackend | None = None):
|
|
22
|
+
if backend is None:
|
|
23
|
+
backend = NetworkBackend()
|
|
24
|
+
|
|
25
|
+
self._connections: list[Connection] = []
|
|
26
|
+
self._network_backend = backend
|
|
27
|
+
self._limit_concurrency = Semaphore(100)
|
|
28
|
+
self._closed = False
|
|
29
|
+
|
|
30
|
+
# Public API...
|
|
31
|
+
def send(self, request: Request) -> Response:
|
|
32
|
+
if self._closed:
|
|
33
|
+
raise RuntimeError("ConnectionPool is closed.")
|
|
34
|
+
|
|
35
|
+
# TODO: concurrency limiting
|
|
36
|
+
self._cleanup()
|
|
37
|
+
connection = self._get_connection(request)
|
|
38
|
+
response = connection.send(request)
|
|
39
|
+
return response
|
|
40
|
+
|
|
41
|
+
def close(self):
|
|
42
|
+
self._closed = True
|
|
43
|
+
closing = list(self._connections)
|
|
44
|
+
self._connections = []
|
|
45
|
+
for conn in closing:
|
|
46
|
+
conn.close()
|
|
47
|
+
|
|
48
|
+
# Create or reuse connections as required...
|
|
49
|
+
def _get_connection(self, request: Request) -> "Connection":
|
|
50
|
+
# Attempt to reuse an existing connection.
|
|
51
|
+
url = request.url
|
|
52
|
+
origin = URL(scheme=url.scheme, host=url.host, port=url.port)
|
|
53
|
+
for conn in self._connections:
|
|
54
|
+
if conn.origin() == origin and conn.is_idle() and not conn.is_expired():
|
|
55
|
+
return conn
|
|
56
|
+
|
|
57
|
+
# Or else create a new connection.
|
|
58
|
+
conn = open_connection(
|
|
59
|
+
origin, hostname=request.headers["Host"], backend=self._network_backend
|
|
60
|
+
)
|
|
61
|
+
self._connections.append(conn)
|
|
62
|
+
return conn
|
|
63
|
+
|
|
64
|
+
# Connection pool management...
|
|
65
|
+
def _cleanup(self) -> None:
|
|
66
|
+
time.monotonic()
|
|
67
|
+
for conn in list(self._connections):
|
|
68
|
+
if conn.is_expired():
|
|
69
|
+
conn.close()
|
|
70
|
+
if conn.is_closed():
|
|
71
|
+
self._connections.remove(conn)
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def connections(self) -> list["Connection"]:
|
|
75
|
+
return [c for c in self._connections]
|
|
76
|
+
|
|
77
|
+
def description(self) -> str:
|
|
78
|
+
counts = {"active": 0}
|
|
79
|
+
for status in [c.description() for c in self._connections]:
|
|
80
|
+
counts[status] = counts.get(status, 0) + 1
|
|
81
|
+
return ", ".join(f"{count} {status}" for status, count in counts.items())
|
|
82
|
+
|
|
83
|
+
# Builtins...
|
|
84
|
+
def __repr__(self) -> str:
|
|
85
|
+
return f"<ConnectionPool [{self.description()}]>"
|
|
86
|
+
|
|
87
|
+
def __del__(self):
|
|
88
|
+
if not self._closed:
|
|
89
|
+
import warnings
|
|
90
|
+
|
|
91
|
+
warnings.warn("ConnectionPool was garbage collected without being closed.")
|
|
92
|
+
|
|
93
|
+
def __enter__(self) -> "Self":
|
|
94
|
+
return self
|
|
95
|
+
|
|
96
|
+
def __exit__(
|
|
97
|
+
self,
|
|
98
|
+
exc_type: type[BaseException] | None = None,
|
|
99
|
+
exc_value: BaseException | None = None,
|
|
100
|
+
traceback: types.TracebackType | None = None,
|
|
101
|
+
) -> None:
|
|
102
|
+
self.close()
|