hip 0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hip/_parsers.py ADDED
@@ -0,0 +1,531 @@
1
+ import enum
2
+ import time
3
+
4
+ from ._streams import Stream
5
+
6
+ __all__ = ["HTTPParser", "Mode", "ProtocolError"]
7
+
8
+
9
+ # TODO...
10
+
11
+ # * Upgrade
12
+ # * CONNECT
13
+
14
+ # * Support 'Expect: 100 Continue'
15
+ # * Add 'Error' state transitions
16
+ # * Add tests to trickle data
17
+ # * Add type annotations
18
+
19
+ # * Optional... HTTP/1.0 support
20
+ # * Read trailing headers on Transfer-Encoding: chunked. Not just '\r\n'.
21
+ # * When writing Transfer-Encoding: chunked, split large writes into buffer size.
22
+ # * When reading Transfer-Encoding: chunked, handle incomplete reads from large chunk sizes.
23
+ # * .read() doesn't document if will always return maximum available.
24
+
25
+ # * validate method, target, protocol in request line
26
+ # * validate protocol, status_code, reason_phrase in response line
27
+ # * validate name, value on headers
28
+
29
+
30
+ class State(enum.Enum):
31
+ WAIT = 0
32
+ SEND_METHOD_LINE = 1
33
+ SEND_STATUS_LINE = 2
34
+ SEND_HEADERS = 3
35
+ SEND_BODY = 4
36
+ RECV_METHOD_LINE = 5
37
+ RECV_STATUS_LINE = 6
38
+ RECV_HEADERS = 7
39
+ RECV_BODY = 8
40
+ DONE = 9
41
+ CLOSED = 10
42
+
43
+
44
+ class Mode(enum.Enum):
45
+ CLIENT = 0
46
+ SERVER = 1
47
+
48
+
49
+ # The usual transitions will be...
50
+
51
+ # IDLE, IDLE
52
+ # SEND_HEADERS, IDLE
53
+ # SEND_BODY, IDLE
54
+ # DONE, IDLE
55
+ # DONE, SEND_HEADERS
56
+ # DONE, SEND_BODY
57
+ # DONE, DONE
58
+
59
+ # Then either back to IDLE, IDLE
60
+ # or move to CLOSED, CLOSED
61
+
62
+ # 1. It is also valid for the server to start
63
+ # sending the response without waiting for the
64
+ # complete request.
65
+ # 2. 1xx status codes are interim states, and
66
+ # transition from SEND_HEADERS back to IDLE
67
+ # 3. ...
68
+
69
+
70
+ class ProtocolError(Exception):
71
+ pass
72
+
73
+
74
+ class HTTPParser:
75
+ """
76
+ Usage...
77
+
78
+ client = HTTPParser(writer, reader)
79
+ client.send_method_line()
80
+ client.send_headers()
81
+ client.send_body()
82
+ client.recv_status_line()
83
+ client.recv_headers()
84
+ client.recv_body()
85
+ client.complete()
86
+ client.close()
87
+ """
88
+
89
+ def __init__(self, stream: Stream, mode: str) -> None:
90
+ self.stream = stream
91
+ self.parser = ReadAheadParser(stream)
92
+ self.mode = {"CLIENT": Mode.CLIENT, "SERVER": Mode.SERVER}[mode]
93
+ self.keepalive_duration = 5.0
94
+
95
+ # Track state...
96
+ if self.mode == Mode.CLIENT:
97
+ self.send_state: State = State.SEND_METHOD_LINE
98
+ self.recv_state: State = State.WAIT
99
+ else:
100
+ self.recv_state = State.RECV_METHOD_LINE
101
+ self.send_state = State.WAIT
102
+
103
+ # Track message framing...
104
+ self.send_content_length: int | None = 0
105
+ self.recv_content_length: int | None = 0
106
+ self.send_seen_length = 0
107
+ self.recv_seen_length = 0
108
+
109
+ # Track connection keep alive...
110
+ self.send_keep_alive = True
111
+ self.recv_keep_alive = True
112
+ self.keepalive_until: float | None = None
113
+
114
+ # Special states...
115
+ self.processing_1xx = False
116
+
117
+ def send_method_line(self, method: bytes, target: bytes, protocol: bytes) -> None:
118
+ """
119
+ Send the initial request line:
120
+
121
+ >>> p.send_method_line(b'GET', b'/', b'HTTP/1.1')
122
+
123
+ Sending state will switch to SEND_HEADERS state.
124
+ """
125
+ # Scrub connection keepalive
126
+ self.keepalive_until = None
127
+
128
+ if self.send_state != State.SEND_METHOD_LINE:
129
+ msg = f"Called 'send_method_line' in invalid state {self.send_state}"
130
+ raise ProtocolError(msg)
131
+
132
+ # Send initial request line, eg. "GET / HTTP/1.1"
133
+ if protocol != b"HTTP/1.1":
134
+ raise ProtocolError("Sent unsupported protocol version")
135
+ data = b" ".join([method, target, protocol]) + b"\r\n"
136
+ self.stream.write(data)
137
+
138
+ self.send_state = State.SEND_HEADERS
139
+ self.recv_state = State.RECV_STATUS_LINE
140
+
141
+ def send_status_line(
142
+ self, protocol: bytes, status_code: int, reason: bytes
143
+ ) -> None:
144
+ """
145
+ Send the initial response line:
146
+
147
+ >>> p.send_method_line(b'HTTP/1.1', 200, b'OK')
148
+
149
+ Sending state will switch to SEND_HEADERS state.
150
+ """
151
+ if self.send_state != State.SEND_STATUS_LINE:
152
+ msg = f"Called 'send_status_line' in invalid state {self.send_state}"
153
+ raise ProtocolError(msg)
154
+
155
+ # Send initial request line, eg. "GET / HTTP/1.1"
156
+ if protocol != b"HTTP/1.1":
157
+ raise ProtocolError("Sent unsupported protocol version")
158
+ status_code_bytes = str(status_code).encode("ascii")
159
+ data = b" ".join([protocol, status_code_bytes, reason]) + b"\r\n"
160
+ self.stream.write(data)
161
+
162
+ self.send_state = State.SEND_HEADERS
163
+
164
+ def send_headers(self, headers: list[tuple[bytes, bytes]]) -> None:
165
+ """
166
+ Send the request headers:
167
+
168
+ >>> p.send_headers([(b'Host', b'www.example.com')])
169
+
170
+ Sending state will switch to SEND_BODY state.
171
+ """
172
+ if self.send_state != State.SEND_HEADERS:
173
+ msg = f"Called 'send_headers' in invalid state {self.send_state}"
174
+ raise ProtocolError(msg)
175
+
176
+ # Update header state
177
+ seen_host = False
178
+ for name, value in headers:
179
+ lname = name.lower()
180
+ if lname == b"host":
181
+ seen_host = True
182
+ elif lname == b"content-length":
183
+ self.send_content_length = bounded_int(
184
+ value, max_digits=20, exc_text="Sent invalid Content-Length"
185
+ )
186
+ elif lname == b"connection" and value == b"close":
187
+ self.send_keep_alive = False
188
+ elif lname == b"transfer-encoding" and value == b"chunked":
189
+ self.send_content_length = None
190
+
191
+ if self.mode == Mode.CLIENT and not seen_host:
192
+ raise ProtocolError("Request missing 'Host' header")
193
+
194
+ # Send request headers
195
+ lines = [name + b": " + value + b"\r\n" for name, value in headers]
196
+ data = b"".join(lines) + b"\r\n"
197
+ self.stream.write(data)
198
+
199
+ self.send_state = State.SEND_BODY
200
+
201
+ def send_body(self, body: bytes) -> None:
202
+ """
203
+ Send the request body. An empty bytes argument indicates the end of the stream:
204
+
205
+ >>> p.send_body(b'')
206
+
207
+ Sending state will switch to DONE.
208
+ """
209
+ if self.send_state != State.SEND_BODY:
210
+ msg = f"Called 'send_body' in invalid state {self.send_state}"
211
+ raise ProtocolError(msg)
212
+
213
+ if self.send_content_length is None:
214
+ # Transfer-Encoding: chunked
215
+ self.send_seen_length += len(body)
216
+ marker = f"{len(body):x}\r\n".encode("ascii")
217
+ self.stream.write(marker + body + b"\r\n")
218
+
219
+ else:
220
+ # Content-Length: xxx
221
+ self.send_seen_length += len(body)
222
+ if self.send_seen_length > self.send_content_length:
223
+ msg = "Too much data sent for declared Content-Length"
224
+ raise ProtocolError(msg)
225
+ if self.send_seen_length < self.send_content_length and body == b"":
226
+ msg = "Not enough data sent for declared Content-Length"
227
+ raise ProtocolError(msg)
228
+ if body:
229
+ self.stream.write(body)
230
+
231
+ if body == b"":
232
+ # Handle body close
233
+ self.send_state = State.DONE
234
+
235
+ def recv_method_line(self) -> tuple[bytes, bytes, bytes]:
236
+ """
237
+ Receive the initial request method line:
238
+
239
+ >>> method, target, protocol = p.recv_status_line()
240
+
241
+ Receive state will switch to RECV_HEADERS.
242
+ """
243
+ # Scrub connection keepalive
244
+ self.keepalive_until = None
245
+
246
+ if self.recv_state != State.RECV_METHOD_LINE:
247
+ msg = f"Called 'recv_method_line' in invalid state {self.recv_state}"
248
+ raise ProtocolError(msg)
249
+
250
+ # Read initial response line, eg. "GET / HTTP/1.1"
251
+ exc_text = "reading request method line"
252
+ line = self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text)
253
+ method, target, protocol = line.split(b" ", 2)
254
+ if protocol != b"HTTP/1.1":
255
+ raise ProtocolError("Received unsupported protocol version")
256
+
257
+ self.recv_state = State.RECV_HEADERS
258
+ self.send_state = State.SEND_STATUS_LINE
259
+ return method, target, protocol
260
+
261
+ def recv_status_line(self) -> tuple[bytes, int, bytes]:
262
+ """
263
+ Receive the initial response status line:
264
+
265
+ >>> protocol, status_code, reason_phrase = p.recv_status_line()
266
+
267
+ Receive state will switch to RECV_HEADERS.
268
+ """
269
+ if self.recv_state != State.RECV_STATUS_LINE:
270
+ msg = f"Called 'recv_status_line' in invalid state {self.recv_state}"
271
+ raise ProtocolError(msg)
272
+
273
+ # Read initial response line, eg. "HTTP/1.1 200 OK"
274
+ exc_text = "reading response status line"
275
+ line = self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text)
276
+ protocol, status_code_str, reason_phrase = line.split(b" ", 2)
277
+ if protocol != b"HTTP/1.1":
278
+ raise ProtocolError("Received unsupported protocol version")
279
+
280
+ status_code = bounded_int(
281
+ status_code_str, max_digits=3, exc_text="Received invalid status code"
282
+ )
283
+ if status_code < 100:
284
+ raise ProtocolError("Received invalid status code")
285
+ # 1xx status codes preceed the final response status code
286
+ self.processing_1xx = status_code < 200
287
+
288
+ self.recv_state = State.RECV_HEADERS
289
+ return protocol, status_code, reason_phrase
290
+
291
+ def recv_headers(self) -> list[tuple[bytes, bytes]]:
292
+ """
293
+ Receive the response headers:
294
+
295
+ >>> headers = p.recv_status_line()
296
+
297
+ Receive state will switch to RECV_BODY by default.
298
+ Receive state will revert to RECV_STATUS_CODE for interim 1xx responses.
299
+ """
300
+ if self.recv_state != State.RECV_HEADERS:
301
+ msg = f"Called 'recv_headers' in invalid state {self.recv_state}"
302
+ raise ProtocolError(msg)
303
+
304
+ # Read response headers
305
+ headers = []
306
+ exc_text = "reading response headers"
307
+ while line := self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text):
308
+ name, value = line.split(b":", 1)
309
+ value = value.strip(b" ")
310
+ headers.append((name, value))
311
+
312
+ # Update header state
313
+ seen_host = False
314
+ for name, value in headers:
315
+ lname = name.lower()
316
+ if lname == b"host":
317
+ seen_host = True
318
+ elif lname == b"content-length":
319
+ self.recv_content_length = bounded_int(
320
+ value, max_digits=20, exc_text="Received invalid Content-Length"
321
+ )
322
+ elif lname == b"connection" and value == b"close":
323
+ self.recv_keep_alive = False
324
+ elif lname == b"transfer-encoding" and value == b"chunked":
325
+ self.recv_content_length = None
326
+
327
+ if self.mode == Mode.SERVER and not seen_host:
328
+ raise ProtocolError("Request missing 'Host' header")
329
+
330
+ if self.processing_1xx:
331
+ # 1xx status codes preceed the final response status code
332
+ self.processing_1xx = False
333
+ self.recv_state = State.RECV_STATUS_LINE
334
+ else:
335
+ self.recv_state = State.RECV_BODY
336
+ return headers
337
+
338
+ def recv_body(self) -> bytes:
339
+ """
340
+ Receive the response body. An empty byte string indicates the end of the stream:
341
+
342
+ >>> buffer = bytearray()
343
+ >>> while body := p.recv_body()
344
+ >>> buffer.extend(body)
345
+
346
+ The server will switch to DONE.
347
+ """
348
+ if self.recv_state != State.RECV_BODY:
349
+ msg = f"Called 'recv_body' in invalid state {self.recv_state}"
350
+ raise ProtocolError(msg)
351
+
352
+ if self.recv_content_length is None:
353
+ # Transfer-Encoding: chunked
354
+ exc_text = "reading chunk size"
355
+ line = self.parser.read_until(b"\r\n", max_size=4096, exc_text=exc_text)
356
+ sizestr, _, _ = line.partition(b";")
357
+
358
+ exc_text = "Received invalid chunk size"
359
+ size = bounded_hex(sizestr, max_digits=8, exc_text=exc_text)
360
+ if size > 0:
361
+ body = self.parser.read(size=size)
362
+ exc_text = "reading chunk data"
363
+ self.parser.read_until(b"\r\n", max_size=2, exc_text=exc_text)
364
+ self.recv_seen_length += len(body)
365
+ else:
366
+ body = b""
367
+ exc_text = "reading chunk termination"
368
+ self.parser.read_until(b"\r\n", max_size=2, exc_text=exc_text)
369
+
370
+ else:
371
+ # Content-Length: xxx
372
+ remaining = self.recv_content_length - self.recv_seen_length
373
+ size = min(remaining, 4096)
374
+ body = self.parser.read(size=size)
375
+ self.recv_seen_length += len(body)
376
+ if self.recv_seen_length < self.recv_content_length and body == b"":
377
+ msg = "Not enough data received for declared Content-Length"
378
+ raise ProtocolError(msg)
379
+
380
+ if body == b"":
381
+ # Handle body close
382
+ self.recv_state = State.DONE
383
+ return body
384
+
385
+ def complete(self):
386
+ is_fully_complete = (
387
+ self.send_state == State.DONE and self.recv_state == State.DONE
388
+ )
389
+ is_keepalive = self.send_keep_alive and self.recv_keep_alive
390
+
391
+ if not (is_fully_complete and is_keepalive):
392
+ self.close()
393
+ return
394
+
395
+ if self.mode == Mode.CLIENT:
396
+ self.send_state = State.SEND_METHOD_LINE
397
+ self.recv_state = State.WAIT
398
+ else:
399
+ self.recv_state = State.RECV_METHOD_LINE
400
+ self.send_state = State.WAIT
401
+
402
+ self.send_content_length = 0
403
+ self.recv_content_length = 0
404
+ self.send_seen_length = 0
405
+ self.recv_seen_length = 0
406
+ self.send_keep_alive = True
407
+ self.recv_keep_alive = True
408
+ self.processing_1xx = False
409
+ self.keepalive_until = time.monotonic() + self.keepalive_duration
410
+
411
+ def close(self):
412
+ if self.send_state != State.CLOSED:
413
+ self.send_state = State.CLOSED
414
+ self.recv_state = State.CLOSED
415
+ self.stream.close()
416
+
417
+ def is_idle(self) -> bool:
418
+ return (
419
+ self.send_state == State.SEND_METHOD_LINE
420
+ or self.recv_state == State.RECV_METHOD_LINE
421
+ )
422
+
423
+ def is_closed(self) -> bool:
424
+ return self.send_state == State.CLOSED
425
+
426
+ def keepalive_expired(self) -> bool:
427
+ return (self.keepalive_until is not None) and (
428
+ time.monotonic() > self.keepalive_until
429
+ )
430
+
431
+ def description(self) -> str:
432
+ return {
433
+ State.SEND_METHOD_LINE: "idle",
434
+ State.CLOSED: "closed",
435
+ }.get(self.send_state, "active")
436
+
437
+ def __repr__(self) -> str:
438
+ cl_state = self.send_state.name
439
+ sr_state = self.recv_state.name
440
+ detail = f"client {cl_state}, server {sr_state}"
441
+ return f"<HTTPParser [{detail}]>"
442
+
443
+
444
+ class ReadAheadParser:
445
+ """
446
+ A buffered I/O stream, with methods for read-ahead parsing.
447
+ """
448
+
449
+ def __init__(self, stream: Stream) -> None:
450
+ self._buffer = b""
451
+ self._stream = stream
452
+ self._chunk_size = 4096
453
+
454
+ def _read_some(self) -> bytes:
455
+ if self._buffer:
456
+ ret, self._buffer = self._buffer, b""
457
+ return ret
458
+ return self._stream.read(self._chunk_size)
459
+
460
+ def _push_back(self, buffer):
461
+ assert self._buffer == b""
462
+ self._buffer = buffer
463
+
464
+ def read(self, size: int) -> bytes:
465
+ """
466
+ Read and return up to 'size' bytes from the stream, with I/O buffering provided.
467
+
468
+ * Returns b'' to indicate connection close.
469
+ """
470
+ buffer = bytearray()
471
+ while len(buffer) < size:
472
+ chunk = self._read_some()
473
+ if not chunk:
474
+ break
475
+ buffer.extend(chunk)
476
+
477
+ if len(buffer) > size:
478
+ buffer, push_back = buffer[:size], buffer[size:]
479
+ self._push_back(bytes(push_back))
480
+ return bytes(buffer)
481
+
482
+ def read_until(self, marker: bytes, max_size: int, exc_text: str) -> bytes:
483
+ """
484
+ Read and return bytes from the stream, delimited by marker.
485
+
486
+ * The marker is not included in the return bytes.
487
+ * The marker is consumed from the I/O stream.
488
+ * Raises `ProtocolError` if the stream closes before a marker occurance.
489
+ * Raises `ProtocolError` if marker did not occur within 'max_size + len(marker)' bytes.
490
+ """
491
+ buffer = bytearray()
492
+ while len(buffer) <= max_size:
493
+ chunk = self._read_some()
494
+ if not chunk:
495
+ # stream closed before marker found.
496
+ raise ProtocolError(f"Stream closed early {exc_text}")
497
+ start_search = max(len(buffer) - len(marker), 0)
498
+ buffer.extend(chunk)
499
+ index = buffer.find(marker, start_search)
500
+
501
+ if index > max_size:
502
+ # marker was found, though 'max_size' exceeded.
503
+ raise ProtocolError(f"Exceeded maximum size {exc_text}")
504
+ elif index >= 0:
505
+ endindex = index + len(marker)
506
+ self._push_back(bytes(buffer[endindex:]))
507
+ return bytes(buffer[:index])
508
+
509
+ raise ProtocolError(f"Exceeded maximum size {exc_text}")
510
+
511
+
512
+ def bounded_int(intstr: bytes, max_digits: int, exc_text: str):
513
+ if len(intstr) > max_digits:
514
+ # Length of bytestring exceeds maximum.
515
+ raise ProtocolError(exc_text)
516
+ if len(intstr.strip(b"0123456789")) != 0:
517
+ # Contains invalid characters.
518
+ raise ProtocolError(exc_text)
519
+
520
+ return int(intstr)
521
+
522
+
523
+ def bounded_hex(hexstr: bytes, max_digits: int, exc_text: str):
524
+ if len(hexstr) > max_digits:
525
+ # Length of bytestring exceeds maximum.
526
+ raise ProtocolError(exc_text)
527
+ if len(hexstr.strip(b"0123456789abcdefABCDEF")) != 0:
528
+ # Contains invalid characters.
529
+ raise ProtocolError(exc_text)
530
+
531
+ return int(hexstr, base=16)
hip/_pool.py ADDED
@@ -0,0 +1,102 @@
1
+ import time
2
+ import types
3
+ import typing
4
+
5
+ if typing.TYPE_CHECKING:
6
+ # Once we drop 3.10 support, we can use 'typing.Self'.
7
+ from typing_extensions import Self
8
+
9
+ from ._connection import Connection, Transport, open_connection
10
+ from ._network import NetworkBackend, Semaphore
11
+ from ._request import Request
12
+ from ._response import Response
13
+ from ._urls import URL
14
+
15
+ __all__ = [
16
+ "ConnectionPool",
17
+ ]
18
+
19
+
20
+ class ConnectionPool(Transport):
21
+ def __init__(self, backend: NetworkBackend | None = None):
22
+ if backend is None:
23
+ backend = NetworkBackend()
24
+
25
+ self._connections: list[Connection] = []
26
+ self._network_backend = backend
27
+ self._limit_concurrency = Semaphore(100)
28
+ self._closed = False
29
+
30
+ # Public API...
31
+ def send(self, request: Request) -> Response:
32
+ if self._closed:
33
+ raise RuntimeError("ConnectionPool is closed.")
34
+
35
+ # TODO: concurrency limiting
36
+ self._cleanup()
37
+ connection = self._get_connection(request)
38
+ response = connection.send(request)
39
+ return response
40
+
41
+ def close(self):
42
+ self._closed = True
43
+ closing = list(self._connections)
44
+ self._connections = []
45
+ for conn in closing:
46
+ conn.close()
47
+
48
+ # Create or reuse connections as required...
49
+ def _get_connection(self, request: Request) -> "Connection":
50
+ # Attempt to reuse an existing connection.
51
+ url = request.url
52
+ origin = URL(scheme=url.scheme, host=url.host, port=url.port)
53
+ for conn in self._connections:
54
+ if conn.origin() == origin and conn.is_idle() and not conn.is_expired():
55
+ return conn
56
+
57
+ # Or else create a new connection.
58
+ conn = open_connection(
59
+ origin, hostname=request.headers["Host"], backend=self._network_backend
60
+ )
61
+ self._connections.append(conn)
62
+ return conn
63
+
64
+ # Connection pool management...
65
+ def _cleanup(self) -> None:
66
+ time.monotonic()
67
+ for conn in list(self._connections):
68
+ if conn.is_expired():
69
+ conn.close()
70
+ if conn.is_closed():
71
+ self._connections.remove(conn)
72
+
73
+ @property
74
+ def connections(self) -> list["Connection"]:
75
+ return [c for c in self._connections]
76
+
77
+ def description(self) -> str:
78
+ counts = {"active": 0}
79
+ for status in [c.description() for c in self._connections]:
80
+ counts[status] = counts.get(status, 0) + 1
81
+ return ", ".join(f"{count} {status}" for status, count in counts.items())
82
+
83
+ # Builtins...
84
+ def __repr__(self) -> str:
85
+ return f"<ConnectionPool [{self.description()}]>"
86
+
87
+ def __del__(self):
88
+ if not self._closed:
89
+ import warnings
90
+
91
+ warnings.warn("ConnectionPool was garbage collected without being closed.")
92
+
93
+ def __enter__(self) -> "Self":
94
+ return self
95
+
96
+ def __exit__(
97
+ self,
98
+ exc_type: type[BaseException] | None = None,
99
+ exc_value: BaseException | None = None,
100
+ traceback: types.TracebackType | None = None,
101
+ ) -> None:
102
+ self.close()