punkreq 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {punkreq-0.2.1 → punkreq-0.2.3}/PKG-INFO +3 -3
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_connect.py +12 -3
- punkreq-0.2.3/punkreq/_proxies.py +153 -0
- punkreq-0.2.3/punkreq/_version.py +1 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/pyproject.toml +7 -3
- punkreq-0.2.3/tests/__init__.py +0 -0
- punkreq-0.2.3/tests/conftest.py +77 -0
- punkreq-0.2.3/tests/e2e.py +79 -0
- punkreq-0.2.3/tests/fakes.py +151 -0
- punkreq-0.2.3/tests/servers.py +101 -0
- punkreq-0.2.3/tests/servers_tonio.py +161 -0
- punkreq-0.2.3/tests/test_client.py +448 -0
- punkreq-0.2.3/tests/test_config.py +108 -0
- punkreq-0.2.3/tests/test_connect.py +52 -0
- punkreq-0.2.3/tests/test_content.py +156 -0
- punkreq-0.2.3/tests/test_cookies.py +120 -0
- punkreq-0.2.3/tests/test_decoders.py +124 -0
- punkreq-0.2.3/tests/test_e2e_asyncio.py +43 -0
- punkreq-0.2.3/tests/test_e2e_tonio.py +94 -0
- punkreq-0.2.3/tests/test_exceptions.py +50 -0
- punkreq-0.2.3/tests/test_headers.py +129 -0
- punkreq-0.2.3/tests/test_models.py +312 -0
- punkreq-0.2.3/tests/test_multipart.py +112 -0
- punkreq-0.2.3/tests/test_pool.py +476 -0
- punkreq-0.2.3/tests/test_redirects.py +145 -0
- punkreq-0.2.3/tests/test_transport.py +330 -0
- punkreq-0.2.3/tests/test_urls.py +248 -0
- punkreq-0.2.1/punkreq/_proxies.py +0 -282
- punkreq-0.2.1/punkreq/_version.py +0 -1
- punkreq-0.2.1/tests/test_client.py +0 -656
- punkreq-0.2.1/tests/test_config.py +0 -88
- punkreq-0.2.1/tests/test_content.py +0 -126
- punkreq-0.2.1/tests/test_cookies.py +0 -123
- punkreq-0.2.1/tests/test_decoders.py +0 -127
- punkreq-0.2.1/tests/test_e2e_client.py +0 -149
- punkreq-0.2.1/tests/test_e2e_transport.py +0 -211
- punkreq-0.2.1/tests/test_exceptions.py +0 -74
- punkreq-0.2.1/tests/test_headers.py +0 -127
- punkreq-0.2.1/tests/test_models.py +0 -318
- punkreq-0.2.1/tests/test_multipart.py +0 -95
- punkreq-0.2.1/tests/test_pool.py +0 -682
- punkreq-0.2.1/tests/test_proxies.py +0 -328
- punkreq-0.2.1/tests/test_redirects.py +0 -128
- punkreq-0.2.1/tests/test_transport.py +0 -488
- punkreq-0.2.1/tests/test_urls.py +0 -221
- {punkreq-0.2.1 → punkreq-0.2.3}/.gitignore +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/LICENSE +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/README.md +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/__init__.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_auth.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_client.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_config.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_content.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_cookies.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_decoders.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_exceptions.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_headers.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_models.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_multipart.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_pool.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_redirects.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_transport.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/_urls.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/asyncio.py +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/py.typed +0 -0
- {punkreq-0.2.1 → punkreq-0.2.3}/punkreq/tonio.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: punkreq
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: An async HTTP client built on httpunk
|
|
5
5
|
Project-URL: Homepage, https://github.com/gi0baro/punkreq
|
|
6
6
|
Project-URL: Funding, https://github.com/sponsors/gi0baro
|
|
@@ -24,11 +24,11 @@ Classifier: Programming Language :: Python :: 3.15
|
|
|
24
24
|
Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
|
|
25
25
|
Classifier: Topic :: Internet :: WWW/HTTP
|
|
26
26
|
Requires-Python: >=3.10
|
|
27
|
-
Requires-Dist: httpunk~=0.
|
|
27
|
+
Requires-Dist: httpunk~=0.4.4
|
|
28
28
|
Provides-Extra: brotli
|
|
29
29
|
Requires-Dist: brotli~=1.2.0; extra == 'brotli'
|
|
30
30
|
Provides-Extra: tonio
|
|
31
|
-
Requires-Dist: tonio~=0.
|
|
31
|
+
Requires-Dist: tonio~=0.10.0; (python_version >= '3.14') and extra == 'tonio'
|
|
32
32
|
Description-Content-Type: text/markdown
|
|
33
33
|
|
|
34
34
|
# punkreq
|
|
@@ -10,7 +10,7 @@ from httpunk.h2.client import H2Connection
|
|
|
10
10
|
from ._exceptions import ConnectError, UnsupportedProtocol
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
__all__ = ["Connector", "Origin", "create_ssl_context", "origin_for_url"]
|
|
13
|
+
__all__ = ["Connector", "Origin", "connect_errors", "create_ssl_context", "origin_for_url"]
|
|
14
14
|
|
|
15
15
|
DEFAULT_PORTS = {"http": 80, "https": 443}
|
|
16
16
|
|
|
@@ -82,6 +82,15 @@ def create_ssl_context(verify: VerifyTypes = True, cert: CertTypes | None = None
|
|
|
82
82
|
return context
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
def connect_errors(backend: typing.Any) -> tuple[type[BaseException], ...]:
|
|
86
|
+
"""What a dial or a TLS handshake fails with on `backend`: the OS and ssl
|
|
87
|
+
errors, plus the backend's own transport failure. tonio's `ResourceBroken`
|
|
88
|
+
is a plain `Exception`, not an `OSError`, and its TLS layer raises it in
|
|
89
|
+
place of the ssl error, so `(OSError, ssl.SSLError)` alone lets a failed
|
|
90
|
+
handshake on tonio escape as an unmapped exception."""
|
|
91
|
+
return (OSError, ssl.SSLError, *backend.broken_transport_errors)
|
|
92
|
+
|
|
93
|
+
|
|
85
94
|
class Connector:
|
|
86
95
|
"""The default connector: dial `origin` and return the protocol-matching,
|
|
87
96
|
un-entered httpunk connection."""
|
|
@@ -108,7 +117,7 @@ class Connector:
|
|
|
108
117
|
self.alpn = ("http/1.1",)
|
|
109
118
|
# The ALPN offer is configured on the context ONCE, here: a context is
|
|
110
119
|
# shared by every dial that uses it (and by `ProxyConnector`'s CONNECT
|
|
111
|
-
# tunnels, whose `
|
|
120
|
+
# tunnels, whose `wrap_tls` takes ALPN from the context alone), so the
|
|
112
121
|
# dial path never mutates it (hyper-util / httpunk `util.connect`
|
|
113
122
|
# discipline: a caller-supplied context is never touched per connect).
|
|
114
123
|
if ssl_context is not None:
|
|
@@ -119,7 +128,7 @@ class Connector:
|
|
|
119
128
|
if origin.scheme == "https":
|
|
120
129
|
return await self._connect_tls(origin)
|
|
121
130
|
return await self._connect_tcp(origin)
|
|
122
|
-
except (
|
|
131
|
+
except connect_errors(self._backend) as exc:
|
|
123
132
|
raise ConnectError(f"Failed to connect to {origin}: {exc}")
|
|
124
133
|
|
|
125
134
|
async def _connect_tls(self, origin: Origin) -> H1Connection | H2Connection:
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ssl
|
|
4
|
+
import typing
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
|
|
7
|
+
import httpunk
|
|
8
|
+
from httpunk.h1.client import H1Connection
|
|
9
|
+
from httpunk.h2.client import H2Connection
|
|
10
|
+
from httpunk.util.proxy import Matcher
|
|
11
|
+
|
|
12
|
+
from ._config import Proxy
|
|
13
|
+
from ._connect import Connector, Origin, connect_errors
|
|
14
|
+
from ._exceptions import ConnectError, ProxyError
|
|
15
|
+
from ._headers import Headers
|
|
16
|
+
from ._urls import URL
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
__all__ = ["ProxyConfig", "ProxyConnector", "ProxyInfo"]
|
|
20
|
+
|
|
21
|
+
ProxyTypes = typing.Union[Proxy, URL, str, None]
|
|
22
|
+
|
|
23
|
+
# attribute name tagging plain-http proxied connections for the transport
|
|
24
|
+
PROXY_ATTR = "punkreq_proxy"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ProxyInfo(typing.NamedTuple):
|
|
28
|
+
"""Per-request proxy metadata for absolute-form (plain http) proxying."""
|
|
29
|
+
|
|
30
|
+
auth: str | None # ready `Proxy-Authorization` value
|
|
31
|
+
headers: Headers
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ProxyConfig:
|
|
35
|
+
"""Resolved proxy configuration: a matcher plus proxy-level extras."""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self, matcher: Matcher, *, headers: Headers | None = None, ssl_context: ssl.SSLContext | None = None
|
|
39
|
+
) -> None:
|
|
40
|
+
self.matcher = matcher
|
|
41
|
+
self.headers = headers if headers is not None else Headers()
|
|
42
|
+
self.ssl_context = ssl_context # for TLS to the proxy itself (https:// proxies)
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def resolve(cls, proxy: ProxyTypes, trust_env: bool) -> ProxyConfig | None:
|
|
46
|
+
if proxy is not None:
|
|
47
|
+
proxy = proxy if isinstance(proxy, Proxy) else Proxy(proxy)
|
|
48
|
+
url = proxy.url
|
|
49
|
+
if proxy.auth is not None:
|
|
50
|
+
url = url.copy_with(username=proxy.auth[0], password=proxy.auth[1])
|
|
51
|
+
matcher = Matcher.from_parts(all=str(url))
|
|
52
|
+
return cls(matcher, headers=proxy.headers, ssl_context=proxy.ssl_context)
|
|
53
|
+
if trust_env:
|
|
54
|
+
return cls(Matcher.from_env())
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ProxyConnector:
|
|
59
|
+
"""Connector that routes each origin through its matched proxy, delegating
|
|
60
|
+
unmatched origins to the direct `Connector`."""
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
*,
|
|
65
|
+
backend: typing.Any,
|
|
66
|
+
ssl_context: ssl.SSLContext,
|
|
67
|
+
http1: bool = True,
|
|
68
|
+
http2: bool = True,
|
|
69
|
+
config: ProxyConfig,
|
|
70
|
+
) -> None:
|
|
71
|
+
self._backend = backend
|
|
72
|
+
self._ssl_context = ssl_context
|
|
73
|
+
self._http1 = http1
|
|
74
|
+
self._http2 = http2
|
|
75
|
+
self._config = config
|
|
76
|
+
# the direct connector configures the ALPN offer on `ssl_context` at
|
|
77
|
+
# construction; `wrap_tls` (the CONNECT tunnel) takes it from the context
|
|
78
|
+
self._direct = Connector(backend=backend, ssl_context=ssl_context, http1=http1, http2=http2)
|
|
79
|
+
|
|
80
|
+
async def __call__(self, origin: Origin) -> typing.Any:
|
|
81
|
+
intercept = self._config.matcher.intercept(f"{origin.scheme}://{origin.authority}/")
|
|
82
|
+
if intercept is None:
|
|
83
|
+
return await self._direct(origin)
|
|
84
|
+
try:
|
|
85
|
+
if origin.scheme == "http":
|
|
86
|
+
return await self._absolute_form(origin, intercept)
|
|
87
|
+
return await self._tunnel(origin, intercept)
|
|
88
|
+
except connect_errors(self._backend) as exc:
|
|
89
|
+
raise ConnectError(f"Failed to connect to {origin} via proxy {intercept.uri}: {exc}")
|
|
90
|
+
|
|
91
|
+
async def _dial_proxy(self, intercept: typing.Any) -> typing.Any:
|
|
92
|
+
parts = urlsplit(intercept.uri)
|
|
93
|
+
scheme, host = parts.scheme, parts.hostname
|
|
94
|
+
port = parts.port or (443 if scheme == "https" else 80)
|
|
95
|
+
if scheme == "https":
|
|
96
|
+
stream, _ = await self._backend.connect_tls(
|
|
97
|
+
host, port, alpn=("http/1.1",), ssl_context=self._config.ssl_context
|
|
98
|
+
)
|
|
99
|
+
return stream
|
|
100
|
+
return await self._backend.connect_tcp(host, port)
|
|
101
|
+
|
|
102
|
+
def _proxy_headers(self, intercept: typing.Any, authority: str) -> httpunk.HeaderMap:
|
|
103
|
+
headers = httpunk.HeaderMap({"host": authority})
|
|
104
|
+
auth = intercept.basic_auth()
|
|
105
|
+
if auth is not None:
|
|
106
|
+
headers["proxy-authorization"] = auth
|
|
107
|
+
for key, value in self._config.headers.raw:
|
|
108
|
+
headers.add(key, value)
|
|
109
|
+
return headers
|
|
110
|
+
|
|
111
|
+
async def _absolute_form(self, origin: Origin, intercept: typing.Any) -> typing.Any:
|
|
112
|
+
transport = await self._dial_proxy(intercept)
|
|
113
|
+
conn = H1Connection(transport, authority=origin.authority, backend=self._backend)
|
|
114
|
+
setattr(conn, PROXY_ATTR, ProxyInfo(auth=intercept.basic_auth(), headers=self._config.headers))
|
|
115
|
+
return conn
|
|
116
|
+
|
|
117
|
+
async def _tunnel(self, origin: Origin, intercept: typing.Any) -> typing.Any:
|
|
118
|
+
transport = await self._dial_proxy(intercept)
|
|
119
|
+
host = f"[{origin.host}]" if ":" in origin.host else origin.host
|
|
120
|
+
target = f"{host}:{origin.port}" # CONNECT wants authority form with an explicit port
|
|
121
|
+
try:
|
|
122
|
+
proxy_conn = H1Connection(transport, backend=self._backend)
|
|
123
|
+
await proxy_conn.__aenter__()
|
|
124
|
+
response = await proxy_conn.send_request(
|
|
125
|
+
httpunk.Request("CONNECT", target, headers=self._proxy_headers(intercept, target))
|
|
126
|
+
)
|
|
127
|
+
if not (200 <= response.status < 300):
|
|
128
|
+
raise ProxyError(f"The proxy refused CONNECT to {target}: {response.status}")
|
|
129
|
+
tunnel = response.upgraded
|
|
130
|
+
if tunnel is None:
|
|
131
|
+
raise ProxyError(f"The proxy did not open a CONNECT tunnel to {target}")
|
|
132
|
+
except BaseException:
|
|
133
|
+
self._backend.close_transport(transport)
|
|
134
|
+
raise
|
|
135
|
+
|
|
136
|
+
# The tunnel's IO, and whatever was read past the CONNECT response (the
|
|
137
|
+
# start of the origin's TLS conversation), go to the backend's TLS layer
|
|
138
|
+
# (httpunk >= 0.4.0). On a handshake failure the backend closes the
|
|
139
|
+
# transport itself before raising; `__call__` maps the error.
|
|
140
|
+
stream, read_buf = tunnel.downcast()
|
|
141
|
+
try:
|
|
142
|
+
tls, selected = await self._backend.wrap_tls(
|
|
143
|
+
stream, server_hostname=origin.host, ssl_context=self._ssl_context, prefix=read_buf
|
|
144
|
+
)
|
|
145
|
+
except TypeError as exc:
|
|
146
|
+
# the backend refuses the stream at setup (tonio: no TLS over TLS, so an
|
|
147
|
+
# https:// proxy cannot carry a CONNECT tunnel there)
|
|
148
|
+
self._backend.close_transport(stream)
|
|
149
|
+
raise ProxyError(f"Cannot open a TLS tunnel to {origin} via proxy {intercept.uri}: {exc}")
|
|
150
|
+
|
|
151
|
+
if selected == "h2" or not self._http1:
|
|
152
|
+
return H2Connection(tls, authority=origin.authority, scheme="https", backend=self._backend)
|
|
153
|
+
return H1Connection(tls, authority=origin.authority, backend=self._backend)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.3"
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "punkreq"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.3"
|
|
8
8
|
description = "An async HTTP client built on httpunk"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "BSD-3-Clause"
|
|
@@ -32,12 +32,12 @@ classifiers = [
|
|
|
32
32
|
]
|
|
33
33
|
|
|
34
34
|
dependencies = [
|
|
35
|
-
"httpunk~=0.
|
|
35
|
+
"httpunk~=0.4.4",
|
|
36
36
|
]
|
|
37
37
|
|
|
38
38
|
[project.optional-dependencies]
|
|
39
39
|
brotli = ["brotli~=1.2.0"]
|
|
40
|
-
tonio = ["tonio~=0.
|
|
40
|
+
tonio = ["tonio~=0.10.0; python_version >= '3.14'"]
|
|
41
41
|
|
|
42
42
|
[dependency-groups]
|
|
43
43
|
lint = [
|
|
@@ -45,6 +45,7 @@ lint = [
|
|
|
45
45
|
]
|
|
46
46
|
test = [
|
|
47
47
|
"pytest>=7.4",
|
|
48
|
+
"pytest-asyncio>=1.0",
|
|
48
49
|
"trustme>=1.2.1",
|
|
49
50
|
]
|
|
50
51
|
|
|
@@ -54,6 +55,9 @@ Funding = 'https://github.com/sponsors/gi0baro'
|
|
|
54
55
|
Source = 'https://github.com/gi0baro/punkreq'
|
|
55
56
|
Issues = 'https://github.com/gi0baro/punkreq/issues'
|
|
56
57
|
|
|
58
|
+
[tool.pytest.ini_options]
|
|
59
|
+
asyncio_mode = "strict"
|
|
60
|
+
|
|
57
61
|
[tool.hatch.build.targets.sdist]
|
|
58
62
|
include = [
|
|
59
63
|
'/README.md',
|
|
File without changes
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
import pytest_asyncio
|
|
3
|
+
import trustme
|
|
4
|
+
from httpunk import Backend
|
|
5
|
+
|
|
6
|
+
from punkreq import Limits
|
|
7
|
+
from punkreq._pool import ConnectionPool
|
|
8
|
+
from punkreq.asyncio import Client
|
|
9
|
+
from tests.fakes import FakeConnector
|
|
10
|
+
from tests.servers import RawProxy, start_server
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@pytest.fixture(scope="session")
|
|
14
|
+
def ca():
|
|
15
|
+
"""A throwaway certificate authority for TLS peers."""
|
|
16
|
+
return trustme.CA()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@pytest.fixture
|
|
20
|
+
def make_client():
|
|
21
|
+
"""`(client, connector)`: a Client over a fake connector whose connections
|
|
22
|
+
answer every request from `handler(request, conn)`."""
|
|
23
|
+
|
|
24
|
+
def factory(handler, *, multiplexed=False, **kwargs):
|
|
25
|
+
connector = FakeConnector(handler, multiplexed=multiplexed)
|
|
26
|
+
return Client(connector=connector, **kwargs), connector
|
|
27
|
+
|
|
28
|
+
return factory
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture
|
|
32
|
+
def make_pool():
|
|
33
|
+
"""A ConnectionPool over `connector` on the asyncio backend (or `backend`),
|
|
34
|
+
with `Limits(**limits)`."""
|
|
35
|
+
|
|
36
|
+
def factory(connector, *, backend=None, **limits):
|
|
37
|
+
return ConnectionPool(connector, backend=backend or Backend.asyncio.create(), limits=Limits(**limits))
|
|
38
|
+
|
|
39
|
+
return factory
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ----- real peers on the asyncio test's own loop (`@pytest.mark.asyncio` tests only) -----
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest_asyncio.fixture
|
|
46
|
+
async def echo_server():
|
|
47
|
+
"""`await echo_server(ssl_context=None) -> port`: a JSON echo server on this
|
|
48
|
+
test's loop; every server started is closed at teardown."""
|
|
49
|
+
servers = []
|
|
50
|
+
|
|
51
|
+
async def start(ssl_context=None):
|
|
52
|
+
server, port = await start_server(ssl_context=ssl_context)
|
|
53
|
+
servers.append(server)
|
|
54
|
+
return port
|
|
55
|
+
|
|
56
|
+
yield start
|
|
57
|
+
for server in servers:
|
|
58
|
+
server.close()
|
|
59
|
+
if hasattr(server, "close_clients"): # 3.13+: `wait_closed` waits for clients too
|
|
60
|
+
server.close_clients()
|
|
61
|
+
await server.wait_closed()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest_asyncio.fixture
|
|
65
|
+
async def proxy():
|
|
66
|
+
"""`await proxy(connect_status=200) -> RawProxy`: a CONNECT proxy on this
|
|
67
|
+
test's loop; every proxy started is stopped at teardown."""
|
|
68
|
+
proxies = []
|
|
69
|
+
|
|
70
|
+
async def start(connect_status=200):
|
|
71
|
+
started = await RawProxy(connect_status).start()
|
|
72
|
+
proxies.append(started)
|
|
73
|
+
return started
|
|
74
|
+
|
|
75
|
+
yield start
|
|
76
|
+
for started in proxies:
|
|
77
|
+
await started.stop()
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""End-to-end scenarios shared by the asyncio and tonio suites: each takes the
|
|
2
|
+
backend's `Client` class and the peers the suite started, so both backends run
|
|
3
|
+
the same body against their own runtime."""
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
import punkreq
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def client_h1_get_and_post_with_keepalive(client_cls, upstream_port):
|
|
11
|
+
async with client_cls(base_url=f"http://127.0.0.1:{upstream_port}") as client:
|
|
12
|
+
response = await client.get("/hello", params={"a": "1"})
|
|
13
|
+
assert response.status_code == 200
|
|
14
|
+
echo = await response.json()
|
|
15
|
+
assert echo["method"] == "GET"
|
|
16
|
+
assert echo["path"] == "/hello?a=1"
|
|
17
|
+
assert echo["headers"]["host"] == f"127.0.0.1:{upstream_port}"
|
|
18
|
+
assert echo["headers"]["user-agent"].startswith("python-punkreq/")
|
|
19
|
+
|
|
20
|
+
response = await client.post("/items", json={"name": "x"})
|
|
21
|
+
assert (await response.json())["body"] == '{"name":"x"}'
|
|
22
|
+
assert client._pool.connection_count == 1 # keep-alive reuse
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
async def client_h2_prior_knowledge(client_cls, upstream_port):
|
|
26
|
+
async with client_cls(base_url=f"http://127.0.0.1:{upstream_port}", http1=False) as client:
|
|
27
|
+
first = await client.get("/a")
|
|
28
|
+
second = await client.get("/b")
|
|
29
|
+
assert first.http_version == "HTTP/2"
|
|
30
|
+
assert {(await first.json())["path"], (await second.json())["path"]} == {"/a", "/b"}
|
|
31
|
+
assert client._pool.connection_count == 1 # one multiplexed connection
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
async def tunnel_https_h1(client_cls, ca, upstream_port, proxy):
|
|
35
|
+
with ca.cert_pem.tempfile() as ca_path:
|
|
36
|
+
async with client_cls(proxy=f"http://127.0.0.1:{proxy.port}", verify=ca_path) as client:
|
|
37
|
+
response = await client.get(f"https://127.0.0.1:{upstream_port}/tunneled")
|
|
38
|
+
assert response.status_code == 200
|
|
39
|
+
assert (await response.json())["path"] == "/tunneled"
|
|
40
|
+
assert response.http_version == "HTTP/1.1"
|
|
41
|
+
# a second request reuses the tunnel: the h1 send-time peek runs on the wrapped stream
|
|
42
|
+
response = await client.get(f"https://127.0.0.1:{upstream_port}/again")
|
|
43
|
+
assert (await response.json())["path"] == "/again"
|
|
44
|
+
assert client._pool.connection_count == 1
|
|
45
|
+
assert proxy.connect_targets == [f"127.0.0.1:{upstream_port}"]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
async def tunnel_https_h2(client_cls, ca, upstream_port, proxy):
|
|
49
|
+
with ca.cert_pem.tempfile() as ca_path:
|
|
50
|
+
async with client_cls(proxy=f"http://127.0.0.1:{proxy.port}", verify=ca_path) as client:
|
|
51
|
+
first = await client.get(f"https://127.0.0.1:{upstream_port}/a")
|
|
52
|
+
second = await client.get(f"https://127.0.0.1:{upstream_port}/b")
|
|
53
|
+
assert first.http_version == "HTTP/2"
|
|
54
|
+
assert {(await first.json())["path"], (await second.json())["path"]} == {"/a", "/b"}
|
|
55
|
+
assert client._pool.connection_count == 1 # one multiplexed tunnel
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
async def tunnel_handshake_failure_raises_connect_error(client_cls, upstream_port, proxy):
|
|
59
|
+
"""The origin's certificate is not trusted (system store): the backend's TLS
|
|
60
|
+
layer fails the handshake inside the tunnel and closes it; punkreq reports it
|
|
61
|
+
as a connect failure, like a failed direct dial."""
|
|
62
|
+
async with client_cls(proxy=f"http://127.0.0.1:{proxy.port}") as client:
|
|
63
|
+
with pytest.raises(punkreq.ConnectError, match="via proxy"):
|
|
64
|
+
await client.get(f"https://127.0.0.1:{upstream_port}/untrusted")
|
|
65
|
+
assert proxy.connect_targets == [f"127.0.0.1:{upstream_port}"]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
async def direct_handshake_failure_raises_connect_error(client_cls, upstream_port):
|
|
69
|
+
"""The same on a direct dial: whatever the backend raises for a failed
|
|
70
|
+
handshake surfaces as punkreq's `ConnectError`."""
|
|
71
|
+
async with client_cls() as client:
|
|
72
|
+
with pytest.raises(punkreq.ConnectError):
|
|
73
|
+
await client.get(f"https://127.0.0.1:{upstream_port}/untrusted")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
async def tunnel_refused_raises_proxy_error(client_cls, proxy):
|
|
77
|
+
async with client_cls(proxy=f"http://127.0.0.1:{proxy.port}") as client:
|
|
78
|
+
with pytest.raises(punkreq.ProxyError):
|
|
79
|
+
await client.get("https://denied.example/")
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Stand-ins for the httpunk objects punkreq drives: a response, a pool-facing
|
|
2
|
+
connection and a connector, plus a backend wrapper with a deterministic clock."""
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
|
|
6
|
+
from httpunk import HeaderMap, Version
|
|
7
|
+
|
|
8
|
+
import punkreq
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class FakeHttpunkResponse:
|
|
12
|
+
"""The slice of an httpunk response punkreq reads: status, headers, version
|
|
13
|
+
and a body iterated once through `aiter_bytes`, then `aclose`d. Records how
|
|
14
|
+
it was driven (`aclose_calls`, `iter_finalized`); `aclose_error` makes the
|
|
15
|
+
close itself fail."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, status=200, headers=None, body=b"", *, chunk_delay=0.0, aclose_error=None):
|
|
18
|
+
self.status = status
|
|
19
|
+
self.headers = HeaderMap(headers or {})
|
|
20
|
+
self.version = Version.HTTP_11 # stamped per protocol by FakeConnection.send_request
|
|
21
|
+
self._chunks = [body] if isinstance(body, bytes) else list(body)
|
|
22
|
+
self._chunk_delay = chunk_delay
|
|
23
|
+
self._aclose_error = aclose_error
|
|
24
|
+
self._conn = None # set by FakeConnection.send_request
|
|
25
|
+
self._consumed = False
|
|
26
|
+
self.closed = False
|
|
27
|
+
self.aclose_calls = 0
|
|
28
|
+
self.iter_finalized = False
|
|
29
|
+
|
|
30
|
+
async def aiter_bytes(self):
|
|
31
|
+
try:
|
|
32
|
+
for chunk in self._chunks:
|
|
33
|
+
if self._chunk_delay:
|
|
34
|
+
await asyncio.sleep(self._chunk_delay)
|
|
35
|
+
yield chunk
|
|
36
|
+
self._consumed = True
|
|
37
|
+
finally:
|
|
38
|
+
self.iter_finalized = True
|
|
39
|
+
|
|
40
|
+
async def aclose(self):
|
|
41
|
+
self.aclose_calls += 1
|
|
42
|
+
self.closed = True
|
|
43
|
+
# mirror httpunk h1: aborting a partially-read body closes the connection
|
|
44
|
+
if not self._consumed and self._conn is not None and not self._conn.multiplexed:
|
|
45
|
+
self._conn.closed = True
|
|
46
|
+
if self._aclose_error is not None:
|
|
47
|
+
raise self._aclose_error
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class FakeConnection:
|
|
51
|
+
"""A pool-facing connection. `handler(request, conn)` answers each
|
|
52
|
+
`send_request` (a FakeHttpunkResponse, an awaitable of one, or by raising); with no
|
|
53
|
+
handler a streamed request body is drained and a 200 "ok" returned.
|
|
54
|
+
`fail_next` makes the next send raise that exception instead."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, handler=None, *, multiplexed=False):
|
|
57
|
+
self.handler = handler
|
|
58
|
+
self.multiplexed = multiplexed
|
|
59
|
+
self.closed = False
|
|
60
|
+
self.busy = False # httpunk: an exchange still holds the in-flight slot
|
|
61
|
+
self.entered = False
|
|
62
|
+
self.fail_next = None
|
|
63
|
+
self.requests = []
|
|
64
|
+
self.bodies_touched = 0
|
|
65
|
+
|
|
66
|
+
async def __aenter__(self):
|
|
67
|
+
self.entered = True
|
|
68
|
+
return self
|
|
69
|
+
|
|
70
|
+
async def __aexit__(self, exc_type, exc_value, exc_tb):
|
|
71
|
+
self.closed = True
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
async def ready(self):
|
|
75
|
+
if self.closed:
|
|
76
|
+
raise RuntimeError("connection is closed")
|
|
77
|
+
|
|
78
|
+
async def send_request(self, request):
|
|
79
|
+
self.requests.append(request)
|
|
80
|
+
if self.fail_next is not None:
|
|
81
|
+
exc, self.fail_next = self.fail_next, None
|
|
82
|
+
raise exc
|
|
83
|
+
if self.handler is not None:
|
|
84
|
+
result = self.handler(request, self)
|
|
85
|
+
if asyncio.iscoroutine(result):
|
|
86
|
+
result = await result
|
|
87
|
+
else:
|
|
88
|
+
if request.body is not None and not isinstance(request.body, bytes):
|
|
89
|
+
self.bodies_touched += 1
|
|
90
|
+
async for _ in request.body:
|
|
91
|
+
pass
|
|
92
|
+
result = FakeHttpunkResponse(200, body=b"ok")
|
|
93
|
+
if isinstance(result, FakeHttpunkResponse):
|
|
94
|
+
result._conn = self
|
|
95
|
+
result.version = Version.HTTP_2 if self.multiplexed else Version.HTTP_11
|
|
96
|
+
return result
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class FakeConnector:
|
|
100
|
+
"""Dials a fresh FakeConnection per call and records them. `delay` sleeps
|
|
101
|
+
before each dial; `fail` makes that many dials raise ConnectError first."""
|
|
102
|
+
|
|
103
|
+
def __init__(self, handler=None, *, multiplexed=False, delay=0.0, fail=0):
|
|
104
|
+
self.handler = handler
|
|
105
|
+
self.multiplexed = multiplexed
|
|
106
|
+
self.delay = delay
|
|
107
|
+
self.fail = fail
|
|
108
|
+
self.dials = 0
|
|
109
|
+
self.connections = []
|
|
110
|
+
|
|
111
|
+
async def __call__(self, origin):
|
|
112
|
+
self.dials += 1
|
|
113
|
+
if self.delay:
|
|
114
|
+
await asyncio.sleep(self.delay)
|
|
115
|
+
if self.fail > 0:
|
|
116
|
+
self.fail -= 1
|
|
117
|
+
raise punkreq.ConnectError(f"boom dialing {origin}")
|
|
118
|
+
conn = FakeConnection(self.handler, multiplexed=self.multiplexed)
|
|
119
|
+
self.connections.append(conn)
|
|
120
|
+
return conn
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def requests(self):
|
|
124
|
+
"""Every request sent, across all connections, in order."""
|
|
125
|
+
return [request for conn in self.connections for request in conn.requests]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def ok_handler(body=b"hello", headers=None):
|
|
129
|
+
"""A connection handler answering every request with one 200 response."""
|
|
130
|
+
|
|
131
|
+
def handler(request, conn):
|
|
132
|
+
return FakeHttpunkResponse(200, headers=headers, body=body)
|
|
133
|
+
|
|
134
|
+
return handler
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class SteppingBackend:
|
|
138
|
+
"""Delegates to a real backend, but `monotonic()` strictly increases on every
|
|
139
|
+
call: on coarse OS clocks (Windows, ~16ms) consecutive releases can get equal
|
|
140
|
+
idle timestamps, making eviction order tie-dependent."""
|
|
141
|
+
|
|
142
|
+
def __init__(self, backend):
|
|
143
|
+
self._backend = backend
|
|
144
|
+
self._now = 0.0
|
|
145
|
+
|
|
146
|
+
def monotonic(self):
|
|
147
|
+
self._now += 1.0
|
|
148
|
+
return self._now
|
|
149
|
+
|
|
150
|
+
def __getattr__(self, name):
|
|
151
|
+
return getattr(self._backend, name)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Real peers for the end-to-end tests: an httpunk echo server on asyncio, its
|
|
2
|
+
TLS context, and a minimal CONNECT proxy."""
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
import contextlib
|
|
6
|
+
import json
|
|
7
|
+
import ssl
|
|
8
|
+
|
|
9
|
+
import httpunk.asyncio
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class EchoJsonProtocol(httpunk.asyncio.AutoServerProtocol):
|
|
13
|
+
"""Answers every request with a JSON echo of its method, path, headers and body."""
|
|
14
|
+
|
|
15
|
+
async def handle(self, request):
|
|
16
|
+
body = await request.read()
|
|
17
|
+
payload = {
|
|
18
|
+
"method": request.method,
|
|
19
|
+
"path": request.path,
|
|
20
|
+
"headers": {key: value.decode("latin-1") for key, value in request.headers.items()},
|
|
21
|
+
"body": body.decode("utf-8", errors="replace"),
|
|
22
|
+
}
|
|
23
|
+
await request.respond(200, headers={"content-type": "application/json"}, body=json.dumps(payload).encode())
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def start_server(protocol=EchoJsonProtocol, ssl_context=None):
|
|
27
|
+
"""`(server, port)` for `protocol` listening on a free loopback port."""
|
|
28
|
+
loop = asyncio.get_running_loop()
|
|
29
|
+
server = await loop.create_server(protocol, "127.0.0.1", 0, ssl=ssl_context)
|
|
30
|
+
return server, server.sockets[0].getsockname()[1]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def server_tls_context(ca, alpn):
|
|
34
|
+
"""A server-side TLS context with a `ca`-issued loopback certificate offering `alpn`."""
|
|
35
|
+
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
|
36
|
+
ca.issue_cert("localhost", "127.0.0.1").configure_cert(context)
|
|
37
|
+
context.set_alpn_protocols(alpn)
|
|
38
|
+
return context
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class RawProxy:
|
|
42
|
+
"""A minimal HTTP proxy on asyncio streams that relays CONNECT tunnels to the
|
|
43
|
+
requested destination (or refuses them with `connect_status`)."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, connect_status=200):
|
|
46
|
+
self.connect_status = connect_status
|
|
47
|
+
self.connect_targets: list[str] = []
|
|
48
|
+
self.server = None
|
|
49
|
+
self.port = None
|
|
50
|
+
|
|
51
|
+
async def start(self):
|
|
52
|
+
self.server = await asyncio.start_server(self._handle, "127.0.0.1", 0)
|
|
53
|
+
self.port = self.server.sockets[0].getsockname()[1]
|
|
54
|
+
return self
|
|
55
|
+
|
|
56
|
+
async def stop(self):
|
|
57
|
+
self.server.close()
|
|
58
|
+
await self.server.wait_closed()
|
|
59
|
+
|
|
60
|
+
async def _handle(self, reader, writer):
|
|
61
|
+
head = b""
|
|
62
|
+
while b"\r\n\r\n" not in head:
|
|
63
|
+
chunk = await reader.read(65536)
|
|
64
|
+
if not chunk:
|
|
65
|
+
writer.close()
|
|
66
|
+
return
|
|
67
|
+
head += chunk
|
|
68
|
+
head, _, leftover = head.partition(b"\r\n\r\n")
|
|
69
|
+
method, target, _ = head.split(b"\r\n")[0].decode().split(" ")
|
|
70
|
+
if method != "CONNECT":
|
|
71
|
+
writer.write(b"HTTP/1.1 405 Method Not Allowed\r\ncontent-length: 0\r\n\r\n")
|
|
72
|
+
await writer.drain()
|
|
73
|
+
writer.close()
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
self.connect_targets.append(target)
|
|
77
|
+
if self.connect_status != 200:
|
|
78
|
+
writer.write(f"HTTP/1.1 {self.connect_status} Denied\r\ncontent-length: 0\r\n\r\n".encode())
|
|
79
|
+
await writer.drain()
|
|
80
|
+
writer.close()
|
|
81
|
+
return
|
|
82
|
+
host, _, port = target.rpartition(":")
|
|
83
|
+
upstream_reader, upstream_writer = await asyncio.open_connection(host.strip("[]"), int(port))
|
|
84
|
+
writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
|
85
|
+
await writer.drain()
|
|
86
|
+
if leftover:
|
|
87
|
+
upstream_writer.write(leftover)
|
|
88
|
+
await upstream_writer.drain()
|
|
89
|
+
|
|
90
|
+
async def pipe(src, dst):
|
|
91
|
+
try:
|
|
92
|
+
while data := await src.read(65536):
|
|
93
|
+
dst.write(data)
|
|
94
|
+
await dst.drain()
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
finally:
|
|
98
|
+
with contextlib.suppress(Exception):
|
|
99
|
+
dst.close()
|
|
100
|
+
|
|
101
|
+
await asyncio.gather(pipe(reader, upstream_writer), pipe(upstream_reader, writer))
|