headgate 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- headgate/__init__.py +131 -0
- headgate/_attempts.py +50 -0
- headgate/_auth.py +504 -0
- headgate/_body.py +115 -0
- headgate/_client.py +927 -0
- headgate/_codec.py +24 -0
- headgate/_core/__init__.py +88 -0
- headgate/_core/_context.py +228 -0
- headgate/_core/_events.py +248 -0
- headgate/_core/_origin.py +39 -0
- headgate/_core/_retry_after.py +76 -0
- headgate/_core/_seams.py +50 -0
- headgate/_core/_switch.py +40 -0
- headgate/_core/_time.py +102 -0
- headgate/_curl.py +287 -0
- headgate/_deprecation.py +101 -0
- headgate/_failures.py +439 -0
- headgate/_middleware.py +90 -0
- headgate/_options.py +358 -0
- headgate/_redirects.py +86 -0
- headgate/_request_body.py +81 -0
- headgate/_responses.py +290 -0
- headgate/_stack.py +185 -0
- headgate/_url.py +258 -0
- headgate/_user_agent.py +30 -0
- headgate/_view.py +105 -0
- headgate/breaker/__init__.py +24 -0
- headgate/breaker/_breaker.py +302 -0
- headgate/breaker/_policy.py +37 -0
- headgate/breaker/_registry.py +168 -0
- headgate/breaker/_scope.py +43 -0
- headgate/cache/__init__.py +35 -0
- headgate/cache/_conditional.py +89 -0
- headgate/cache/_credentials.py +143 -0
- headgate/cache/_policy.py +131 -0
- headgate/cache/_transport.py +568 -0
- headgate/curl.py +8 -0
- headgate/errors.py +329 -0
- headgate/events.py +45 -0
- headgate/health.py +73 -0
- headgate/msgspec.py +125 -0
- headgate/otel.py +236 -0
- headgate/pagination/__init__.py +25 -0
- headgate/pagination/_link.py +141 -0
- headgate/pagination/_pager.py +197 -0
- headgate/pagination/_paginated.py +82 -0
- headgate/policy.py +43 -0
- headgate/py.typed +0 -0
- headgate/pydantic.py +131 -0
- headgate/ratelimit/__init__.py +23 -0
- headgate/ratelimit/_budget.py +275 -0
- headgate/ratelimit/_budgets.py +62 -0
- headgate/ratelimit/_feedback.py +133 -0
- headgate/ratelimit/_fields.py +187 -0
- headgate/ratelimit/_inflight.py +92 -0
- headgate/ratelimit/_rate.py +103 -0
- headgate/ratelimit/_transport.py +106 -0
- headgate/retry/__init__.py +40 -0
- headgate/retry/_budget.py +73 -0
- headgate/retry/_classify.py +135 -0
- headgate/retry/_policy.py +254 -0
- headgate/retry/_tracking.py +121 -0
- headgate/retry/_transport.py +448 -0
- headgate/security/__init__.py +36 -0
- headgate/security/_addresses.py +120 -0
- headgate/security/_credentials.py +124 -0
- headgate/security/_downgrade.py +42 -0
- headgate/security/_transport.py +119 -0
- headgate/streaming/__init__.py +12 -0
- headgate/streaming/_stream.py +164 -0
- headgate/testing/__init__.py +58 -0
- headgate/testing/_breakers.py +33 -0
- headgate/testing/_recording.py +120 -0
- headgate/testing/_time.py +84 -0
- headgate/testing/_transport.py +115 -0
- headgate/testing/faults.py +275 -0
- headgate-0.2.0.dist-info/METADATA +373 -0
- headgate-0.2.0.dist-info/RECORD +80 -0
- headgate-0.2.0.dist-info/WHEEL +4 -0
- headgate-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Copyright (c) 2026 Khaled Elsayed
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""How long a response asks its client to wait: ``Retry-After`` (RFC 9110 section 10.2.3) and ``retry-after-ms``.
|
|
5
|
+
|
|
6
|
+
Two layers act on the same answer, so it is read in one place: the retry layer waits it before re-sending, and the rate
|
|
7
|
+
budget pauses every sender sharing it until it has passed. Reading it the same way is what makes the two waits one.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
import re
|
|
14
|
+
import time
|
|
15
|
+
from datetime import UTC
|
|
16
|
+
from email.utils import parsedate_to_datetime
|
|
17
|
+
from typing import TYPE_CHECKING, Final
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
import httpx2
|
|
21
|
+
|
|
22
|
+
__all__ = ["RETRY_AFTER_STATUSES", "parse_retry_after", "retry_after_of"]
|
|
23
|
+
|
|
24
|
+
#: The statuses a ``Retry-After`` means something on: rate limited (RFC 6585) and unavailable (RFC 9110).
|
|
25
|
+
RETRY_AFTER_STATUSES: Final = frozenset({429, 503})
|
|
26
|
+
|
|
27
|
+
#: A non-negative decimal number of seconds: ``Retry-After`` defines whole seconds, and fractions are accepted too.
|
|
28
|
+
_DECIMAL: Final = re.compile(r"\d+(?:\.\d+)?")
|
|
29
|
+
|
|
30
|
+
#: Milliseconds in a second, for ``retry-after-ms``.
|
|
31
|
+
_MS_PER_SECOND: Final = 1000.0
|
|
32
|
+
|
|
33
|
+
#: What ``parsedate_to_datetime`` raises for a value it cannot parse.
|
|
34
|
+
_DATE_ERRORS: Final = (TypeError, ValueError)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _decimal_seconds(value: str) -> float | None:
|
|
38
|
+
"""Return *value* as a finite non-negative number, or ``None`` when it is not a plain ASCII decimal."""
|
|
39
|
+
if not (value.isascii() and _DECIMAL.fullmatch(value)):
|
|
40
|
+
return None
|
|
41
|
+
seconds = float(value)
|
|
42
|
+
return seconds if math.isfinite(seconds) else None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def parse_retry_after(value: str, *, now: float) -> float | None:
|
|
46
|
+
"""Return a ``Retry-After`` value as seconds from *now* (a Unix time); ``None`` when it is in neither form.
|
|
47
|
+
|
|
48
|
+
Whole and fractional seconds are read as they are. An HTTP-date is measured from *now*: a zone-less one is read as
|
|
49
|
+
GMT, which an HTTP-date always is, and a past one is no wait at all.
|
|
50
|
+
"""
|
|
51
|
+
value = value.strip()
|
|
52
|
+
seconds = _decimal_seconds(value)
|
|
53
|
+
if seconds is not None:
|
|
54
|
+
return seconds
|
|
55
|
+
try:
|
|
56
|
+
when = parsedate_to_datetime(value)
|
|
57
|
+
except _DATE_ERRORS:
|
|
58
|
+
return None
|
|
59
|
+
if when.tzinfo is None:
|
|
60
|
+
when = when.replace(tzinfo=UTC)
|
|
61
|
+
return max(0.0, when.timestamp() - now)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def retry_after_of(response: httpx2.Response, *, now: float | None = None) -> float | None:
|
|
65
|
+
"""Return the seconds a 429 or 503 *response* asks to wait: ``retry-after-ms`` if it parses, else ``Retry-After``.
|
|
66
|
+
|
|
67
|
+
``None`` for any other status, which RFC 9110 and RFC 6585 give the header no meaning on, and when neither header
|
|
68
|
+
is present or parses. *now* is the Unix time an HTTP-date is measured from, the system's wall clock by default.
|
|
69
|
+
"""
|
|
70
|
+
if response.status_code not in RETRY_AFTER_STATUSES:
|
|
71
|
+
return None
|
|
72
|
+
millis = _decimal_seconds(response.headers.get("retry-after-ms", "").strip())
|
|
73
|
+
if millis is not None:
|
|
74
|
+
return millis / _MS_PER_SECOND
|
|
75
|
+
header = response.headers.get("retry-after")
|
|
76
|
+
return None if header is None else parse_retry_after(header, now=time.time() if now is None else now)
|
headgate/_core/_seams.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Copyright (c) 2026 Khaled Elsayed
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""Seams two policy areas meet at, defined once so neither has to import the other.
|
|
5
|
+
|
|
6
|
+
* :class:`CredentialStamper`: what an auth declares about the credentials it puts on a request. The credential guard
|
|
7
|
+
uses it to keep them on trusted origins; the cache uses it to partition entries by credential and to keep
|
|
8
|
+
credentials out of storage. Its first two members, :class:`CredentialCarrier`, are what a signing middleware
|
|
9
|
+
declares, for the guard alone.
|
|
10
|
+
* :data:`BREAKER_GATE`: set by the client on every call, read by the retry layer, so a retry sequence stops as soon as
|
|
11
|
+
the origin's breaker opens without the retry layer knowing about breakers.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
from typing import Protocol, runtime_checkable
|
|
18
|
+
|
|
19
|
+
from headgate._core._context import Slot
|
|
20
|
+
|
|
21
|
+
__all__ = ["BREAKER_GATE", "CredentialCarrier", "CredentialStamper"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@runtime_checkable
|
|
25
|
+
class CredentialCarrier(Protocol):
|
|
26
|
+
"""Something that declares the credentials it puts on a request: an auth, or a signing middleware."""
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def credential_headers(self) -> frozenset[str]:
|
|
30
|
+
"""Return the names of the headers it sets (compared case-insensitively)."""
|
|
31
|
+
...
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def credential_params(self) -> frozenset[str]:
|
|
35
|
+
"""Return the names of the query parameters it sets."""
|
|
36
|
+
...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@runtime_checkable
|
|
40
|
+
class CredentialStamper(CredentialCarrier, Protocol):
|
|
41
|
+
"""An auth that declares the credentials it stamps on a request, and names the credential they stand for."""
|
|
42
|
+
|
|
43
|
+
def credential_identity(self) -> bytes:
|
|
44
|
+
"""Return stable bytes identifying the credential (its secret material); never stored, only hashed."""
|
|
45
|
+
...
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
#: Answers whether the breaker for the call's origin is refusing right now. The client puts it on every call's
|
|
49
|
+
#: context; the retry layer checks it before scheduling another attempt.
|
|
50
|
+
BREAKER_GATE = Slot[Callable[[], bool]]("breaker_gate")
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Copyright (c) 2026 Khaled Elsayed
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""The construction-time switch that builds clients with their transport policy off.
|
|
5
|
+
|
|
6
|
+
It lives here, below every layer, because a rate budget honours it when it is built, whoever builds it, and the
|
|
7
|
+
policy subpackages may import only this core. :mod:`headgate.policy` is its public face and documents what "off" means.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from contextvars import ContextVar
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from collections.abc import Generator
|
|
18
|
+
|
|
19
|
+
__all__ = ["policy_disabled", "policy_is_disabled"]
|
|
20
|
+
|
|
21
|
+
#: Task-local: is transport policy switched off for clients built right now?
|
|
22
|
+
_POLICY_OFF: ContextVar[bool] = ContextVar("headgate_policy_off", default=False)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@contextmanager
|
|
26
|
+
def policy_disabled() -> Generator[None]:
|
|
27
|
+
"""Build clients with retry, the circuit breaker, and rate limiting switched off.
|
|
28
|
+
|
|
29
|
+
Affects only clients constructed inside the block. Safe to nest.
|
|
30
|
+
"""
|
|
31
|
+
token = _POLICY_OFF.set(True)
|
|
32
|
+
try:
|
|
33
|
+
yield
|
|
34
|
+
finally:
|
|
35
|
+
_POLICY_OFF.reset(token)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def policy_is_disabled() -> bool:
|
|
39
|
+
"""Return whether a client being constructed right now should skip its transport policy."""
|
|
40
|
+
return _POLICY_OFF.get()
|
headgate/_core/_time.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Copyright (c) 2026 Khaled Elsayed
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""Time and randomness as parameters: the clock, the sleep and the random source every layer is handed.
|
|
5
|
+
|
|
6
|
+
Each layer that waits, measures or draws takes the function it needs as a parameter with a production default, so a
|
|
7
|
+
test drives time instead of waiting for it. The three shapes are protocols over plain callables, so ``time.monotonic``,
|
|
8
|
+
``asyncio.sleep`` and ``random.random`` satisfy them as they are.
|
|
9
|
+
|
|
10
|
+
``AsyncBaseClient`` reads the :class:`Timing` it hands its layers from :func:`current_timing` when it is built. A client
|
|
11
|
+
class usually fixes its own options in ``super().__init__()`` and exposes none of them, so a test replaces the timing
|
|
12
|
+
of the clients it builds with :func:`timing_override` instead of reaching into each one.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
import random
|
|
19
|
+
import time
|
|
20
|
+
from contextlib import contextmanager
|
|
21
|
+
from contextvars import ContextVar
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from typing import TYPE_CHECKING, Protocol
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from collections.abc import Awaitable, Generator
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"SYSTEM_TIMING",
|
|
30
|
+
"Clock",
|
|
31
|
+
"RandomSource",
|
|
32
|
+
"Sleep",
|
|
33
|
+
"Timing",
|
|
34
|
+
"asyncio_sleep",
|
|
35
|
+
"current_timing",
|
|
36
|
+
"system_random",
|
|
37
|
+
"timing_override",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Clock(Protocol):
|
|
42
|
+
"""Return monotonic seconds: only differences between two readings mean anything."""
|
|
43
|
+
|
|
44
|
+
def __call__(self) -> float:
|
|
45
|
+
"""Return the current reading."""
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Sleep(Protocol):
|
|
50
|
+
"""Wait *seconds* without blocking the event loop."""
|
|
51
|
+
|
|
52
|
+
def __call__(self, seconds: float, /) -> Awaitable[None]:
|
|
53
|
+
"""Return an awaitable that completes after *seconds*."""
|
|
54
|
+
...
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class RandomSource(Protocol):
|
|
58
|
+
"""Return a uniform draw in ``[0, 1)``."""
|
|
59
|
+
|
|
60
|
+
def __call__(self) -> float:
|
|
61
|
+
"""Return the next draw."""
|
|
62
|
+
...
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def asyncio_sleep(seconds: float, /) -> None:
|
|
66
|
+
"""Sleep through ``asyncio.sleep``, looked up on every call so a process-wide patch of it still applies."""
|
|
67
|
+
await asyncio.sleep(seconds)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
#: The operating system's generator, not the ``random`` module's shared one: an application that seeds ``random`` (for
|
|
71
|
+
#: reproducible tests, say) would otherwise give every process it runs the same jitter, which exists to spread them.
|
|
72
|
+
system_random: RandomSource = random.SystemRandom().random
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
76
|
+
class Timing:
|
|
77
|
+
"""The clock and the sleep a client hands every layer it builds."""
|
|
78
|
+
|
|
79
|
+
clock: Clock = field(default=time.monotonic)
|
|
80
|
+
sleep: Sleep = field(default=asyncio_sleep)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
#: Production timing: ``time.monotonic`` and ``asyncio.sleep``.
|
|
84
|
+
SYSTEM_TIMING = Timing()
|
|
85
|
+
|
|
86
|
+
#: The timing clients built in this context take. A ``ContextVar`` so concurrent tasks cannot see each other's override.
|
|
87
|
+
_TIMING: ContextVar[Timing] = ContextVar("headgate_timing", default=SYSTEM_TIMING)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def current_timing() -> Timing:
|
|
91
|
+
"""Return the timing a client constructed right now takes."""
|
|
92
|
+
return _TIMING.get()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@contextmanager
|
|
96
|
+
def timing_override(timing: Timing) -> Generator[None]:
|
|
97
|
+
"""Build clients with *timing* instead of :data:`SYSTEM_TIMING`; only clients constructed inside the block."""
|
|
98
|
+
token = _TIMING.set(timing)
|
|
99
|
+
try:
|
|
100
|
+
yield
|
|
101
|
+
finally:
|
|
102
|
+
_TIMING.reset(token)
|
headgate/_curl.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
# Copyright (c) 2026 Khaled Elsayed
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
#
|
|
4
|
+
# This file is derived from httpx-curl-cffi 0.1.5 (https://github.com/vgavro/httpx-curl-cffi), under its license:
|
|
5
|
+
#
|
|
6
|
+
# BSD 3-Clause License
|
|
7
|
+
#
|
|
8
|
+
# Copyright (c) 2025, Victor Gavro
|
|
9
|
+
#
|
|
10
|
+
# Redistribution and use in source and binary forms, with or without
|
|
11
|
+
# modification, are permitted provided that the following conditions are met:
|
|
12
|
+
#
|
|
13
|
+
# 1. Redistributions of source code must retain the above copyright notice, this
|
|
14
|
+
# list of conditions and the following disclaimer.
|
|
15
|
+
#
|
|
16
|
+
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
17
|
+
# this list of conditions and the following disclaimer in the documentation
|
|
18
|
+
# and/or other materials provided with the distribution.
|
|
19
|
+
#
|
|
20
|
+
# 3. Neither the name of the copyright holder nor the names of its
|
|
21
|
+
# contributors may be used to endorse or promote products derived from
|
|
22
|
+
# this software without specific prior written permission.
|
|
23
|
+
#
|
|
24
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
25
|
+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
26
|
+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
27
|
+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
28
|
+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
29
|
+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
30
|
+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
31
|
+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
32
|
+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
33
|
+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
34
|
+
|
|
35
|
+
"""The ``curl_cffi`` transport for hosts fronted by TLS-fingerprinting CDNs (the ``headgate[curl]`` extra).
|
|
36
|
+
|
|
37
|
+
Vendored from ``httpx-curl-cffi`` 0.1.5 (BSD-3-Clause, Copyright (c) 2025 Victor Gavro; the licence is the comment block
|
|
38
|
+
above, as clause 1 requires) and cut down to what headgate calls. The upstream package speaks plain ``httpx``; this copy
|
|
39
|
+
speaks ``httpx2``. ``curl_cffi`` itself ships no httpx or httpx2 transport.
|
|
40
|
+
|
|
41
|
+
The kept surface is ``impersonate``, ``default_headers``, ``curl_options`` and ``max_connections``. The sync transport,
|
|
42
|
+
the fingerprint and proxy parameter bag and the shims for ``curl_cffi`` older than the declared floor are not kept.
|
|
43
|
+
|
|
44
|
+
The transport behaves like a native httpx2 one wherever the layers above it can tell:
|
|
45
|
+
|
|
46
|
+
* curl ignores the environment's proxy (``NOPROXY="*"``), keeps no cookie and follows no redirect, so the stack above
|
|
47
|
+
it (redirect handling, the breaker, retry) sees every hop and every status.
|
|
48
|
+
* curl decompresses the body itself and exposes no raw one (``curl_cffi`` issue 438). The response therefore carries no
|
|
49
|
+
``Content-Encoding`` or ``Content-Length``: its stream already holds the decoded bytes, so httpx2 picks its identity
|
|
50
|
+
decoder and a cache that stores the stream replays it correctly. The headers as received stay on the ``curl_cffi``
|
|
51
|
+
response, in ``response.extensions["curl"]["response"]``.
|
|
52
|
+
* ``default_headers=True`` hands the header set to curl's impersonation profile, so httpx2's own defaults
|
|
53
|
+
(``User-Agent: python-httpx2/...``, ``Accept-Encoding``, ...) and the ``User-Agent`` a client sends without
|
|
54
|
+
``HttpOptions(user_agent=...)`` are removed from the request when the caller left them untouched.
|
|
55
|
+
* A curl error code, raised while sending or while streaming the body, maps onto the httpx2 exception a native
|
|
56
|
+
transport would raise, so the policy layers classify it.
|
|
57
|
+
* Closing a response before its body is read aborts the transfer. ``curl_cffi``'s own ``aclose`` waits for the whole
|
|
58
|
+
body to arrive into an unbounded queue, which would defeat headgate's response size cap.
|
|
59
|
+
|
|
60
|
+
**The SSRF guard** (``HttpOptions(allow_private_hosts=False)``) vets every hop's host before this transport sends it,
|
|
61
|
+
but cannot pin the connection to the address it vetted, because curl resolves the name itself. Pinning would take
|
|
62
|
+
``CURLOPT_RESOLVE`` per request, and ``curl_cffi`` 0.16 sets libcurl options only session-wide, after the handle is
|
|
63
|
+
taken from the pool (an ``await``, so a per-request value would race between concurrent requests), and a resolve entry
|
|
64
|
+
stays in the multi handle's shared DNS cache after the transfer. Over this transport the guard therefore refuses a
|
|
65
|
+
forbidden host, but a rebinding DNS answer between the check and curl's own lookup is not ruled out.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
from __future__ import annotations
|
|
69
|
+
|
|
70
|
+
import asyncio
|
|
71
|
+
import functools
|
|
72
|
+
from typing import TYPE_CHECKING, Any, get_args, override
|
|
73
|
+
|
|
74
|
+
import httpx2
|
|
75
|
+
|
|
76
|
+
from headgate._user_agent import DEFAULT_USER_AGENT
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
from curl_cffi import CurlECode, CurlHttpVersion, CurlOpt
|
|
80
|
+
from curl_cffi.requests.exceptions import RequestException as _RequestError
|
|
81
|
+
from curl_cffi.requests.session import AsyncSession as _AsyncSession
|
|
82
|
+
from curl_cffi.requests.session import HttpMethod
|
|
83
|
+
except ImportError as exc:
|
|
84
|
+
msg = "AsyncCurlTransport needs the curl extra: pip install 'headgate[curl]'"
|
|
85
|
+
raise ImportError(msg) from exc
|
|
86
|
+
|
|
87
|
+
if TYPE_CHECKING:
|
|
88
|
+
from collections.abc import AsyncIterator, Mapping
|
|
89
|
+
|
|
90
|
+
from curl_cffi.requests import Response as _Response
|
|
91
|
+
from curl_cffi.requests.impersonate import BrowserTypeLiteral
|
|
92
|
+
|
|
93
|
+
__all__ = ["AsyncCurlTransport"]
|
|
94
|
+
|
|
95
|
+
#: The httpx2 exception a native transport raises for the same failure. A code missing here becomes a plain
|
|
96
|
+
#: :class:`httpx2.TransportError`; ``OPERATION_TIMEDOUT`` is split by :func:`_timeout_error`. ``READ_ERROR`` and
|
|
97
|
+
#: ``WRITE_ERROR`` are absent on purpose: in libcurl they are local callback failures, not network ones
|
|
98
|
+
#: (``WRITE_ERROR`` is what an aborted transfer reports).
|
|
99
|
+
_CURL_ERRORS: dict[int, type[httpx2.TransportError | httpx2.DecodingError]] = {
|
|
100
|
+
CurlECode.UNSUPPORTED_PROTOCOL: httpx2.UnsupportedProtocol,
|
|
101
|
+
CurlECode.COULDNT_RESOLVE_PROXY: httpx2.ProxyError,
|
|
102
|
+
CurlECode.PROXY: httpx2.ProxyError,
|
|
103
|
+
CurlECode.COULDNT_RESOLVE_HOST: httpx2.ConnectError,
|
|
104
|
+
CurlECode.COULDNT_CONNECT: httpx2.ConnectError,
|
|
105
|
+
CurlECode.QUIC_CONNECT_ERROR: httpx2.ConnectError,
|
|
106
|
+
CurlECode.SSL_CONNECT_ERROR: httpx2.ConnectError,
|
|
107
|
+
CurlECode.PEER_FAILED_VERIFICATION: httpx2.ConnectError,
|
|
108
|
+
CurlECode.SEND_ERROR: httpx2.WriteError,
|
|
109
|
+
CurlECode.RECV_ERROR: httpx2.ReadError,
|
|
110
|
+
CurlECode.GOT_NOTHING: httpx2.RemoteProtocolError,
|
|
111
|
+
CurlECode.WEIRD_SERVER_REPLY: httpx2.RemoteProtocolError,
|
|
112
|
+
CurlECode.PARTIAL_FILE: httpx2.RemoteProtocolError,
|
|
113
|
+
CurlECode.HTTP2: httpx2.RemoteProtocolError,
|
|
114
|
+
CurlECode.HTTP2_STREAM: httpx2.RemoteProtocolError,
|
|
115
|
+
CurlECode.HTTP3: httpx2.RemoteProtocolError,
|
|
116
|
+
CurlECode.BAD_CONTENT_ENCODING: httpx2.DecodingError,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
#: libcurl's messages for a timeout before the connection is up. Message text is the only signal: libcurl reports every
|
|
120
|
+
#: timeout as ``OPERATION_TIMEDOUT``, and the failed handle is reset before the error reaches Python.
|
|
121
|
+
_CONNECT_TIMEOUT_MESSAGES = ("Resolving timed out", "Connection timed out", "Failed to connect to")
|
|
122
|
+
|
|
123
|
+
#: ``CURLINFO_HTTP_VERSION`` values, as httpx2's ``http_version`` extension spells them.
|
|
124
|
+
_HTTP_VERSIONS: dict[int, bytes] = {
|
|
125
|
+
CurlHttpVersion.V1_0: b"HTTP/1.0",
|
|
126
|
+
CurlHttpVersion.V1_1: b"HTTP/1.1",
|
|
127
|
+
CurlHttpVersion.V2_0: b"HTTP/2",
|
|
128
|
+
CurlHttpVersion.V3: b"HTTP/3",
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
#: The response headers that describe the body as it was on the wire, which curl has already decoded.
|
|
132
|
+
_WIRE_ENCODING_HEADERS = ("content-encoding", "content-length")
|
|
133
|
+
|
|
134
|
+
_METHODS: dict[str, HttpMethod] = {method: method for method in get_args(HttpMethod)}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class AsyncCurlTransport(httpx2.AsyncBaseTransport):
|
|
138
|
+
"""Send each request through ``curl_cffi`` with a browser's TLS handshake, as an ``httpx2`` async transport.
|
|
139
|
+
|
|
140
|
+
:param impersonate: the ``curl_cffi`` browser profile; ``"chrome"`` is a rolling alias that each ``curl_cffi``
|
|
141
|
+
release re-points at a newer Chrome, which is why ``curl_cffi`` carries a floor in headgate's manifest.
|
|
142
|
+
:param default_headers: let the profile supply the browser's headers (and drop httpx2's) instead of sending none.
|
|
143
|
+
:param curl_options: extra libcurl options, e.g. ``{CurlOpt.FRESH_CONNECT: True}`` to open a new connection per
|
|
144
|
+
request for hosts that fingerprint a reused one. Values are whatever libcurl takes for the option (``curl_cffi``
|
|
145
|
+
annotates them as ``str``, which is narrower than what it accepts).
|
|
146
|
+
:param max_connections: concurrent curl handles; a request beyond them waits for one, with no pool timeout.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
def __init__(
|
|
150
|
+
self,
|
|
151
|
+
*,
|
|
152
|
+
impersonate: BrowserTypeLiteral,
|
|
153
|
+
default_headers: bool = False,
|
|
154
|
+
curl_options: Mapping[CurlOpt, Any] | None = None,
|
|
155
|
+
max_connections: int = 10,
|
|
156
|
+
) -> None:
|
|
157
|
+
# curl reads the environment's proxy settings on its own, and an httpx2 transport ignores them
|
|
158
|
+
# (https://github.com/lexiforest/curl_cffi/issues/345).
|
|
159
|
+
options: dict[CurlOpt, Any] = {**(curl_options or {}), CurlOpt.NOPROXY: "*"}
|
|
160
|
+
self._default_headers = default_headers
|
|
161
|
+
self._session = _AsyncSession(
|
|
162
|
+
max_clients=max_connections,
|
|
163
|
+
impersonate=impersonate,
|
|
164
|
+
default_headers=default_headers,
|
|
165
|
+
curl_options=options,
|
|
166
|
+
trust_env=False,
|
|
167
|
+
discard_cookies=True,
|
|
168
|
+
allow_redirects=False,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
@override
|
|
172
|
+
async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
|
|
173
|
+
method = _METHODS.get(request.method)
|
|
174
|
+
if method is None:
|
|
175
|
+
msg = f"curl cannot send an HTTP {request.method} request"
|
|
176
|
+
raise httpx2.UnsupportedProtocol(msg, request=request)
|
|
177
|
+
await request.aread()
|
|
178
|
+
if self._default_headers:
|
|
179
|
+
for name, value in _httpx_default_headers():
|
|
180
|
+
if request.headers.get(name) == value:
|
|
181
|
+
del request.headers[name]
|
|
182
|
+
try:
|
|
183
|
+
response = await self._session.request(
|
|
184
|
+
method=method,
|
|
185
|
+
url=str(request.url),
|
|
186
|
+
data=request.content,
|
|
187
|
+
headers=request.headers.raw,
|
|
188
|
+
timeout=_curl_timeout(request.extensions.get("timeout", {})),
|
|
189
|
+
stream=True,
|
|
190
|
+
)
|
|
191
|
+
except _RequestError as exc:
|
|
192
|
+
raise _httpx_error(request, exc) from exc
|
|
193
|
+
return _httpx_response(request, response)
|
|
194
|
+
|
|
195
|
+
@override
|
|
196
|
+
async def aclose(self) -> None:
|
|
197
|
+
await self._session.close()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _httpx_response(request: httpx2.Request, response: _Response) -> httpx2.Response:
|
|
201
|
+
"""Wrap the streamed ``curl_cffi`` *response* as the httpx2 response a native transport returns.
|
|
202
|
+
|
|
203
|
+
``elapsed`` is left to httpx2, which measures until the body is closed; curl's own figure stops at the headers.
|
|
204
|
+
"""
|
|
205
|
+
headers = httpx2.Headers([(k, v or b"") for k, v in response.headers.raw])
|
|
206
|
+
if "content-encoding" in headers:
|
|
207
|
+
for name in _WIRE_ENCODING_HEADERS:
|
|
208
|
+
headers.pop(name, None)
|
|
209
|
+
extensions: dict[str, object] = {"curl": {"response": response, "infos": response.infos}}
|
|
210
|
+
if (http_version := _HTTP_VERSIONS.get(response.http_version)) is not None:
|
|
211
|
+
extensions["http_version"] = http_version
|
|
212
|
+
if response.reason: # empty under HTTP/2 and HTTP/3
|
|
213
|
+
extensions["reason_phrase"] = response.reason.encode("ascii", "ignore")
|
|
214
|
+
return httpx2.Response(
|
|
215
|
+
status_code=response.status_code,
|
|
216
|
+
request=request,
|
|
217
|
+
headers=headers,
|
|
218
|
+
stream=_CurlByteStream(request, response),
|
|
219
|
+
extensions=extensions,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class _CurlByteStream(httpx2.AsyncByteStream):
|
|
224
|
+
"""The body of a streamed ``curl_cffi`` response, with curl's errors raised as httpx2 ones."""
|
|
225
|
+
|
|
226
|
+
def __init__(self, request: httpx2.Request, response: _Response) -> None:
|
|
227
|
+
self._request = request
|
|
228
|
+
self._response = response
|
|
229
|
+
# Set to make curl's write callback abort the transfer. Only a streamed response has one; for any other there
|
|
230
|
+
# is nothing left to abort, and a fresh event is never read.
|
|
231
|
+
self._abort: asyncio.Event = response.quit_now or asyncio.Event()
|
|
232
|
+
|
|
233
|
+
@override
|
|
234
|
+
async def __aiter__(self) -> AsyncIterator[bytes]:
|
|
235
|
+
try:
|
|
236
|
+
async for chunk in self._response.aiter_content():
|
|
237
|
+
yield bytes(chunk) # `curl_cffi` leaves the chunk untyped; `bytes(bytes)` returns it without a copy
|
|
238
|
+
except _RequestError as exc:
|
|
239
|
+
raise _httpx_error(self._request, exc) from exc
|
|
240
|
+
|
|
241
|
+
@override
|
|
242
|
+
async def aclose(self) -> None:
|
|
243
|
+
# Without the abort, `aclose` waits for (and queues) the rest of the body.
|
|
244
|
+
self._abort.set()
|
|
245
|
+
await self._response.aclose()
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _httpx_error(request: httpx2.Request, exc: _RequestError) -> httpx2.RequestError:
|
|
249
|
+
"""Return the httpx2 exception a native transport raises for the curl failure *exc*."""
|
|
250
|
+
message = str(exc)
|
|
251
|
+
if exc.code == CurlECode.OPERATION_TIMEDOUT:
|
|
252
|
+
kind: type[httpx2.RequestError] = _timeout_error(message)
|
|
253
|
+
else:
|
|
254
|
+
kind = _CURL_ERRORS.get(exc.code, httpx2.TransportError)
|
|
255
|
+
return kind(message, request=request)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _timeout_error(message: str) -> type[httpx2.TimeoutException]:
|
|
259
|
+
"""Return :class:`httpx2.ConnectTimeout` for a timeout before the connection was up, else ``ReadTimeout``."""
|
|
260
|
+
if any(marker in message for marker in _CONNECT_TIMEOUT_MESSAGES):
|
|
261
|
+
return httpx2.ConnectTimeout
|
|
262
|
+
return httpx2.ReadTimeout
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _curl_timeout(timeout: Mapping[str, float | None]) -> tuple[float, float] | None:
|
|
266
|
+
"""Return the ``(connect, read)`` pair curl takes for httpx2's *timeout* extension.
|
|
267
|
+
|
|
268
|
+
curl has a connect timeout and, in stream mode, a stall window of ``connect + read`` seconds; httpx2's ``write`` and
|
|
269
|
+
``pool`` have no curl twin. A ``None`` (no limit) becomes curl's own "no limit" (``0``, or no tuple at all for an
|
|
270
|
+
unlimited read), which never makes a limit tighter than the one asked for. A missing key means no limit too: a
|
|
271
|
+
transport above may rebuild the request without the extension, as hishel does.
|
|
272
|
+
"""
|
|
273
|
+
read = timeout.get("read")
|
|
274
|
+
if read is None:
|
|
275
|
+
return None
|
|
276
|
+
return (timeout.get("connect") or 0.0, read)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@functools.cache
|
|
280
|
+
def _httpx_default_headers() -> tuple[tuple[str, str], ...]:
|
|
281
|
+
"""Return the headers httpx2 and headgate add to a request the caller gave none.
|
|
282
|
+
|
|
283
|
+
httpx2's are read from a client rather than hard-coded; headgate's is the ``User-Agent`` of a client without a
|
|
284
|
+
product token of its own.
|
|
285
|
+
"""
|
|
286
|
+
with httpx2.Client(trust_env=False) as client:
|
|
287
|
+
return (*client.headers.items(), ("user-agent", DEFAULT_USER_AGENT))
|
headgate/_deprecation.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Copyright (c) 2026 Khaled Elsayed
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""Endpoint deprecation and sunset announcements: the ``Deprecation`` (RFC 9745) and ``Sunset`` (RFC 8594) headers.
|
|
5
|
+
|
|
6
|
+
A provider retiring an endpoint says so on every response from it. :func:`deprecation_of` reads what a response
|
|
7
|
+
announces into a :class:`~headgate.events.Deprecated` event, and :func:`announce_deprecation` emits it to the call's
|
|
8
|
+
observers, once per call, from the final response's head.
|
|
9
|
+
|
|
10
|
+
Both formats of ``Deprecation`` are read: RFC 9745's structured-field date (``@1688169599``, seconds since the epoch)
|
|
11
|
+
and the older drafts' ``true`` or HTTP-date. ``Sunset`` is an HTTP-date. A header whose value is neither is ignored, as
|
|
12
|
+
RFC 9745 asks of an invalid one. The ``Link`` targets with ``rel="deprecation"`` and ``rel="sunset"`` are reported
|
|
13
|
+
when either header is, resolved against the request URL and with any credential in them redacted.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from datetime import UTC, datetime
|
|
20
|
+
from email.utils import parsedate_to_datetime
|
|
21
|
+
from typing import TYPE_CHECKING, Final
|
|
22
|
+
|
|
23
|
+
from headgate._core import Deprecated
|
|
24
|
+
from headgate._url import sanitize_url
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
import httpx2
|
|
28
|
+
|
|
29
|
+
from headgate._core import CallContext, CallInfo
|
|
30
|
+
|
|
31
|
+
__all__ = ["announce_deprecation", "deprecation_of", "parse_deprecation", "parse_http_date"]
|
|
32
|
+
|
|
33
|
+
#: An RFC 8941 Date: ``@`` and an Integer of at most 15 digits.
|
|
34
|
+
_SF_DATE: Final = re.compile(r"@(-?[0-9]{1,15})")
|
|
35
|
+
|
|
36
|
+
#: What parsing a date can raise: a value ``parsedate_to_datetime`` rejects, or a timestamp past ``datetime``'s range.
|
|
37
|
+
_DATE_ERRORS = (TypeError, ValueError, OverflowError, OSError)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def parse_http_date(value: str) -> datetime | None:
|
|
41
|
+
"""Return the HTTP-date *value* as an aware UTC ``datetime``, or ``None`` when it is not one."""
|
|
42
|
+
try:
|
|
43
|
+
when = parsedate_to_datetime(value.strip())
|
|
44
|
+
except _DATE_ERRORS:
|
|
45
|
+
return None
|
|
46
|
+
return when.replace(tzinfo=UTC) if when.tzinfo is None else when.astimezone(UTC)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def parse_deprecation(value: str) -> tuple[bool, datetime | None]:
|
|
50
|
+
"""Return whether a ``Deprecation`` header *value* is valid, and the date it names.
|
|
51
|
+
|
|
52
|
+
``@<seconds>`` (RFC 9745) and an HTTP-date name a date; ``true`` (the older drafts) names none. Anything else is
|
|
53
|
+
invalid: ``(False, None)``.
|
|
54
|
+
"""
|
|
55
|
+
text = value.strip()
|
|
56
|
+
if (match := _SF_DATE.fullmatch(text)) is not None:
|
|
57
|
+
try:
|
|
58
|
+
return True, datetime.fromtimestamp(int(match[1]), UTC)
|
|
59
|
+
except _DATE_ERRORS:
|
|
60
|
+
return False, None
|
|
61
|
+
if text.lower() == "true":
|
|
62
|
+
return True, None
|
|
63
|
+
when = parse_http_date(text)
|
|
64
|
+
return when is not None, when
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _link(response: httpx2.Response, rel: str) -> str | None:
|
|
68
|
+
"""Return the target of *response*'s ``Link`` with relation *rel*, resolved and sanitized, or ``None``."""
|
|
69
|
+
target = response.links.get(rel, {}).get("url")
|
|
70
|
+
if target is None:
|
|
71
|
+
return None
|
|
72
|
+
return sanitize_url(str(response.request.url.join(target)))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def deprecation_of(response: httpx2.Response, call: CallInfo) -> Deprecated | None:
|
|
76
|
+
"""Return the :class:`~headgate.events.Deprecated` event *response* announces for *call*, or ``None``.
|
|
77
|
+
|
|
78
|
+
``None`` unless the response carries a valid ``Deprecation`` or ``Sunset`` header.
|
|
79
|
+
"""
|
|
80
|
+
deprecation = response.headers.get("Deprecation")
|
|
81
|
+
sunset = response.headers.get("Sunset")
|
|
82
|
+
if deprecation is None and sunset is None:
|
|
83
|
+
return None
|
|
84
|
+
deprecated, deprecated_at = (False, None) if deprecation is None else parse_deprecation(deprecation)
|
|
85
|
+
sunset_at = None if sunset is None else parse_http_date(sunset)
|
|
86
|
+
if not deprecated and sunset_at is None:
|
|
87
|
+
return None
|
|
88
|
+
return Deprecated(
|
|
89
|
+
call=call,
|
|
90
|
+
deprecated=deprecated,
|
|
91
|
+
deprecated_at=deprecated_at,
|
|
92
|
+
sunset_at=sunset_at,
|
|
93
|
+
link=_link(response, "deprecation"),
|
|
94
|
+
sunset_link=_link(response, "sunset"),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def announce_deprecation(context: CallContext, response: httpx2.Response) -> None:
|
|
99
|
+
"""Emit the :class:`~headgate.events.Deprecated` event *response* announces, when the call has observers."""
|
|
100
|
+
if context.observed and (event := deprecation_of(response, context.info)) is not None:
|
|
101
|
+
context.emit(event)
|