modern-httplib2 1.32.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modern_httplib2/__init__.py +2944 -0
- modern_httplib2/auth.py +63 -0
- modern_httplib2/auto.py +5 -0
- modern_httplib2/cacerts.txt +2225 -0
- modern_httplib2/certs.py +42 -0
- modern_httplib2/decode.py +183 -0
- modern_httplib2/error.py +50 -0
- modern_httplib2/iri2uri.py +65 -0
- modern_httplib2/patch.py +164 -0
- modern_httplib2-1.32.0.dist-info/METADATA +187 -0
- modern_httplib2-1.32.0.dist-info/RECORD +14 -0
- modern_httplib2-1.32.0.dist-info/WHEEL +5 -0
- modern_httplib2-1.32.0.dist-info/licenses/LICENSE +201 -0
- modern_httplib2-1.32.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2944 @@
|
|
|
1
|
+
"""Small, fast, modern HTTP client library for Python 3.11+ powered by httpx."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import calendar
|
|
5
|
+
import contextlib
|
|
6
|
+
import copy
|
|
7
|
+
import email
|
|
8
|
+
import email.feedparser
|
|
9
|
+
from email import header
|
|
10
|
+
import email.message
|
|
11
|
+
import email.utils
|
|
12
|
+
import errno
|
|
13
|
+
import functools
|
|
14
|
+
from gettext import gettext as _
|
|
15
|
+
from hashlib import md5 as _md5
|
|
16
|
+
from hashlib import sha1 as _sha
|
|
17
|
+
import hmac
|
|
18
|
+
import http.client
|
|
19
|
+
import os
|
|
20
|
+
import queue
|
|
21
|
+
import random
|
|
22
|
+
import re
|
|
23
|
+
import select
|
|
24
|
+
import socket
|
|
25
|
+
import ssl
|
|
26
|
+
import sys
|
|
27
|
+
import threading
|
|
28
|
+
import time
|
|
29
|
+
import urllib.parse
|
|
30
|
+
import zlib
|
|
31
|
+
|
|
32
|
+
if sys.version_info < (3, 11):
|
|
33
|
+
raise RuntimeError("modern-httplib2 requires Python 3.11 or newer")
|
|
34
|
+
|
|
35
|
+
from .decode import DecoderProtocol, DeflateDecoder, LimitDecoder, ZlibDecoder
|
|
36
|
+
from .patch import is_patched, patch, unpatch
|
|
37
|
+
|
|
38
|
+
__author__ = "Jay Lee"
|
|
39
|
+
__email__ = "jay0lee@gmail.com"
|
|
40
|
+
__copyright__ = "Copyright 2026, Jay Lee"
|
|
41
|
+
__contributors__ = ["Jay Lee"]
|
|
42
|
+
__license__ = "MIT"
|
|
43
|
+
__version__ = "1.32.0"
|
|
44
|
+
|
|
45
|
+
import httpcore
|
|
46
|
+
from httpcore._backends.base import (
|
|
47
|
+
NetworkBackend as _HttpcoreNetworkBackend,
|
|
48
|
+
NetworkStream as _HttpcoreNetworkStream,
|
|
49
|
+
)
|
|
50
|
+
from httpcore._exceptions import (
|
|
51
|
+
ConnectError as _HttpcoreConnectError,
|
|
52
|
+
ConnectTimeout as _HttpcoreConnectTimeout,
|
|
53
|
+
LocalProtocolError as _HttpcoreLocalProtocolError,
|
|
54
|
+
PoolTimeout as _HttpcorePoolTimeout,
|
|
55
|
+
ProtocolError as _HttpcoreProtocolError,
|
|
56
|
+
ProxyError as _HttpcoreProxyError,
|
|
57
|
+
ReadError as _HttpcoreReadError,
|
|
58
|
+
ReadTimeout as _HttpcoreReadTimeout,
|
|
59
|
+
RemoteProtocolError as _HttpcoreRemoteProtocolError,
|
|
60
|
+
TimeoutException as _HttpcoreTimeoutException,
|
|
61
|
+
WriteError as _HttpcoreWriteError,
|
|
62
|
+
WriteTimeout as _HttpcoreWriteTimeout,
|
|
63
|
+
)
|
|
64
|
+
from httpcore._models import (
|
|
65
|
+
Origin as _HttpcoreOrigin,
|
|
66
|
+
Request as _HttpcoreRequest,
|
|
67
|
+
Response as _HttpcoreResponse,
|
|
68
|
+
URL as _HttpcoreURL,
|
|
69
|
+
)
|
|
70
|
+
import httpcore._sync.connection as _httpcore_connection_mod
|
|
71
|
+
from httpcore._sync.connection import HTTPConnection as _HttpcoreHTTPConnection
|
|
72
|
+
from httpcore._sync.connection_pool import ConnectionPool as _HttpcoreConnectionPool
|
|
73
|
+
from httpcore._sync.http2 import HTTP2Connection as _HttpcoreHTTP2Connection
|
|
74
|
+
from httpcore._synchronization import Semaphore as _HttpcoreSemaphore
|
|
75
|
+
from httpcore._utils import is_socket_readable as _is_socket_readable
|
|
76
|
+
import httpx
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
import socks
|
|
80
|
+
except ImportError:
|
|
81
|
+
socks = None
|
|
82
|
+
|
|
83
|
+
from . import auth, certs
|
|
84
|
+
from .error import *
|
|
85
|
+
from .iri2uri import iri2uri
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def has_timeout(timeout):
|
|
89
|
+
return timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
__all__ = [
|
|
93
|
+
"AUTO_DOWNGRADE_HTTP2",
|
|
94
|
+
"ENABLE_HTTP2",
|
|
95
|
+
"FailedToDecompressContent",
|
|
96
|
+
"Http",
|
|
97
|
+
"HttpLib2Error",
|
|
98
|
+
"ProxyInfo",
|
|
99
|
+
"REDIRECT_CODES",
|
|
100
|
+
"RETRIES",
|
|
101
|
+
"RedirectLimit",
|
|
102
|
+
"RedirectMissingLocation",
|
|
103
|
+
"Response",
|
|
104
|
+
"debug_print",
|
|
105
|
+
"debuglevel",
|
|
106
|
+
"disable_http2",
|
|
107
|
+
"enable_http2",
|
|
108
|
+
"force_http11",
|
|
109
|
+
"is_patched",
|
|
110
|
+
"patch",
|
|
111
|
+
"reset_downgraded_hosts",
|
|
112
|
+
"set_auto_downgrade_http2",
|
|
113
|
+
"set_enable_http2",
|
|
114
|
+
"unpatch",
|
|
115
|
+
"UnimplementedDigestAuthOptionError",
|
|
116
|
+
"UnimplementedHmacDigestAuthOptionError",
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
# The httplib debug level, set to a non-zero value to get debug output
|
|
120
|
+
debuglevel = 0
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _debug_print(*args, **kwargs):
|
|
124
|
+
"""Output debug messages, honoring custom print monkey-patches on httplib2, modern_httplib2, or http.client."""
|
|
125
|
+
httplib2_mod = sys.modules.get("httplib2") or sys.modules.get(__name__)
|
|
126
|
+
printer = getattr(httplib2_mod, "debug_print", None)
|
|
127
|
+
if printer is None or printer is _debug_print:
|
|
128
|
+
printer = getattr(httplib2_mod, "print", None)
|
|
129
|
+
if printer is None:
|
|
130
|
+
printer = getattr(http.client, "print", None)
|
|
131
|
+
if printer is None:
|
|
132
|
+
printer = print
|
|
133
|
+
str_args = tuple(str(a) if not isinstance(a, str) else a for a in args)
|
|
134
|
+
printer(*str_args, **kwargs)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
debug_print = _debug_print
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _env_bool(var_name: str, default: bool) -> bool:
|
|
141
|
+
val = os.environ.get(var_name)
|
|
142
|
+
if val is None:
|
|
143
|
+
return default
|
|
144
|
+
val = val.strip().lower()
|
|
145
|
+
return val not in ("0", "false", "no", "off", "")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# Global flags to enable or disable HTTP/2 and auto-downgrade fallback
|
|
149
|
+
ENABLE_HTTP2 = _env_bool(
|
|
150
|
+
"MODERN_HTTPLIB2_ENABLE_HTTP2",
|
|
151
|
+
_env_bool("HTTPLIB2_ENABLE_HTTP2", True),
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
AUTO_DOWNGRADE_HTTP2 = _env_bool(
|
|
155
|
+
"MODERN_HTTPLIB2_AUTO_DOWNGRADE",
|
|
156
|
+
_env_bool("HTTPLIB2_AUTO_DOWNGRADE", True),
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
# Thread-safe set of hosts (authorities) that have been downgraded to HTTP/1.1
|
|
160
|
+
DOWNGRADED_HOSTS = set()
|
|
161
|
+
_DOWNGRADE_LOCK = threading.Lock()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def force_http11():
|
|
165
|
+
"""Globally force all modern-httplib2 connections to HTTP/1.1."""
|
|
166
|
+
set_enable_http2(False)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def disable_http2():
|
|
170
|
+
"""Globally disable HTTP/2 for all modern-httplib2 connections."""
|
|
171
|
+
set_enable_http2(False)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def enable_http2():
|
|
175
|
+
"""Globally enable HTTP/2 for modern-httplib2 connections."""
|
|
176
|
+
set_enable_http2(True)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def set_enable_http2(enabled: bool):
|
|
180
|
+
"""Globally enable or disable HTTP/2.
|
|
181
|
+
|
|
182
|
+
When disabled, ALPN will only advertise HTTP/1.1 and connections will use
|
|
183
|
+
standard HTTP/1.1 pooling.
|
|
184
|
+
"""
|
|
185
|
+
global ENABLE_HTTP2
|
|
186
|
+
ENABLE_HTTP2 = bool(enabled)
|
|
187
|
+
Http.enable_http2 = bool(enabled)
|
|
188
|
+
HTTPSConnectionWithTimeout.enable_http2 = bool(enabled)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def set_auto_downgrade_http2(enabled: bool):
|
|
192
|
+
"""Globally enable or disable automatic fallback to HTTP/1.1 when HTTP/2 fails."""
|
|
193
|
+
global AUTO_DOWNGRADE_HTTP2
|
|
194
|
+
AUTO_DOWNGRADE_HTTP2 = bool(enabled)
|
|
195
|
+
Http.auto_downgrade_http2 = bool(enabled)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def reset_downgraded_hosts():
|
|
199
|
+
"""Reset the set of hosts that were automatically downgraded to HTTP/1.1."""
|
|
200
|
+
with _DOWNGRADE_LOCK:
|
|
201
|
+
DOWNGRADED_HOSTS.clear()
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# A request will be tried 'RETRIES' times if it fails at the socket/connection level.
|
|
205
|
+
RETRIES = 2
|
|
206
|
+
|
|
207
|
+
# The number of redirections to follow before giving up.
|
|
208
|
+
# Note that only GET redirects are automatically followed.
|
|
209
|
+
# Will also honor 301 requests by saving that info and never
|
|
210
|
+
# requesting that URI again.
|
|
211
|
+
DEFAULT_MAX_REDIRECTS = 5
|
|
212
|
+
|
|
213
|
+
# Default connection pool size per (scheme, authority)
|
|
214
|
+
DEFAULT_POOL_MAXSIZE = 16
|
|
215
|
+
|
|
216
|
+
# Which headers are hop-by-hop headers by default
|
|
217
|
+
HOP_BY_HOP = [
|
|
218
|
+
"connection",
|
|
219
|
+
"keep-alive",
|
|
220
|
+
"proxy-authenticate",
|
|
221
|
+
"proxy-authorization",
|
|
222
|
+
"te",
|
|
223
|
+
"trailers",
|
|
224
|
+
"transfer-encoding",
|
|
225
|
+
"upgrade",
|
|
226
|
+
]
|
|
227
|
+
|
|
228
|
+
# https://tools.ietf.org/html/rfc7231#section-8.1.3
|
|
229
|
+
SAFE_METHODS = ("GET", "HEAD", "OPTIONS", "TRACE")
|
|
230
|
+
|
|
231
|
+
# To change, assign to `Http().redirect_codes`
|
|
232
|
+
REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308))
|
|
233
|
+
|
|
234
|
+
CA_CERTS = certs.where()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _resolve_ca_certs(ca_certs):
|
|
238
|
+
if ca_certs:
|
|
239
|
+
return ca_certs
|
|
240
|
+
if "HTTPLIB2_CA_CERTS" in os.environ:
|
|
241
|
+
return certs.where()
|
|
242
|
+
return CA_CERTS
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLS_CLIENT
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _build_ssl_context(
|
|
249
|
+
disable_ssl_certificate_validation,
|
|
250
|
+
ca_certs,
|
|
251
|
+
cert_file=None,
|
|
252
|
+
key_file=None,
|
|
253
|
+
maximum_version=None,
|
|
254
|
+
minimum_version=None,
|
|
255
|
+
key_password=None,
|
|
256
|
+
enable_http2=False,
|
|
257
|
+
):
|
|
258
|
+
context = ssl.SSLContext(DEFAULT_TLS_VERSION)
|
|
259
|
+
context.check_hostname = not disable_ssl_certificate_validation
|
|
260
|
+
context.verify_mode = ssl.CERT_NONE if disable_ssl_certificate_validation else ssl.CERT_REQUIRED
|
|
261
|
+
|
|
262
|
+
if maximum_version is not None:
|
|
263
|
+
context.maximum_version = (
|
|
264
|
+
getattr(ssl.TLSVersion, maximum_version)
|
|
265
|
+
if isinstance(maximum_version, str)
|
|
266
|
+
else maximum_version
|
|
267
|
+
)
|
|
268
|
+
if minimum_version is not None:
|
|
269
|
+
context.minimum_version = (
|
|
270
|
+
getattr(ssl.TLSVersion, minimum_version)
|
|
271
|
+
if isinstance(minimum_version, str)
|
|
272
|
+
else minimum_version
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
if not disable_ssl_certificate_validation:
|
|
276
|
+
context.load_verify_locations(ca_certs)
|
|
277
|
+
|
|
278
|
+
if cert_file:
|
|
279
|
+
context.load_cert_chain(cert_file, key_file, key_password)
|
|
280
|
+
|
|
281
|
+
if enable_http2 and ENABLE_HTTP2:
|
|
282
|
+
with contextlib.suppress(NotImplementedError, ssl.SSLError):
|
|
283
|
+
context.set_alpn_protocols(["h2", "http/1.1"])
|
|
284
|
+
else:
|
|
285
|
+
with contextlib.suppress(NotImplementedError, ssl.SSLError):
|
|
286
|
+
context.set_alpn_protocols(["http/1.1"])
|
|
287
|
+
|
|
288
|
+
return context
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _get_end2end_headers(response):
|
|
292
|
+
hopbyhop = set(HOP_BY_HOP)
|
|
293
|
+
hopbyhop.update(x.strip() for x in response.get("connection", "").split(","))
|
|
294
|
+
return [header for header in response if header not in hopbyhop]
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _errno_from_exception(e):
|
|
298
|
+
try:
|
|
299
|
+
return e.errno
|
|
300
|
+
except AttributeError:
|
|
301
|
+
pass
|
|
302
|
+
|
|
303
|
+
# socket.error and common wrap in .args
|
|
304
|
+
try:
|
|
305
|
+
if e.args:
|
|
306
|
+
return _errno_from_exception(e.args[0])
|
|
307
|
+
except AttributeError:
|
|
308
|
+
pass
|
|
309
|
+
|
|
310
|
+
# pysocks.ProxyError wraps in .socket_err
|
|
311
|
+
try:
|
|
312
|
+
if e.socket_err:
|
|
313
|
+
return _errno_from_exception(e.socket_err)
|
|
314
|
+
except AttributeError:
|
|
315
|
+
pass
|
|
316
|
+
|
|
317
|
+
return None
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def parse_uri(uri):
|
|
324
|
+
"""Parses a URI using the regex given in Appendix B of RFC 3986.
|
|
325
|
+
|
|
326
|
+
(scheme, authority, path, query, fragment) = parse_uri(uri)
|
|
327
|
+
"""
|
|
328
|
+
groups = URI.match(uri).groups()
|
|
329
|
+
return (groups[1], groups[3], groups[4], groups[6], groups[8])
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def urlnorm(uri):
|
|
333
|
+
scheme, authority, path, query, fragment = parse_uri(uri)
|
|
334
|
+
if not scheme or not authority:
|
|
335
|
+
raise RelativeURIError(f"Only absolute URIs are allowed. uri = {uri}")
|
|
336
|
+
authority = authority.lower()
|
|
337
|
+
scheme = scheme.lower()
|
|
338
|
+
if not path:
|
|
339
|
+
path = "/"
|
|
340
|
+
request_uri = f"{path}?{query}" if query else path
|
|
341
|
+
defrag_uri = f"{scheme}://{authority}{request_uri}"
|
|
342
|
+
return scheme, authority, request_uri, defrag_uri
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# Cache filename construction
|
|
346
|
+
re_url_scheme = re.compile(r"^\w+://")
|
|
347
|
+
re_unsafe = re.compile(r"[^\w\-_.()=!]+", re.ASCII)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def safename(filename):
|
|
351
|
+
"""Return a filename suitable for the cache.
|
|
352
|
+
Strips dangerous and common characters to create a filename we
|
|
353
|
+
can use to store the cache in.
|
|
354
|
+
"""
|
|
355
|
+
if isinstance(filename, bytes):
|
|
356
|
+
filename_bytes = filename
|
|
357
|
+
filename = filename.decode("utf-8")
|
|
358
|
+
else:
|
|
359
|
+
filename_bytes = filename.encode("utf-8")
|
|
360
|
+
filemd5 = _md5(filename_bytes).hexdigest()
|
|
361
|
+
filename = re_url_scheme.sub("", filename)
|
|
362
|
+
filename = re_unsafe.sub("", filename)
|
|
363
|
+
|
|
364
|
+
# Limit length of filename to 90 chars (plus 1 comma + 32 md5 = 123 chars)
|
|
365
|
+
return f"{filename[:90]},{filemd5}"
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
NORMALIZE_SPACE = re.compile(r"(?:\r\n)?[ \t]+")
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _normalize_headers(headers):
|
|
372
|
+
return {
|
|
373
|
+
_convert_byte_str(key).lower(): NORMALIZE_SPACE.sub(_convert_byte_str(value), " ").strip()
|
|
374
|
+
for key, value in headers.items()
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _convert_byte_str(s):
|
|
379
|
+
if not isinstance(s, str):
|
|
380
|
+
return str(s, "utf-8")
|
|
381
|
+
return s
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _parse_cache_control(headers):
|
|
385
|
+
if "cache-control" not in headers:
|
|
386
|
+
return {}
|
|
387
|
+
retval = {}
|
|
388
|
+
for part in headers["cache-control"].split(","):
|
|
389
|
+
if "=" in part:
|
|
390
|
+
name, _, val = part.partition("=")
|
|
391
|
+
retval[name.strip().lower()] = val.strip().lower()
|
|
392
|
+
else:
|
|
393
|
+
retval[part.strip().lower()] = 1
|
|
394
|
+
return retval
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# Whether to use a strict mode to parse WWW-Authenticate headers
|
|
398
|
+
# Might lead to bad results in case of ill-formed header value,
|
|
399
|
+
# so disabled by default, falling back to relaxed parsing.
|
|
400
|
+
# Set to true to turn on, useful for testing servers.
|
|
401
|
+
USE_WWW_AUTH_STRICT_PARSING = 0
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _entry_disposition(response_headers, request_headers):
|
|
405
|
+
"""Determine freshness from the Date, Expires and Cache-Control headers."""
|
|
406
|
+
retval = "STALE"
|
|
407
|
+
cc = _parse_cache_control(request_headers)
|
|
408
|
+
cc_response = _parse_cache_control(response_headers)
|
|
409
|
+
|
|
410
|
+
if "pragma" in request_headers and request_headers["pragma"].lower().find("no-cache") != -1:
|
|
411
|
+
retval = "TRANSPARENT"
|
|
412
|
+
if "cache-control" not in request_headers:
|
|
413
|
+
request_headers["cache-control"] = "no-cache"
|
|
414
|
+
elif "no-cache" in cc:
|
|
415
|
+
retval = "TRANSPARENT"
|
|
416
|
+
elif "no-cache" in cc_response:
|
|
417
|
+
retval = "STALE"
|
|
418
|
+
elif "only-if-cached" in cc:
|
|
419
|
+
retval = "FRESH"
|
|
420
|
+
elif "date" in response_headers:
|
|
421
|
+
date = calendar.timegm(email.utils.parsedate_tz(response_headers["date"]))
|
|
422
|
+
now = time.time()
|
|
423
|
+
current_age = max(0, now - date)
|
|
424
|
+
if "max-age" in cc_response:
|
|
425
|
+
try:
|
|
426
|
+
freshness_lifetime = int(cc_response["max-age"])
|
|
427
|
+
except ValueError:
|
|
428
|
+
freshness_lifetime = 0
|
|
429
|
+
elif "expires" in response_headers:
|
|
430
|
+
expires = email.utils.parsedate_tz(response_headers["expires"])
|
|
431
|
+
if None == expires:
|
|
432
|
+
freshness_lifetime = 0
|
|
433
|
+
else:
|
|
434
|
+
freshness_lifetime = max(0, calendar.timegm(expires) - date)
|
|
435
|
+
else:
|
|
436
|
+
freshness_lifetime = 0
|
|
437
|
+
if "max-age" in cc:
|
|
438
|
+
try:
|
|
439
|
+
freshness_lifetime = int(cc["max-age"])
|
|
440
|
+
except ValueError:
|
|
441
|
+
freshness_lifetime = 0
|
|
442
|
+
if "min-fresh" in cc:
|
|
443
|
+
try:
|
|
444
|
+
min_fresh = int(cc["min-fresh"])
|
|
445
|
+
except ValueError:
|
|
446
|
+
min_fresh = 0
|
|
447
|
+
current_age += min_fresh
|
|
448
|
+
if freshness_lifetime > current_age:
|
|
449
|
+
retval = "FRESH"
|
|
450
|
+
return retval
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _decompressContent(response, new_content, limit_kwargs):
|
|
454
|
+
content = new_content
|
|
455
|
+
encoding_header = "content-encoding"
|
|
456
|
+
encoding = response.get(encoding_header, None)
|
|
457
|
+
limit_wrap = functools.partial(LimitDecoder, **limit_kwargs)
|
|
458
|
+
try:
|
|
459
|
+
if encoding in ["gzip", "deflate", "zlib"]:
|
|
460
|
+
try:
|
|
461
|
+
content = limit_wrap(ZlibDecoder()).consume_bytes(new_content, 0)
|
|
462
|
+
except (IOError, zlib.error):
|
|
463
|
+
content = limit_wrap(DeflateDecoder()).consume_bytes(new_content, 0)
|
|
464
|
+
response["content-length"] = str(len(content))
|
|
465
|
+
# Record the historical presence of the encoding in a way the won't interfere.
|
|
466
|
+
response["-content-encoding"] = response.pop(encoding_header)
|
|
467
|
+
except (IOError, zlib.error):
|
|
468
|
+
content = ""
|
|
469
|
+
raise FailedToDecompressContent(
|
|
470
|
+
_("Content purported to be compressed with %s but failed to decompress.")
|
|
471
|
+
% encoding,
|
|
472
|
+
response,
|
|
473
|
+
content,
|
|
474
|
+
)
|
|
475
|
+
return content
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _bind_write_headers(msg):
|
|
479
|
+
def _write_headers(self):
|
|
480
|
+
# Self refers to the Generator object.
|
|
481
|
+
for h, v in msg.items():
|
|
482
|
+
print("%s:" % h, end=" ", file=self._fp)
|
|
483
|
+
if isinstance(v, header.Header):
|
|
484
|
+
print(v.encode(maxlinelen=self._maxheaderlen), file=self._fp)
|
|
485
|
+
else:
|
|
486
|
+
# email.Header got lots of smarts, so use it.
|
|
487
|
+
headers = header.Header(v, maxlinelen=self._maxheaderlen, charset="utf-8", header_name=h)
|
|
488
|
+
print(headers.encode(), file=self._fp)
|
|
489
|
+
# A blank line always separates headers from body.
|
|
490
|
+
print(file=self._fp)
|
|
491
|
+
|
|
492
|
+
return _write_headers
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _updateCache(request_headers, response_headers, content, cache, cachekey):
|
|
496
|
+
if cachekey:
|
|
497
|
+
cc = _parse_cache_control(request_headers)
|
|
498
|
+
cc_response = _parse_cache_control(response_headers)
|
|
499
|
+
if "no-store" in cc or "no-store" in cc_response:
|
|
500
|
+
cache.delete(cachekey)
|
|
501
|
+
else:
|
|
502
|
+
info = email.message.Message()
|
|
503
|
+
for key, value in response_headers.items():
|
|
504
|
+
if key not in ["status", "content-encoding", "transfer-encoding"]:
|
|
505
|
+
info[key] = value
|
|
506
|
+
|
|
507
|
+
# Add annotations to the cache to indicate what headers
|
|
508
|
+
# are variant for this request.
|
|
509
|
+
vary = response_headers.get("vary", None)
|
|
510
|
+
if vary:
|
|
511
|
+
vary_headers = vary.lower().replace(" ", "").split(",")
|
|
512
|
+
for header in vary_headers:
|
|
513
|
+
key = "-varied-%s" % header
|
|
514
|
+
try:
|
|
515
|
+
info[key] = request_headers[header]
|
|
516
|
+
except KeyError:
|
|
517
|
+
pass
|
|
518
|
+
|
|
519
|
+
status = response_headers.status
|
|
520
|
+
if status == 304:
|
|
521
|
+
status = 200
|
|
522
|
+
|
|
523
|
+
status_header = "status: %d\r\n" % status
|
|
524
|
+
|
|
525
|
+
try:
|
|
526
|
+
header_str = info.as_string()
|
|
527
|
+
except UnicodeEncodeError:
|
|
528
|
+
setattr(info, "_write_headers", _bind_write_headers(info))
|
|
529
|
+
header_str = info.as_string()
|
|
530
|
+
|
|
531
|
+
header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str)
|
|
532
|
+
text = b"".join([status_header.encode("utf-8"), header_str.encode("utf-8"), content])
|
|
533
|
+
|
|
534
|
+
cache.set(cachekey, text)
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _cnonce():
|
|
538
|
+
dig = _md5(
|
|
539
|
+
("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).encode("utf-8")
|
|
540
|
+
).hexdigest()
|
|
541
|
+
return dig[:16]
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def _wsse_username_token(cnonce, iso_now, password):
|
|
545
|
+
return (
|
|
546
|
+
base64.b64encode(_sha(f"{cnonce}{iso_now}{password}".encode("utf-8")).digest()).strip().decode("utf-8")
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
class Authentication:
|
|
551
|
+
def __init__(self, credentials, host, request_uri, headers, response, content, http):
|
|
552
|
+
(scheme, authority, path, query, fragment) = parse_uri(request_uri)
|
|
553
|
+
self.path = path
|
|
554
|
+
self.host = host
|
|
555
|
+
self.credentials = credentials
|
|
556
|
+
self.http = http
|
|
557
|
+
|
|
558
|
+
def depth(self, request_uri):
|
|
559
|
+
(scheme, authority, path, query, fragment) = parse_uri(request_uri)
|
|
560
|
+
return request_uri[len(self.path) :].count("/")
|
|
561
|
+
|
|
562
|
+
def inscope(self, host, request_uri):
|
|
563
|
+
(scheme, authority, path, query, fragment) = parse_uri(request_uri)
|
|
564
|
+
return (host == self.host) and path.startswith(self.path)
|
|
565
|
+
|
|
566
|
+
def request(self, method, request_uri, headers, content):
|
|
567
|
+
"""Modify the request headers to add the appropriate
|
|
568
|
+
Authorization header. Override this in sub-classes."""
|
|
569
|
+
pass
|
|
570
|
+
|
|
571
|
+
def response(self, response, content):
|
|
572
|
+
"""Gives us a chance to update with new nonces
|
|
573
|
+
or such returned from the last authorized response.
|
|
574
|
+
Override this in sub-classes if necessary.
|
|
575
|
+
|
|
576
|
+
Return TRUE if the request is to be retried, for
|
|
577
|
+
example Digest may return stale=true.
|
|
578
|
+
"""
|
|
579
|
+
return False
|
|
580
|
+
|
|
581
|
+
def __eq__(self, auth):
|
|
582
|
+
return False
|
|
583
|
+
|
|
584
|
+
def __ne__(self, auth):
|
|
585
|
+
return True
|
|
586
|
+
|
|
587
|
+
def __lt__(self, auth):
|
|
588
|
+
return True
|
|
589
|
+
|
|
590
|
+
def __gt__(self, auth):
|
|
591
|
+
return False
|
|
592
|
+
|
|
593
|
+
def __le__(self, auth):
|
|
594
|
+
return True
|
|
595
|
+
|
|
596
|
+
def __ge__(self, auth):
|
|
597
|
+
return False
|
|
598
|
+
|
|
599
|
+
def __bool__(self):
|
|
600
|
+
return True
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
class BasicAuthentication(Authentication):
|
|
604
|
+
def __init__(self, credentials, host, request_uri, headers, response, content, http):
|
|
605
|
+
super().__init__(credentials, host, request_uri, headers, response, content, http)
|
|
606
|
+
|
|
607
|
+
def request(self, method, request_uri, headers, content):
|
|
608
|
+
"""Modify the request headers to add the appropriate
|
|
609
|
+
Authorization header."""
|
|
610
|
+
user, password = self.credentials
|
|
611
|
+
encoded = base64.b64encode(f"{user}:{password}".encode("utf-8")).strip().decode("utf-8")
|
|
612
|
+
headers["authorization"] = f"Basic {encoded}"
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
class DigestAuthentication(Authentication):
|
|
616
|
+
"""Only do qop='auth' and MD5, since that
|
|
617
|
+
is all Apache currently implements"""
|
|
618
|
+
|
|
619
|
+
def __init__(self, credentials, host, request_uri, headers, response, content, http):
|
|
620
|
+
super().__init__(credentials, host, request_uri, headers, response, content, http)
|
|
621
|
+
self.challenge = auth._parse_www_authenticate(response, "www-authenticate")["digest"]
|
|
622
|
+
qop = self.challenge.get("qop", "auth")
|
|
623
|
+
self.challenge["qop"] = "auth" if "auth" in [x.strip() for x in qop.split()] else None
|
|
624
|
+
if self.challenge["qop"] is None:
|
|
625
|
+
raise UnimplementedDigestAuthOptionError(_(f"Unsupported value for qop: {qop}."))
|
|
626
|
+
self.challenge["algorithm"] = self.challenge.get("algorithm", "MD5").upper()
|
|
627
|
+
if self.challenge["algorithm"] != "MD5":
|
|
628
|
+
raise UnimplementedDigestAuthOptionError(
|
|
629
|
+
_(f"Unsupported value for algorithm: {self.challenge['algorithm']}.")
|
|
630
|
+
)
|
|
631
|
+
self.A1 = "".join([self.credentials[0], ":", self.challenge["realm"], ":", self.credentials[1]])
|
|
632
|
+
self.challenge["nc"] = 1
|
|
633
|
+
|
|
634
|
+
def request(self, method, request_uri, headers, content, cnonce=None):
|
|
635
|
+
"""Modify the request headers"""
|
|
636
|
+
H = lambda x: _md5(x.encode("utf-8")).hexdigest()
|
|
637
|
+
KD = lambda s, d: H(f"{s}:{d}")
|
|
638
|
+
A2 = f"{method}:{request_uri}"
|
|
639
|
+
self.challenge["cnonce"] = cnonce or _cnonce()
|
|
640
|
+
request_digest = '"%s"' % KD(
|
|
641
|
+
H(self.A1),
|
|
642
|
+
f"{self.challenge['nonce']}:{self.challenge['nc']:08x}:{self.challenge['cnonce']}:{self.challenge['qop']}:{H(A2)}",
|
|
643
|
+
)
|
|
644
|
+
headers["authorization"] = (
|
|
645
|
+
f'Digest username="{self.credentials[0]}", realm="{self.challenge["realm"]}", '
|
|
646
|
+
f'nonce="{self.challenge["nonce"]}", uri="{request_uri}", '
|
|
647
|
+
f'algorithm={self.challenge["algorithm"]}, response={request_digest}, '
|
|
648
|
+
f'qop={self.challenge["qop"]}, nc={self.challenge["nc"]:08x}, '
|
|
649
|
+
f'cnonce="{self.challenge["cnonce"]}"'
|
|
650
|
+
)
|
|
651
|
+
if self.challenge.get("opaque"):
|
|
652
|
+
headers["authorization"] += f', opaque="{self.challenge["opaque"]}"'
|
|
653
|
+
self.challenge["nc"] += 1
|
|
654
|
+
|
|
655
|
+
def response(self, response, content):
|
|
656
|
+
if "authentication-info" not in response:
|
|
657
|
+
challenge = auth._parse_www_authenticate(response, "www-authenticate").get("digest", {})
|
|
658
|
+
if "true" == challenge.get("stale"):
|
|
659
|
+
self.challenge["nonce"] = challenge["nonce"]
|
|
660
|
+
self.challenge["nc"] = 1
|
|
661
|
+
return True
|
|
662
|
+
else:
|
|
663
|
+
updated_challenge = auth._parse_authentication_info(response, "authentication-info")
|
|
664
|
+
|
|
665
|
+
if "nextnonce" in updated_challenge:
|
|
666
|
+
self.challenge["nonce"] = updated_challenge["nextnonce"]
|
|
667
|
+
self.challenge["nc"] = 1
|
|
668
|
+
return False
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
class HmacDigestAuthentication(Authentication):
|
|
672
|
+
"""HMAC Digest authentication implementation."""
|
|
673
|
+
|
|
674
|
+
__author__ = "Jay Lee"
|
|
675
|
+
|
|
676
|
+
def __init__(self, credentials, host, request_uri, headers, response, content, http):
|
|
677
|
+
super().__init__(credentials, host, request_uri, headers, response, content, http)
|
|
678
|
+
challenge = auth._parse_www_authenticate(response, "www-authenticate")
|
|
679
|
+
self.challenge = challenge["hmacdigest"]
|
|
680
|
+
self.challenge["reason"] = self.challenge.get("reason", "unauthorized")
|
|
681
|
+
if self.challenge["reason"] not in ["unauthorized", "integrity"]:
|
|
682
|
+
self.challenge["reason"] = "unauthorized"
|
|
683
|
+
self.challenge["salt"] = self.challenge.get("salt", "")
|
|
684
|
+
if not self.challenge.get("snonce"):
|
|
685
|
+
raise UnimplementedHmacDigestAuthOptionError(
|
|
686
|
+
_("The challenge doesn't contain a server nonce, or this one is empty.")
|
|
687
|
+
)
|
|
688
|
+
self.challenge["algorithm"] = self.challenge.get("algorithm", "HMAC-SHA-1")
|
|
689
|
+
if self.challenge["algorithm"] not in ["HMAC-SHA-1", "HMAC-MD5"]:
|
|
690
|
+
raise UnimplementedHmacDigestAuthOptionError(
|
|
691
|
+
_(f"Unsupported value for algorithm: {self.challenge['algorithm']}.")
|
|
692
|
+
)
|
|
693
|
+
self.challenge["pw-algorithm"] = self.challenge.get("pw-algorithm", "SHA-1")
|
|
694
|
+
if self.challenge["pw-algorithm"] not in ["SHA-1", "MD5"]:
|
|
695
|
+
raise UnimplementedHmacDigestAuthOptionError(
|
|
696
|
+
_(f"Unsupported value for pw-algorithm: {self.challenge['pw-algorithm']}.")
|
|
697
|
+
)
|
|
698
|
+
if self.challenge["algorithm"] == "HMAC-MD5":
|
|
699
|
+
self.hashmod = _md5
|
|
700
|
+
else:
|
|
701
|
+
self.hashmod = _sha
|
|
702
|
+
if self.challenge["pw-algorithm"] == "MD5":
|
|
703
|
+
self.pwhashmod = _md5
|
|
704
|
+
else:
|
|
705
|
+
self.pwhashmod = _sha
|
|
706
|
+
self.key = "".join(
|
|
707
|
+
[
|
|
708
|
+
self.credentials[0],
|
|
709
|
+
":",
|
|
710
|
+
self.pwhashmod.new("".join([self.credentials[1], self.challenge["salt"]])).hexdigest().lower(),
|
|
711
|
+
":",
|
|
712
|
+
self.challenge["realm"],
|
|
713
|
+
]
|
|
714
|
+
)
|
|
715
|
+
self.key = self.pwhashmod.new(self.key).hexdigest().lower()
|
|
716
|
+
|
|
717
|
+
def request(self, method, request_uri, headers, content):
|
|
718
|
+
"""Modify the request headers"""
|
|
719
|
+
keys = _get_end2end_headers(headers)
|
|
720
|
+
keylist = "".join([f"{k} " for k in keys])
|
|
721
|
+
headers_val = "".join([headers[k] for k in keys])
|
|
722
|
+
created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
723
|
+
cnonce = _cnonce()
|
|
724
|
+
request_digest = f"{method}:{request_uri}:{cnonce}:{self.challenge['snonce']}:{headers_val}"
|
|
725
|
+
request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
|
|
726
|
+
headers["authorization"] = (
|
|
727
|
+
f'HMACDigest username="{self.credentials[0]}", realm="{self.challenge["realm"]}", '
|
|
728
|
+
f'snonce="{self.challenge["snonce"]}", cnonce="{cnonce}", uri="{request_uri}", '
|
|
729
|
+
f'created="{created}", response="{request_digest}", headers="{keylist}"'
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
def response(self, response, content):
|
|
733
|
+
challenge = auth._parse_www_authenticate(response, "www-authenticate").get("hmacdigest", {})
|
|
734
|
+
return challenge.get("reason") in ["integrity", "stale"]
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
class WsseAuthentication(Authentication):
|
|
738
|
+
def __init__(self, credentials, host, request_uri, headers, response, content, http):
|
|
739
|
+
super().__init__(credentials, host, request_uri, headers, response, content, http)
|
|
740
|
+
|
|
741
|
+
def request(self, method, request_uri, headers, content):
|
|
742
|
+
"""Modify the request headers to add the appropriate
|
|
743
|
+
Authorization header."""
|
|
744
|
+
headers["authorization"] = 'WSSE profile="UsernameToken"'
|
|
745
|
+
iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
746
|
+
cnonce = _cnonce()
|
|
747
|
+
password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
|
|
748
|
+
headers["X-WSSE"] = (
|
|
749
|
+
f'UsernameToken Username="{self.credentials[0]}", PasswordDigest="{password_digest}", '
|
|
750
|
+
f'Nonce="{cnonce}", Created="{iso_now}"'
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
class GoogleLoginAuthentication(Authentication):
|
|
755
|
+
def __init__(self, credentials, host, request_uri, headers, response, content, http):
|
|
756
|
+
from urllib.parse import urlencode
|
|
757
|
+
|
|
758
|
+
super().__init__(credentials, host, request_uri, headers, response, content, http)
|
|
759
|
+
challenge = auth._parse_www_authenticate(response, "www-authenticate")
|
|
760
|
+
service = challenge["googlelogin"].get("service", "xapi")
|
|
761
|
+
if service == "xapi" and request_uri.find("calendar") > 0:
|
|
762
|
+
service = "cl"
|
|
763
|
+
|
|
764
|
+
auth_dict = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers["user-agent"])
|
|
765
|
+
resp, content = self.http.request(
|
|
766
|
+
"https://www.google.com/accounts/ClientLogin",
|
|
767
|
+
method="POST",
|
|
768
|
+
body=urlencode(auth_dict),
|
|
769
|
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
770
|
+
)
|
|
771
|
+
lines = content.split("\n")
|
|
772
|
+
d = dict(tuple(line.split("=", 1)) for line in lines if line)
|
|
773
|
+
if resp.status == 403:
|
|
774
|
+
self.Auth = ""
|
|
775
|
+
else:
|
|
776
|
+
self.Auth = d["Auth"]
|
|
777
|
+
|
|
778
|
+
def request(self, method, request_uri, headers, content):
|
|
779
|
+
"""Modify the request headers to add the appropriate
|
|
780
|
+
Authorization header."""
|
|
781
|
+
headers["authorization"] = f"GoogleLogin Auth={self.Auth}"
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
AUTH_SCHEME_CLASSES = {
|
|
785
|
+
"basic": BasicAuthentication,
|
|
786
|
+
"wsse": WsseAuthentication,
|
|
787
|
+
"digest": DigestAuthentication,
|
|
788
|
+
"hmacdigest": HmacDigestAuthentication,
|
|
789
|
+
"googlelogin": GoogleLoginAuthentication,
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"]
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
class FileCache:
|
|
796
|
+
"""Uses a local directory as a store for cached files.
|
|
797
|
+
Not really safe to use if multiple threads or processes are going to
|
|
798
|
+
be running on the same cache.
|
|
799
|
+
"""
|
|
800
|
+
|
|
801
|
+
def __init__(self, cache, safe=safename):
|
|
802
|
+
self.cache = cache
|
|
803
|
+
self.safe = safe
|
|
804
|
+
os.makedirs(self.cache, exist_ok=True)
|
|
805
|
+
|
|
806
|
+
def get(self, key):
|
|
807
|
+
cache_full_path = os.path.join(self.cache, self.safe(key))
|
|
808
|
+
with contextlib.suppress(OSError):
|
|
809
|
+
with open(cache_full_path, "rb") as f:
|
|
810
|
+
return f.read()
|
|
811
|
+
return None
|
|
812
|
+
|
|
813
|
+
def set(self, key, value):
|
|
814
|
+
cache_full_path = os.path.join(self.cache, self.safe(key))
|
|
815
|
+
with open(cache_full_path, "wb") as f:
|
|
816
|
+
f.write(value)
|
|
817
|
+
|
|
818
|
+
def delete(self, key):
|
|
819
|
+
cache_full_path = os.path.join(self.cache, self.safe(key))
|
|
820
|
+
with contextlib.suppress(FileNotFoundError):
|
|
821
|
+
os.remove(cache_full_path)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
class Credentials:
|
|
825
|
+
def __init__(self):
|
|
826
|
+
self.credentials = []
|
|
827
|
+
|
|
828
|
+
def add(self, name, password, domain=""):
|
|
829
|
+
self.credentials.append((domain.lower(), name, password))
|
|
830
|
+
|
|
831
|
+
def clear(self):
|
|
832
|
+
self.credentials = []
|
|
833
|
+
|
|
834
|
+
def iter(self, domain):
|
|
835
|
+
for cdomain, name, password in self.credentials:
|
|
836
|
+
if cdomain == "" or domain == cdomain:
|
|
837
|
+
yield (name, password)
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
class KeyCerts(Credentials):
|
|
841
|
+
"""Identical to Credentials except that
|
|
842
|
+
name/password are mapped to key/cert."""
|
|
843
|
+
|
|
844
|
+
def add(self, key, cert, domain, password):
|
|
845
|
+
self.credentials.append((domain.lower(), key, cert, password))
|
|
846
|
+
|
|
847
|
+
def iter(self, domain):
|
|
848
|
+
for cdomain, key, cert, password in self.credentials:
|
|
849
|
+
if cdomain == "" or domain == cdomain:
|
|
850
|
+
yield (key, cert, password)
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
class AllHosts:
|
|
854
|
+
pass
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
class ProxyInfo:
|
|
858
|
+
"""Collect information required to use a proxy."""
|
|
859
|
+
|
|
860
|
+
bypass_hosts = ()
|
|
861
|
+
|
|
862
|
+
def __init__(
|
|
863
|
+
self, proxy_type, proxy_host, proxy_port, proxy_rdns=True, proxy_user=None, proxy_pass=None, proxy_headers=None,
|
|
864
|
+
):
|
|
865
|
+
if isinstance(proxy_user, bytes):
|
|
866
|
+
proxy_user = proxy_user.decode()
|
|
867
|
+
if isinstance(proxy_pass, bytes):
|
|
868
|
+
proxy_pass = proxy_pass.decode()
|
|
869
|
+
(
|
|
870
|
+
self.proxy_type,
|
|
871
|
+
self.proxy_host,
|
|
872
|
+
self.proxy_port,
|
|
873
|
+
self.proxy_rdns,
|
|
874
|
+
self.proxy_user,
|
|
875
|
+
self.proxy_pass,
|
|
876
|
+
self.proxy_headers,
|
|
877
|
+
) = (
|
|
878
|
+
proxy_type,
|
|
879
|
+
proxy_host,
|
|
880
|
+
proxy_port,
|
|
881
|
+
proxy_rdns,
|
|
882
|
+
proxy_user,
|
|
883
|
+
proxy_pass,
|
|
884
|
+
proxy_headers,
|
|
885
|
+
)
|
|
886
|
+
|
|
887
|
+
def astuple(self):
|
|
888
|
+
return (
|
|
889
|
+
self.proxy_type,
|
|
890
|
+
self.proxy_host,
|
|
891
|
+
self.proxy_port,
|
|
892
|
+
self.proxy_rdns,
|
|
893
|
+
self.proxy_user,
|
|
894
|
+
self.proxy_pass,
|
|
895
|
+
self.proxy_headers,
|
|
896
|
+
)
|
|
897
|
+
|
|
898
|
+
def isgood(self):
|
|
899
|
+
return socks and (self.proxy_host is not None) and (self.proxy_port is not None)
|
|
900
|
+
|
|
901
|
+
def applies_to(self, hostname):
|
|
902
|
+
return not self.bypass_host(hostname)
|
|
903
|
+
|
|
904
|
+
def bypass_host(self, hostname):
|
|
905
|
+
"""Has this host been excluded from the proxy config"""
|
|
906
|
+
if self.bypass_hosts is AllHosts:
|
|
907
|
+
return True
|
|
908
|
+
|
|
909
|
+
hostname = "." + hostname.lstrip(".")
|
|
910
|
+
for skip_name in self.bypass_hosts:
|
|
911
|
+
# *.suffix
|
|
912
|
+
if skip_name.startswith(".") and hostname.endswith(skip_name):
|
|
913
|
+
return True
|
|
914
|
+
# exact match
|
|
915
|
+
if hostname == "." + skip_name:
|
|
916
|
+
return True
|
|
917
|
+
return False
|
|
918
|
+
|
|
919
|
+
def __repr__(self):
|
|
920
|
+
return (
|
|
921
|
+
f"<ProxyInfo type={self.proxy_type} "
|
|
922
|
+
f"host:port={self.proxy_host}:{self.proxy_port} rdns={self.proxy_rdns} "
|
|
923
|
+
f"user={self.proxy_user} headers={self.proxy_headers}>"
|
|
924
|
+
)
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
def proxy_info_from_environment(method="http"):
|
|
928
|
+
"""Read proxy info from the environment variables."""
|
|
929
|
+
if method not in ("http", "https"):
|
|
930
|
+
return
|
|
931
|
+
|
|
932
|
+
env_var = method + "_proxy"
|
|
933
|
+
url = os.environ.get(env_var, os.environ.get(env_var.upper()))
|
|
934
|
+
if not url:
|
|
935
|
+
return
|
|
936
|
+
return proxy_info_from_url(url, method, noproxy=None)
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def proxy_info_from_url(url, method="http", noproxy=None):
|
|
940
|
+
"""Construct a ProxyInfo from a URL (such as http_proxy env var)"""
|
|
941
|
+
url = urllib.parse.urlparse(url)
|
|
942
|
+
|
|
943
|
+
proxy_type = 3 # socks.PROXY_TYPE_HTTP
|
|
944
|
+
if url.scheme == "socks4":
|
|
945
|
+
proxy_type = 1 # socks.PROXY_TYPE_SOCKS4
|
|
946
|
+
elif url.scheme == "socks5" or url.scheme == "socks":
|
|
947
|
+
proxy_type = 2 # socks.PROXY_TYPE_SOCKS5
|
|
948
|
+
pi = ProxyInfo(
|
|
949
|
+
proxy_type=proxy_type,
|
|
950
|
+
proxy_host=url.hostname,
|
|
951
|
+
proxy_port=url.port or dict(https=443, http=80)[method],
|
|
952
|
+
proxy_user=url.username or None,
|
|
953
|
+
proxy_pass=url.password or None,
|
|
954
|
+
proxy_headers=None,
|
|
955
|
+
)
|
|
956
|
+
|
|
957
|
+
bypass_hosts = []
|
|
958
|
+
# If not given an explicit noproxy value, respect values in env vars.
|
|
959
|
+
if noproxy is None:
|
|
960
|
+
noproxy = os.environ.get("no_proxy", os.environ.get("NO_PROXY", ""))
|
|
961
|
+
# Special case: A single '*' character means all hosts should be bypassed.
|
|
962
|
+
if noproxy == "*":
|
|
963
|
+
bypass_hosts = AllHosts
|
|
964
|
+
elif noproxy.strip():
|
|
965
|
+
bypass_hosts = noproxy.split(",")
|
|
966
|
+
bypass_hosts = tuple(filter(bool, bypass_hosts)) # To exclude empty string.
|
|
967
|
+
|
|
968
|
+
pi.bypass_hosts = bypass_hosts
|
|
969
|
+
return pi
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
import h2.connection as _h2_connection
|
|
973
|
+
import h2.events as _h2_events
|
|
974
|
+
import h2.exceptions as _h2_exceptions
|
|
975
|
+
from httpcore._sync.http11 import HTTP11Connection as _HttpcoreHTTP11Connection
|
|
976
|
+
from httpcore._sync.http2 import (
|
|
977
|
+
HTTP2ConnectionByteStream as _HttpcoreHTTP2ConnectionByteStream,
|
|
978
|
+
HTTPConnectionState as _HttpcoreHTTPConnectionState,
|
|
979
|
+
has_body_headers as _httpcore_has_body_headers,
|
|
980
|
+
)
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def _unwrap_httpcore_exception(exc):
|
|
984
|
+
"""Translate httpcore/httpx exceptions into the stdlib/socket/http.client exceptions
|
|
985
|
+
expected by httplib2 callers and _conn_request retry logic.
|
|
986
|
+
"""
|
|
987
|
+
if isinstance(
|
|
988
|
+
exc,
|
|
989
|
+
(
|
|
990
|
+
socket.timeout,
|
|
991
|
+
socket.gaierror,
|
|
992
|
+
ssl.SSLError,
|
|
993
|
+
ssl.CertificateError,
|
|
994
|
+
http.client.HTTPException,
|
|
995
|
+
HttpLib2Error,
|
|
996
|
+
),
|
|
997
|
+
):
|
|
998
|
+
return exc
|
|
999
|
+
|
|
1000
|
+
cause = getattr(exc, "__cause__", None) or getattr(exc, "__context__", None)
|
|
1001
|
+
if isinstance(
|
|
1002
|
+
cause,
|
|
1003
|
+
(
|
|
1004
|
+
socket.timeout,
|
|
1005
|
+
socket.gaierror,
|
|
1006
|
+
ssl.SSLError,
|
|
1007
|
+
ssl.CertificateError,
|
|
1008
|
+
http.client.HTTPException,
|
|
1009
|
+
HttpLib2Error,
|
|
1010
|
+
OSError,
|
|
1011
|
+
),
|
|
1012
|
+
):
|
|
1013
|
+
return cause
|
|
1014
|
+
|
|
1015
|
+
if isinstance(exc, _HttpcoreTimeoutException):
|
|
1016
|
+
return socket.timeout(str(exc))
|
|
1017
|
+
|
|
1018
|
+
if isinstance(exc, _HttpcoreRemoteProtocolError):
|
|
1019
|
+
msg = str(exc)
|
|
1020
|
+
if "Server disconnected" in msg or "ConnectionClosed" in msg:
|
|
1021
|
+
return http.client.RemoteDisconnected(msg)
|
|
1022
|
+
return http.client.BadStatusLine(msg)
|
|
1023
|
+
|
|
1024
|
+
if isinstance(exc, (_HttpcoreLocalProtocolError, _HttpcoreProtocolError)):
|
|
1025
|
+
return http.client.HTTPException(str(exc))
|
|
1026
|
+
|
|
1027
|
+
if isinstance(exc, (_HttpcoreConnectError, _HttpcoreReadError, _HttpcoreWriteError, _HttpcoreProxyError)):
|
|
1028
|
+
return OSError(str(exc))
|
|
1029
|
+
|
|
1030
|
+
return exc
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def _is_cert_verification_error(exc):
|
|
1034
|
+
"""Check if exception represents an SSL certificate verification or TLS policy failure."""
|
|
1035
|
+
if isinstance(exc, (ssl.CertificateError, getattr(ssl, "SSLCertVerificationError", ()))):
|
|
1036
|
+
return True
|
|
1037
|
+
msg = str(exc).lower()
|
|
1038
|
+
non_downgrade_ssl_terms = (
|
|
1039
|
+
"certificate verify failed",
|
|
1040
|
+
"cert_has_expired",
|
|
1041
|
+
"certificate_unknown",
|
|
1042
|
+
"unknown_ca",
|
|
1043
|
+
"self_signed_cert",
|
|
1044
|
+
"self signed certificate",
|
|
1045
|
+
"hostname",
|
|
1046
|
+
"ip address mismatch",
|
|
1047
|
+
"wrong_version_number",
|
|
1048
|
+
"unsupported_protocol",
|
|
1049
|
+
"protocol_version",
|
|
1050
|
+
"no_ciphers_available",
|
|
1051
|
+
"no shared cipher",
|
|
1052
|
+
"handshake_failure",
|
|
1053
|
+
"insufficient_security",
|
|
1054
|
+
)
|
|
1055
|
+
return any(term in msg for term in non_downgrade_ssl_terms)
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def _should_downgrade_on_error(exc):
|
|
1059
|
+
"""Determine whether an error on an HTTP/2 connection warrants automatic fallback to HTTP/1.1."""
|
|
1060
|
+
if _is_cert_verification_error(exc):
|
|
1061
|
+
return False
|
|
1062
|
+
if isinstance(exc, (socket.timeout, socket.gaierror, ConnectionRefusedError)):
|
|
1063
|
+
return False
|
|
1064
|
+
if isinstance(
|
|
1065
|
+
exc,
|
|
1066
|
+
(
|
|
1067
|
+
http.client.BadStatusLine,
|
|
1068
|
+
http.client.RemoteDisconnected,
|
|
1069
|
+
http.client.HTTPException,
|
|
1070
|
+
ConnectionResetError,
|
|
1071
|
+
ConnectionAbortedError,
|
|
1072
|
+
BrokenPipeError,
|
|
1073
|
+
ssl.SSLError,
|
|
1074
|
+
),
|
|
1075
|
+
):
|
|
1076
|
+
return True
|
|
1077
|
+
if "h2" in type(exc).__module__:
|
|
1078
|
+
return True
|
|
1079
|
+
if isinstance(exc, OSError):
|
|
1080
|
+
errno_ = _errno_from_exception(exc)
|
|
1081
|
+
if errno_ in (
|
|
1082
|
+
errno.ECONNREFUSED,
|
|
1083
|
+
errno.ENETUNREACH,
|
|
1084
|
+
errno.EHOSTUNREACH,
|
|
1085
|
+
errno.EADDRNOTAVAIL,
|
|
1086
|
+
errno.EACCES,
|
|
1087
|
+
errno.EPERM,
|
|
1088
|
+
):
|
|
1089
|
+
return False
|
|
1090
|
+
if errno_ in (errno.ECONNRESET, errno.EPIPE, errno.ECONNABORTED, errno.EPROTO):
|
|
1091
|
+
return True
|
|
1092
|
+
return True
|
|
1093
|
+
msg = str(exc).lower()
|
|
1094
|
+
h2_terms = ("http/2", "h2", "protocol", "stream", "frame", "goaway", "rst_stream", "flow control")
|
|
1095
|
+
return any(term in msg for term in h2_terms)
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
_HALF_CLOSED_SOCKS = []
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
def _wait_socket(sock: socket.socket, for_write: bool, timeout: float | None) -> bool:
|
|
1102
|
+
try:
|
|
1103
|
+
if for_write:
|
|
1104
|
+
_, writable, _ = select.select([], [sock], [], timeout)
|
|
1105
|
+
return bool(writable)
|
|
1106
|
+
readable, _, _ = select.select([sock], [], [], timeout)
|
|
1107
|
+
return bool(readable)
|
|
1108
|
+
except ValueError:
|
|
1109
|
+
if sock.fileno() < 0:
|
|
1110
|
+
raise OSError(errno.EBADF, "Bad file descriptor")
|
|
1111
|
+
poller = select.poll()
|
|
1112
|
+
poller.register(sock, select.POLLOUT if for_write else select.POLLIN)
|
|
1113
|
+
timeout_ms = None if timeout is None else max(0, int(timeout * 1000))
|
|
1114
|
+
return bool(poller.poll(timeout_ms))
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
class _Httplib2SyncStream(_HttpcoreNetworkStream):
|
|
1118
|
+
"""Custom httpcore NetworkStream wrapping a raw/SSL/PySocks socket while preserving
|
|
1119
|
+
stdlib exception types, thread-safe concurrent SSLSocket read/write multiplexing,
|
|
1120
|
+
and half-closed socket semantics on read timeouts.
|
|
1121
|
+
"""
|
|
1122
|
+
|
|
1123
|
+
def __init__(self, sock, owner=None, enable_http2=False):
|
|
1124
|
+
self._sock = sock
|
|
1125
|
+
self._owner = owner
|
|
1126
|
+
self._enable_http2 = enable_http2
|
|
1127
|
+
self._timed_out = False
|
|
1128
|
+
self._tls_started = isinstance(sock, ssl.SSLSocket)
|
|
1129
|
+
self._active_request = None
|
|
1130
|
+
self._ssl_lock = threading.Lock()
|
|
1131
|
+
|
|
1132
|
+
def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
|
|
1133
|
+
sock = self._sock
|
|
1134
|
+
if sock is None:
|
|
1135
|
+
return b""
|
|
1136
|
+
if self._active_request is not None:
|
|
1137
|
+
self._active_request.extensions["request_sent"] = True
|
|
1138
|
+
eff_timeout = timeout if has_timeout(timeout) else None
|
|
1139
|
+
if isinstance(sock, ssl.SSLSocket):
|
|
1140
|
+
deadline = (time.monotonic() + eff_timeout) if eff_timeout is not None else None
|
|
1141
|
+
while True:
|
|
1142
|
+
with self._ssl_lock:
|
|
1143
|
+
if self._sock is None:
|
|
1144
|
+
return b""
|
|
1145
|
+
has_pending = sock.pending() > 0
|
|
1146
|
+
if not has_pending:
|
|
1147
|
+
if deadline is not None:
|
|
1148
|
+
remaining = deadline - time.monotonic()
|
|
1149
|
+
if remaining <= 0:
|
|
1150
|
+
self._timed_out = True
|
|
1151
|
+
raise TimeoutError("timed out")
|
|
1152
|
+
else:
|
|
1153
|
+
remaining = None
|
|
1154
|
+
if not _wait_socket(sock, for_write=False, timeout=remaining):
|
|
1155
|
+
self._timed_out = True
|
|
1156
|
+
raise TimeoutError("timed out")
|
|
1157
|
+
with self._ssl_lock:
|
|
1158
|
+
if self._sock is None:
|
|
1159
|
+
return b""
|
|
1160
|
+
try:
|
|
1161
|
+
sock.settimeout(0.0)
|
|
1162
|
+
return sock.recv(max_bytes)
|
|
1163
|
+
except (ssl.SSLWantReadError, BlockingIOError):
|
|
1164
|
+
continue
|
|
1165
|
+
except ssl.SSLWantWriteError:
|
|
1166
|
+
pass
|
|
1167
|
+
except TimeoutError:
|
|
1168
|
+
self._timed_out = True
|
|
1169
|
+
raise
|
|
1170
|
+
finally:
|
|
1171
|
+
with contextlib.suppress(OSError):
|
|
1172
|
+
sock.settimeout(eff_timeout)
|
|
1173
|
+
if deadline is not None:
|
|
1174
|
+
remaining = deadline - time.monotonic()
|
|
1175
|
+
if remaining <= 0:
|
|
1176
|
+
self._timed_out = True
|
|
1177
|
+
raise TimeoutError("timed out")
|
|
1178
|
+
else:
|
|
1179
|
+
remaining = None
|
|
1180
|
+
if not _wait_socket(sock, for_write=True, timeout=remaining):
|
|
1181
|
+
self._timed_out = True
|
|
1182
|
+
raise TimeoutError("timed out")
|
|
1183
|
+
try:
|
|
1184
|
+
sock.settimeout(eff_timeout)
|
|
1185
|
+
return sock.recv(max_bytes)
|
|
1186
|
+
except TimeoutError:
|
|
1187
|
+
self._timed_out = True
|
|
1188
|
+
raise
|
|
1189
|
+
|
|
1190
|
+
def write(self, buffer: bytes, timeout: float | None = None) -> None:
|
|
1191
|
+
sock = self._sock
|
|
1192
|
+
if not buffer or sock is None:
|
|
1193
|
+
return
|
|
1194
|
+
eff_timeout = timeout if has_timeout(timeout) else None
|
|
1195
|
+
if isinstance(sock, ssl.SSLSocket):
|
|
1196
|
+
deadline = (time.monotonic() + eff_timeout) if eff_timeout is not None else None
|
|
1197
|
+
view = memoryview(buffer)
|
|
1198
|
+
sent_total = 0
|
|
1199
|
+
total_len = len(view)
|
|
1200
|
+
try:
|
|
1201
|
+
while sent_total < total_len:
|
|
1202
|
+
want_read = False
|
|
1203
|
+
with self._ssl_lock:
|
|
1204
|
+
if self._sock is None:
|
|
1205
|
+
raise OSError(errno.EBADF, "Socket closed")
|
|
1206
|
+
try:
|
|
1207
|
+
sock.settimeout(0.0)
|
|
1208
|
+
sent = sock.send(view[sent_total:])
|
|
1209
|
+
if sent > 0:
|
|
1210
|
+
sent_total += sent
|
|
1211
|
+
continue
|
|
1212
|
+
except (ssl.SSLWantWriteError, BlockingIOError):
|
|
1213
|
+
want_read = False
|
|
1214
|
+
except ssl.SSLWantReadError:
|
|
1215
|
+
want_read = True
|
|
1216
|
+
finally:
|
|
1217
|
+
with contextlib.suppress(OSError):
|
|
1218
|
+
sock.settimeout(eff_timeout)
|
|
1219
|
+
if deadline is not None:
|
|
1220
|
+
remaining = deadline - time.monotonic()
|
|
1221
|
+
if remaining <= 0:
|
|
1222
|
+
self._timed_out = True
|
|
1223
|
+
raise TimeoutError("timed out")
|
|
1224
|
+
else:
|
|
1225
|
+
remaining = None
|
|
1226
|
+
if not _wait_socket(sock, for_write=not want_read, timeout=remaining):
|
|
1227
|
+
self._timed_out = True
|
|
1228
|
+
raise TimeoutError("timed out")
|
|
1229
|
+
if self._active_request is not None:
|
|
1230
|
+
self._active_request.extensions["request_sent"] = True
|
|
1231
|
+
return
|
|
1232
|
+
except TimeoutError:
|
|
1233
|
+
self._timed_out = True
|
|
1234
|
+
raise
|
|
1235
|
+
except OSError as exc:
|
|
1236
|
+
if self._active_request is not None:
|
|
1237
|
+
self._active_request.extensions["request_sent"] = True
|
|
1238
|
+
raise _HttpcoreWriteError(str(exc)) from exc
|
|
1239
|
+
try:
|
|
1240
|
+
sock.settimeout(eff_timeout)
|
|
1241
|
+
sock.sendall(buffer)
|
|
1242
|
+
if self._active_request is not None:
|
|
1243
|
+
self._active_request.extensions["request_sent"] = True
|
|
1244
|
+
except TimeoutError:
|
|
1245
|
+
self._timed_out = True
|
|
1246
|
+
raise
|
|
1247
|
+
except OSError as exc:
|
|
1248
|
+
if self._active_request is not None:
|
|
1249
|
+
self._active_request.extensions["request_sent"] = True
|
|
1250
|
+
raise _HttpcoreWriteError(str(exc)) from exc
|
|
1251
|
+
|
|
1252
|
+
def close(self) -> None:
|
|
1253
|
+
with self._ssl_lock:
|
|
1254
|
+
sock = self._sock
|
|
1255
|
+
self._sock = None
|
|
1256
|
+
if sock is None:
|
|
1257
|
+
return
|
|
1258
|
+
if self._timed_out:
|
|
1259
|
+
self._timed_out = False
|
|
1260
|
+
with contextlib.suppress(OSError):
|
|
1261
|
+
sock.shutdown(socket.SHUT_WR)
|
|
1262
|
+
_HALF_CLOSED_SOCKS.append(sock)
|
|
1263
|
+
while len(_HALF_CLOSED_SOCKS) > 16:
|
|
1264
|
+
old_sock = _HALF_CLOSED_SOCKS.pop(0)
|
|
1265
|
+
with contextlib.suppress(OSError):
|
|
1266
|
+
old_sock.close()
|
|
1267
|
+
else:
|
|
1268
|
+
with contextlib.suppress(OSError):
|
|
1269
|
+
sock.close()
|
|
1270
|
+
|
|
1271
|
+
def start_tls(
|
|
1272
|
+
self,
|
|
1273
|
+
ssl_context: ssl.SSLContext,
|
|
1274
|
+
server_hostname: str | None = None,
|
|
1275
|
+
timeout: float | None = None,
|
|
1276
|
+
) -> _HttpcoreNetworkStream:
|
|
1277
|
+
if self._tls_started:
|
|
1278
|
+
if (
|
|
1279
|
+
ssl_context.check_hostname
|
|
1280
|
+
and server_hostname is not None
|
|
1281
|
+
and isinstance(self._sock, ssl.SSLSocket)
|
|
1282
|
+
and getattr(self._sock, "server_hostname", None) not in (None, server_hostname)
|
|
1283
|
+
):
|
|
1284
|
+
orig_host = getattr(self._sock, "server_hostname", None)
|
|
1285
|
+
self.close()
|
|
1286
|
+
raise ssl.CertificateError(
|
|
1287
|
+
f"Hostname mismatch on pre-connected TLS stream: expected {server_hostname!r}, got {orig_host!r}"
|
|
1288
|
+
)
|
|
1289
|
+
return self
|
|
1290
|
+
try:
|
|
1291
|
+
if self._enable_http2:
|
|
1292
|
+
with contextlib.suppress(NotImplementedError):
|
|
1293
|
+
ssl_context.set_alpn_protocols(["h2", "http/1.1"])
|
|
1294
|
+
else:
|
|
1295
|
+
with contextlib.suppress(NotImplementedError):
|
|
1296
|
+
ssl_context.set_alpn_protocols(["http/1.1"])
|
|
1297
|
+
self._sock.settimeout(timeout if has_timeout(timeout) else None)
|
|
1298
|
+
ssl_sock = ssl_context.wrap_socket(self._sock, server_hostname=server_hostname)
|
|
1299
|
+
self._sock = ssl_sock
|
|
1300
|
+
self._tls_started = True
|
|
1301
|
+
if self._owner is not None:
|
|
1302
|
+
self._owner._record_sock(ssl_sock)
|
|
1303
|
+
return self
|
|
1304
|
+
except Exception:
|
|
1305
|
+
self.close()
|
|
1306
|
+
raise
|
|
1307
|
+
|
|
1308
|
+
def get_extra_info(self, info: str):
|
|
1309
|
+
if self._sock is None:
|
|
1310
|
+
return None
|
|
1311
|
+
if info == "ssl_object" and isinstance(self._sock, ssl.SSLSocket):
|
|
1312
|
+
return self._sock._sslobj
|
|
1313
|
+
if info == "client_addr":
|
|
1314
|
+
return self._sock.getsockname()
|
|
1315
|
+
if info == "server_addr":
|
|
1316
|
+
return self._sock.getpeername()
|
|
1317
|
+
if info == "socket":
|
|
1318
|
+
return self._sock
|
|
1319
|
+
if info == "is_readable":
|
|
1320
|
+
return _is_socket_readable(self._sock)
|
|
1321
|
+
return None
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
class _Httplib2NetworkBackend(_HttpcoreNetworkBackend):
|
|
1325
|
+
"""Custom httpcore NetworkBackend that uses httplib2's socket/PySocks connection logic
|
|
1326
|
+
and supports pre-connected streams from explicit conn.connect() calls.
|
|
1327
|
+
"""
|
|
1328
|
+
|
|
1329
|
+
def __init__(self, owner):
|
|
1330
|
+
self._owner = owner
|
|
1331
|
+
self._preconnected_streams = queue.Queue()
|
|
1332
|
+
|
|
1333
|
+
def push_preconnected(self, stream: _Httplib2SyncStream) -> None:
|
|
1334
|
+
self._preconnected_streams.put(stream)
|
|
1335
|
+
|
|
1336
|
+
def clear_preconnected(self) -> None:
|
|
1337
|
+
while True:
|
|
1338
|
+
try:
|
|
1339
|
+
stream = self._preconnected_streams.get_nowait()
|
|
1340
|
+
except queue.Empty:
|
|
1341
|
+
break
|
|
1342
|
+
with contextlib.suppress(Exception):
|
|
1343
|
+
stream.close()
|
|
1344
|
+
|
|
1345
|
+
def connect_tcp(
|
|
1346
|
+
self,
|
|
1347
|
+
host: str,
|
|
1348
|
+
port: int,
|
|
1349
|
+
timeout: float | None = None,
|
|
1350
|
+
local_address: str | None = None,
|
|
1351
|
+
socket_options=None,
|
|
1352
|
+
) -> _HttpcoreNetworkStream:
|
|
1353
|
+
while True:
|
|
1354
|
+
try:
|
|
1355
|
+
stream = self._preconnected_streams.get_nowait()
|
|
1356
|
+
except queue.Empty:
|
|
1357
|
+
break
|
|
1358
|
+
if stream._sock is not None:
|
|
1359
|
+
return stream
|
|
1360
|
+
|
|
1361
|
+
sock = self._owner._connect_raw_socket(
|
|
1362
|
+
include_flowinfo=not self._owner._is_https,
|
|
1363
|
+
timeout=timeout,
|
|
1364
|
+
)
|
|
1365
|
+
stream = _Httplib2SyncStream(
|
|
1366
|
+
sock,
|
|
1367
|
+
owner=self._owner,
|
|
1368
|
+
enable_http2=getattr(self._owner, "enable_http2", False),
|
|
1369
|
+
)
|
|
1370
|
+
self._owner._record_sock(sock)
|
|
1371
|
+
return stream
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
class _Httplib2HTTP2Connection(_HttpcoreHTTP2Connection):
|
|
1375
|
+
"""High-concurrency thread-safe HTTP/2 connection subclass for httpcore.
|
|
1376
|
+
|
|
1377
|
+
Unlocks all local concurrent streams immediately on the initial connection flight,
|
|
1378
|
+
strips HTTP/1.1 connection-specific headers forbidden by RFC 9113, and serializes
|
|
1379
|
+
all h2 state machine mutations across concurrent worker threads.
|
|
1380
|
+
"""
|
|
1381
|
+
|
|
1382
|
+
_FORBIDDEN_H2_HEADERS = frozenset(
|
|
1383
|
+
(
|
|
1384
|
+
b"host",
|
|
1385
|
+
b"transfer-encoding",
|
|
1386
|
+
b"connection",
|
|
1387
|
+
b"keep-alive",
|
|
1388
|
+
b"proxy-connection",
|
|
1389
|
+
b"upgrade",
|
|
1390
|
+
)
|
|
1391
|
+
)
|
|
1392
|
+
|
|
1393
|
+
def __init__(self, *args, **kwargs):
|
|
1394
|
+
super().__init__(*args, **kwargs)
|
|
1395
|
+
self._write_lock = threading.RLock()
|
|
1396
|
+
|
|
1397
|
+
def handle_request(self, request: _HttpcoreRequest) -> _HttpcoreResponse:
|
|
1398
|
+
with self._state_lock:
|
|
1399
|
+
if self._state in (_HttpcoreHTTPConnectionState.ACTIVE, _HttpcoreHTTPConnectionState.IDLE):
|
|
1400
|
+
self._request_count += 1
|
|
1401
|
+
self._state = _HttpcoreHTTPConnectionState.ACTIVE
|
|
1402
|
+
else:
|
|
1403
|
+
raise httpcore.ConnectionNotAvailable()
|
|
1404
|
+
|
|
1405
|
+
with self._init_lock:
|
|
1406
|
+
if not self._sent_connection_init:
|
|
1407
|
+
try:
|
|
1408
|
+
self._send_connection_init(request=request)
|
|
1409
|
+
except BaseException:
|
|
1410
|
+
self.close()
|
|
1411
|
+
raise
|
|
1412
|
+
self._sent_connection_init = True
|
|
1413
|
+
local_settings_max_streams = self._h2_state.local_settings.max_concurrent_streams
|
|
1414
|
+
self._max_streams = local_settings_max_streams
|
|
1415
|
+
self._max_streams_semaphore = _HttpcoreSemaphore(local_settings_max_streams)
|
|
1416
|
+
|
|
1417
|
+
self._max_streams_semaphore.acquire()
|
|
1418
|
+
stream_id = None
|
|
1419
|
+
try:
|
|
1420
|
+
with self._write_lock:
|
|
1421
|
+
if (
|
|
1422
|
+
self._state == _HttpcoreHTTPConnectionState.CLOSED
|
|
1423
|
+
or self._connection_error
|
|
1424
|
+
or self._connection_terminated
|
|
1425
|
+
):
|
|
1426
|
+
self._request_count -= 1
|
|
1427
|
+
self._max_streams_semaphore.release()
|
|
1428
|
+
raise httpcore.ConnectionNotAvailable()
|
|
1429
|
+
try:
|
|
1430
|
+
stream_id = self._h2_state.get_next_available_stream_id()
|
|
1431
|
+
self._events[stream_id] = []
|
|
1432
|
+
except _h2_exceptions.NoAvailableStreamIDError:
|
|
1433
|
+
self._used_all_stream_ids = True
|
|
1434
|
+
self._request_count -= 1
|
|
1435
|
+
self._max_streams_semaphore.release()
|
|
1436
|
+
raise httpcore.ConnectionNotAvailable()
|
|
1437
|
+
self._send_request_headers(request=request, stream_id=stream_id)
|
|
1438
|
+
|
|
1439
|
+
self._send_request_body(request=request, stream_id=stream_id)
|
|
1440
|
+
status, headers = self._receive_response(request=request, stream_id=stream_id)
|
|
1441
|
+
return _HttpcoreResponse(
|
|
1442
|
+
status=status,
|
|
1443
|
+
headers=headers,
|
|
1444
|
+
content=_HttpcoreHTTP2ConnectionByteStream(self, request, stream_id=stream_id),
|
|
1445
|
+
extensions={
|
|
1446
|
+
"http_version": b"HTTP/2",
|
|
1447
|
+
"network_stream": self._network_stream,
|
|
1448
|
+
"stream_id": stream_id,
|
|
1449
|
+
},
|
|
1450
|
+
)
|
|
1451
|
+
except BaseException as exc:
|
|
1452
|
+
if stream_id is not None:
|
|
1453
|
+
self._response_closed(stream_id=stream_id)
|
|
1454
|
+
if isinstance(exc, _h2_exceptions.ProtocolError):
|
|
1455
|
+
if self._connection_terminated:
|
|
1456
|
+
raise _HttpcoreRemoteProtocolError(self._connection_terminated)
|
|
1457
|
+
raise _HttpcoreLocalProtocolError(exc)
|
|
1458
|
+
raise
|
|
1459
|
+
|
|
1460
|
+
def _send_request_headers(self, request: _HttpcoreRequest, stream_id: int) -> None:
|
|
1461
|
+
has_nonempty_body = request.extensions.get("has_nonempty_body")
|
|
1462
|
+
if has_nonempty_body is None:
|
|
1463
|
+
end_stream = not _httpcore_has_body_headers(request)
|
|
1464
|
+
else:
|
|
1465
|
+
end_stream = not has_nonempty_body
|
|
1466
|
+
|
|
1467
|
+
hosts = [v for k, v in request.headers if k.lower() == b"host"]
|
|
1468
|
+
authority = hosts[0] if hosts else request.url.origin.host
|
|
1469
|
+
|
|
1470
|
+
headers = [
|
|
1471
|
+
(b":method", request.method),
|
|
1472
|
+
(b":authority", authority),
|
|
1473
|
+
(b":scheme", request.url.scheme),
|
|
1474
|
+
(b":path", request.url.target),
|
|
1475
|
+
]
|
|
1476
|
+
for k, v in request.headers:
|
|
1477
|
+
kl = k.lower()
|
|
1478
|
+
if kl.startswith(b":"):
|
|
1479
|
+
raise _HttpcoreLocalProtocolError(f"Illegal HTTP/2 pseudo-header in request headers: {k!r}")
|
|
1480
|
+
if kl in self._FORBIDDEN_H2_HEADERS:
|
|
1481
|
+
continue
|
|
1482
|
+
if kl == b"te" and v.lower() != b"trailers":
|
|
1483
|
+
continue
|
|
1484
|
+
headers.append((kl, v))
|
|
1485
|
+
|
|
1486
|
+
with self._write_lock:
|
|
1487
|
+
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
|
|
1488
|
+
self._h2_state.increment_flow_control_window(2**24, stream_id=stream_id)
|
|
1489
|
+
self._write_outgoing_data(request)
|
|
1490
|
+
request.extensions["request_sent"] = True
|
|
1491
|
+
|
|
1492
|
+
def _send_request_body(self, request: _HttpcoreRequest, stream_id: int) -> None:
|
|
1493
|
+
has_nonempty_body = request.extensions.get("has_nonempty_body")
|
|
1494
|
+
if has_nonempty_body is False:
|
|
1495
|
+
return
|
|
1496
|
+
if has_nonempty_body is None and not _httpcore_has_body_headers(request):
|
|
1497
|
+
return
|
|
1498
|
+
for data in request.stream:
|
|
1499
|
+
self._send_stream_data(request, stream_id, data)
|
|
1500
|
+
self._send_end_stream(request, stream_id)
|
|
1501
|
+
|
|
1502
|
+
def _send_stream_data(self, request: _HttpcoreRequest, stream_id: int, data: bytes) -> None:
|
|
1503
|
+
while data:
|
|
1504
|
+
max_flow = self._wait_for_outgoing_flow(request, stream_id)
|
|
1505
|
+
chunk_size = min(len(data), max_flow)
|
|
1506
|
+
chunk, data = data[:chunk_size], data[chunk_size:]
|
|
1507
|
+
with self._write_lock:
|
|
1508
|
+
self._h2_state.send_data(stream_id, chunk)
|
|
1509
|
+
self._write_outgoing_data(request)
|
|
1510
|
+
|
|
1511
|
+
def _send_end_stream(self, request: _HttpcoreRequest, stream_id: int) -> None:
|
|
1512
|
+
with self._write_lock:
|
|
1513
|
+
self._h2_state.end_stream(stream_id)
|
|
1514
|
+
self._write_outgoing_data(request)
|
|
1515
|
+
|
|
1516
|
+
def _wait_for_outgoing_flow(self, request: _HttpcoreRequest, stream_id: int) -> int:
|
|
1517
|
+
with self._write_lock:
|
|
1518
|
+
local_flow = self._h2_state.local_flow_control_window(stream_id)
|
|
1519
|
+
max_frame_size = self._h2_state.max_outbound_frame_size
|
|
1520
|
+
flow = min(local_flow, max_frame_size)
|
|
1521
|
+
while flow == 0:
|
|
1522
|
+
self._receive_events(request)
|
|
1523
|
+
with self._write_lock:
|
|
1524
|
+
local_flow = self._h2_state.local_flow_control_window(stream_id)
|
|
1525
|
+
max_frame_size = self._h2_state.max_outbound_frame_size
|
|
1526
|
+
flow = min(local_flow, max_frame_size)
|
|
1527
|
+
return flow
|
|
1528
|
+
|
|
1529
|
+
def _receive_response_body(self, request: _HttpcoreRequest, stream_id: int):
|
|
1530
|
+
while True:
|
|
1531
|
+
event = self._receive_stream_event(request, stream_id)
|
|
1532
|
+
if isinstance(event, _h2_events.DataReceived):
|
|
1533
|
+
amount = event.flow_controlled_length
|
|
1534
|
+
with self._write_lock:
|
|
1535
|
+
self._h2_state.acknowledge_received_data(amount, stream_id)
|
|
1536
|
+
with contextlib.suppress(OSError, _HttpcoreWriteError):
|
|
1537
|
+
self._write_outgoing_data(request)
|
|
1538
|
+
yield event.data
|
|
1539
|
+
elif isinstance(event, _h2_events.StreamEnded):
|
|
1540
|
+
break
|
|
1541
|
+
|
|
1542
|
+
def _read_incoming_data(self, request: _HttpcoreRequest):
|
|
1543
|
+
timeouts = request.extensions.get("timeout", {})
|
|
1544
|
+
timeout = timeouts.get("read", None)
|
|
1545
|
+
|
|
1546
|
+
if self._read_exception is not None:
|
|
1547
|
+
raise self._read_exception
|
|
1548
|
+
|
|
1549
|
+
try:
|
|
1550
|
+
data = self._network_stream.read(self.READ_NUM_BYTES, timeout)
|
|
1551
|
+
if data == b"":
|
|
1552
|
+
raise _HttpcoreRemoteProtocolError("Server disconnected")
|
|
1553
|
+
except Exception as exc:
|
|
1554
|
+
self._read_exception = exc
|
|
1555
|
+
self._connection_error = True
|
|
1556
|
+
raise exc
|
|
1557
|
+
|
|
1558
|
+
with self._write_lock:
|
|
1559
|
+
events = self._h2_state.receive_data(data)
|
|
1560
|
+
return events
|
|
1561
|
+
|
|
1562
|
+
def close(self) -> None:
|
|
1563
|
+
with self._write_lock:
|
|
1564
|
+
with contextlib.suppress(Exception):
|
|
1565
|
+
self._h2_state.close_connection()
|
|
1566
|
+
self._state = _HttpcoreHTTPConnectionState.CLOSED
|
|
1567
|
+
self._network_stream.close()
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
class _Httplib2HttpcoreConnection(_HttpcoreHTTPConnection):
|
|
1571
|
+
"""Subclass of httpcore.HTTPConnection that instantiates _Httplib2HTTP2Connection
|
|
1572
|
+
when HTTP/2 is negotiated via ALPN.
|
|
1573
|
+
"""
|
|
1574
|
+
|
|
1575
|
+
def handle_request(self, request: _HttpcoreRequest) -> _HttpcoreResponse:
|
|
1576
|
+
if not self.can_handle_request(request.url.origin):
|
|
1577
|
+
raise RuntimeError(f"Cannot handle request URL '{request.url.origin}' on '{self._origin}'")
|
|
1578
|
+
|
|
1579
|
+
try:
|
|
1580
|
+
with self._request_lock:
|
|
1581
|
+
if self._connection is None:
|
|
1582
|
+
stream = self._connect(request)
|
|
1583
|
+
ssl_object = stream.get_extra_info("ssl_object")
|
|
1584
|
+
http2_negotiated = (
|
|
1585
|
+
ssl_object is not None
|
|
1586
|
+
and ssl_object.selected_alpn_protocol() == "h2"
|
|
1587
|
+
)
|
|
1588
|
+
if http2_negotiated or (self._http2 and not self._http1):
|
|
1589
|
+
self._connection = _Httplib2HTTP2Connection(
|
|
1590
|
+
origin=self._origin,
|
|
1591
|
+
stream=stream,
|
|
1592
|
+
keepalive_expiry=self._keepalive_expiry,
|
|
1593
|
+
)
|
|
1594
|
+
else:
|
|
1595
|
+
self._connection = _HttpcoreHTTP11Connection(
|
|
1596
|
+
origin=self._origin,
|
|
1597
|
+
stream=stream,
|
|
1598
|
+
keepalive_expiry=self._keepalive_expiry,
|
|
1599
|
+
)
|
|
1600
|
+
except BaseException as exc:
|
|
1601
|
+
self._connect_failed = True
|
|
1602
|
+
raise exc
|
|
1603
|
+
|
|
1604
|
+
if isinstance(self._connection, _HttpcoreHTTP11Connection):
|
|
1605
|
+
stream = getattr(self._connection, "_network_stream", None)
|
|
1606
|
+
if isinstance(stream, _Httplib2SyncStream):
|
|
1607
|
+
stream._active_request = request
|
|
1608
|
+
return self._connection.handle_request(request)
|
|
1609
|
+
|
|
1610
|
+
|
|
1611
|
+
class _Httplib2ConnectionPool(_HttpcoreConnectionPool):
|
|
1612
|
+
"""Subclass of httpcore.ConnectionPool that creates _Httplib2HttpcoreConnection instances."""
|
|
1613
|
+
|
|
1614
|
+
def create_connection(self, origin: _HttpcoreOrigin):
|
|
1615
|
+
return _Httplib2HttpcoreConnection(
|
|
1616
|
+
origin=origin,
|
|
1617
|
+
ssl_context=self._ssl_context,
|
|
1618
|
+
keepalive_expiry=self._keepalive_expiry,
|
|
1619
|
+
http1=self._http1,
|
|
1620
|
+
http2=self._http2,
|
|
1621
|
+
retries=self._retries,
|
|
1622
|
+
local_address=self._local_address,
|
|
1623
|
+
uds=self._uds,
|
|
1624
|
+
network_backend=self._network_backend,
|
|
1625
|
+
socket_options=self._socket_options,
|
|
1626
|
+
)
|
|
1627
|
+
|
|
1628
|
+
def _assign_requests_to_connections(self):
|
|
1629
|
+
closing_connections = []
|
|
1630
|
+
queued_requests = [request for request in self._requests if request.is_queued()]
|
|
1631
|
+
|
|
1632
|
+
for connection in list(self._connections):
|
|
1633
|
+
if connection.is_closed():
|
|
1634
|
+
self._connections.remove(connection)
|
|
1635
|
+
elif queued_requests and connection.has_expired():
|
|
1636
|
+
self._connections.remove(connection)
|
|
1637
|
+
closing_connections.append(connection)
|
|
1638
|
+
elif (
|
|
1639
|
+
connection.is_idle()
|
|
1640
|
+
and sum(1 for c in self._connections if c.is_idle()) > self._max_keepalive_connections
|
|
1641
|
+
):
|
|
1642
|
+
self._connections.remove(connection)
|
|
1643
|
+
closing_connections.append(connection)
|
|
1644
|
+
|
|
1645
|
+
for pool_request in queued_requests:
|
|
1646
|
+
origin = pool_request.request.url.origin
|
|
1647
|
+
available_connections = [
|
|
1648
|
+
connection
|
|
1649
|
+
for connection in self._connections
|
|
1650
|
+
if connection.can_handle_request(origin) and connection.is_available()
|
|
1651
|
+
]
|
|
1652
|
+
idle_connections = [
|
|
1653
|
+
connection for connection in self._connections if connection.is_idle()
|
|
1654
|
+
]
|
|
1655
|
+
|
|
1656
|
+
if available_connections:
|
|
1657
|
+
connection = available_connections[0]
|
|
1658
|
+
pool_request.assign_to_connection(connection)
|
|
1659
|
+
elif len(self._connections) < self._max_connections:
|
|
1660
|
+
connection = self.create_connection(origin)
|
|
1661
|
+
self._connections.append(connection)
|
|
1662
|
+
pool_request.assign_to_connection(connection)
|
|
1663
|
+
elif idle_connections:
|
|
1664
|
+
connection = idle_connections[0]
|
|
1665
|
+
self._connections.remove(connection)
|
|
1666
|
+
closing_connections.append(connection)
|
|
1667
|
+
connection = self.create_connection(origin)
|
|
1668
|
+
self._connections.append(connection)
|
|
1669
|
+
pool_request.assign_to_connection(connection)
|
|
1670
|
+
|
|
1671
|
+
return closing_connections
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
class _Httplib2RawResponse:
|
|
1675
|
+
"""Adapter around httpcore.Response providing an http.client.HTTPResponse-compatible interface."""
|
|
1676
|
+
|
|
1677
|
+
def __init__(self, hc_resp: _HttpcoreResponse, method: str = "GET", debuglevel: int = 0):
|
|
1678
|
+
self._hc_resp = hc_resp
|
|
1679
|
+
self._method = method.upper()
|
|
1680
|
+
self.debuglevel = debuglevel
|
|
1681
|
+
self._stream_iter = iter(hc_resp.stream)
|
|
1682
|
+
self._buffer = b""
|
|
1683
|
+
self._closed = False
|
|
1684
|
+
|
|
1685
|
+
self.status = hc_resp.status
|
|
1686
|
+
http_ver = hc_resp.extensions.get("http_version", b"HTTP/1.1")
|
|
1687
|
+
if http_ver == b"HTTP/2":
|
|
1688
|
+
self.version = 20
|
|
1689
|
+
elif http_ver == b"HTTP/1.0":
|
|
1690
|
+
self.version = 10
|
|
1691
|
+
else:
|
|
1692
|
+
self.version = 11
|
|
1693
|
+
|
|
1694
|
+
reason_bytes = hc_resp.extensions.get("reason_phrase")
|
|
1695
|
+
if reason_bytes is not None:
|
|
1696
|
+
self.reason = (
|
|
1697
|
+
reason_bytes.decode("iso-8859-1")
|
|
1698
|
+
if isinstance(reason_bytes, bytes)
|
|
1699
|
+
else str(reason_bytes)
|
|
1700
|
+
)
|
|
1701
|
+
else:
|
|
1702
|
+
self.reason = http.client.responses.get(self.status, "OK")
|
|
1703
|
+
|
|
1704
|
+
self._headers_list = [
|
|
1705
|
+
(
|
|
1706
|
+
k.decode("iso-8859-1") if isinstance(k, bytes) else str(k),
|
|
1707
|
+
v.decode("iso-8859-1") if isinstance(v, bytes) else str(v),
|
|
1708
|
+
)
|
|
1709
|
+
for k, v in hc_resp.headers
|
|
1710
|
+
]
|
|
1711
|
+
|
|
1712
|
+
self.stream_id = hc_resp.extensions.get("stream_id")
|
|
1713
|
+
|
|
1714
|
+
if self.debuglevel > 0:
|
|
1715
|
+
proto_str = http_ver.decode("ascii") if isinstance(http_ver, bytes) else str(http_ver)
|
|
1716
|
+
status_line = f"{proto_str} {self.status} {self.reason}\r\n"
|
|
1717
|
+
_debug_print("reply:", repr(status_line))
|
|
1718
|
+
if self.stream_id is not None:
|
|
1719
|
+
_debug_print("stream:", str(self.stream_id))
|
|
1720
|
+
for k, v in self._headers_list:
|
|
1721
|
+
_debug_print("header:", k + ":", v)
|
|
1722
|
+
|
|
1723
|
+
def set_debuglevel(self, level: int):
|
|
1724
|
+
self.debuglevel = level
|
|
1725
|
+
|
|
1726
|
+
def getheaders(self):
|
|
1727
|
+
return list(self._headers_list)
|
|
1728
|
+
|
|
1729
|
+
def getheader(self, name, default=None):
|
|
1730
|
+
target = name.lower()
|
|
1731
|
+
vals = [v for k, v in self._headers_list if k.lower() == target]
|
|
1732
|
+
if not vals:
|
|
1733
|
+
return default
|
|
1734
|
+
return ", ".join(vals)
|
|
1735
|
+
|
|
1736
|
+
def read(self, amt=None) -> bytes:
|
|
1737
|
+
if self._method == "HEAD":
|
|
1738
|
+
self.close()
|
|
1739
|
+
return b""
|
|
1740
|
+
|
|
1741
|
+
if amt is None:
|
|
1742
|
+
chunks = [self._buffer] if self._buffer else []
|
|
1743
|
+
self._buffer = b""
|
|
1744
|
+
if not self._closed:
|
|
1745
|
+
try:
|
|
1746
|
+
for chunk in self._stream_iter:
|
|
1747
|
+
if chunk:
|
|
1748
|
+
chunks.append(chunk)
|
|
1749
|
+
except _HttpcoreRemoteProtocolError as exc:
|
|
1750
|
+
if "peer closed connection without sending complete message body" not in str(exc):
|
|
1751
|
+
self.close()
|
|
1752
|
+
raise _unwrap_httpcore_exception(exc) from exc
|
|
1753
|
+
except Exception as exc:
|
|
1754
|
+
self.close()
|
|
1755
|
+
raise _unwrap_httpcore_exception(exc) from exc
|
|
1756
|
+
finally:
|
|
1757
|
+
self.close()
|
|
1758
|
+
return b"".join(chunks)
|
|
1759
|
+
|
|
1760
|
+
while len(self._buffer) < amt and not self._closed:
|
|
1761
|
+
try:
|
|
1762
|
+
chunk = next(self._stream_iter)
|
|
1763
|
+
if chunk:
|
|
1764
|
+
self._buffer += chunk
|
|
1765
|
+
except StopIteration:
|
|
1766
|
+
self.close()
|
|
1767
|
+
break
|
|
1768
|
+
except _HttpcoreRemoteProtocolError as exc:
|
|
1769
|
+
if "peer closed connection without sending complete message body" in str(exc):
|
|
1770
|
+
self.close()
|
|
1771
|
+
break
|
|
1772
|
+
self.close()
|
|
1773
|
+
raise _unwrap_httpcore_exception(exc) from exc
|
|
1774
|
+
except Exception as exc:
|
|
1775
|
+
self.close()
|
|
1776
|
+
raise _unwrap_httpcore_exception(exc) from exc
|
|
1777
|
+
|
|
1778
|
+
result, self._buffer = self._buffer[:amt], self._buffer[amt:]
|
|
1779
|
+
return result
|
|
1780
|
+
|
|
1781
|
+
def close(self) -> None:
|
|
1782
|
+
if not self._closed:
|
|
1783
|
+
self._closed = True
|
|
1784
|
+
with contextlib.suppress(Exception):
|
|
1785
|
+
self._hc_resp.close()
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
class HTTPConnectionWithTimeout(http.client.HTTPConnection):
|
|
1789
|
+
"""HTTPConnection subclass that supports timeouts and thread-safe httpx/httpcore pooling.
|
|
1790
|
+
|
|
1791
|
+
All timeouts are in seconds. If None is passed for timeout then
|
|
1792
|
+
Python's default timeout for sockets will be used. See for example
|
|
1793
|
+
the docs of socket.setdefaulttimeout():
|
|
1794
|
+
http://docs.python.org/library/socket.html#socket.setdefaulttimeout
|
|
1795
|
+
"""
|
|
1796
|
+
|
|
1797
|
+
_is_https = False
|
|
1798
|
+
_uses_httpcore_pool = True
|
|
1799
|
+
enable_http2 = False
|
|
1800
|
+
|
|
1801
|
+
@property
|
|
1802
|
+
def debuglevel(self):
|
|
1803
|
+
if hasattr(self, "_debuglevel"):
|
|
1804
|
+
return self._debuglevel
|
|
1805
|
+
return debuglevel or getattr(http.client.HTTPConnection, "debuglevel", 0)
|
|
1806
|
+
|
|
1807
|
+
@debuglevel.setter
|
|
1808
|
+
def debuglevel(self, val):
|
|
1809
|
+
self._debuglevel = val
|
|
1810
|
+
|
|
1811
|
+
def __init__(self, host, port=None, timeout=None, proxy_info=None, pool_maxsize=DEFAULT_POOL_MAXSIZE):
|
|
1812
|
+
self._initialized = False
|
|
1813
|
+
self._override_sock = None
|
|
1814
|
+
self._last_sock = None
|
|
1815
|
+
self._local = threading.local()
|
|
1816
|
+
self._pool_maxsize = pool_maxsize
|
|
1817
|
+
|
|
1818
|
+
super().__init__(host, port=port, timeout=timeout)
|
|
1819
|
+
if isinstance(self.host, str) and self.host.startswith("[") and self.host.endswith("]"):
|
|
1820
|
+
self.host = self.host[1:-1]
|
|
1821
|
+
|
|
1822
|
+
self.proxy_info = proxy_info
|
|
1823
|
+
if proxy_info and not isinstance(proxy_info, ProxyInfo):
|
|
1824
|
+
self.proxy_info = proxy_info("http")
|
|
1825
|
+
|
|
1826
|
+
self._backend = _Httplib2NetworkBackend(self)
|
|
1827
|
+
self.httpcore_pool = _Httplib2ConnectionPool(
|
|
1828
|
+
max_connections=pool_maxsize,
|
|
1829
|
+
max_keepalive_connections=pool_maxsize,
|
|
1830
|
+
keepalive_expiry=30.0,
|
|
1831
|
+
http1=True,
|
|
1832
|
+
http2=False,
|
|
1833
|
+
network_backend=self._backend,
|
|
1834
|
+
)
|
|
1835
|
+
self._initialized = True
|
|
1836
|
+
|
|
1837
|
+
def _record_sock(self, sock):
|
|
1838
|
+
self._last_sock = sock
|
|
1839
|
+
if hasattr(self, "_local"):
|
|
1840
|
+
self._local.last_sock = sock
|
|
1841
|
+
|
|
1842
|
+
@property
|
|
1843
|
+
def sock(self):
|
|
1844
|
+
if getattr(self, "_override_sock", None) is not None:
|
|
1845
|
+
return self._override_sock
|
|
1846
|
+
local_obj = getattr(self, "_local", None)
|
|
1847
|
+
if local_obj is not None:
|
|
1848
|
+
tsock = getattr(local_obj, "last_sock", None)
|
|
1849
|
+
if tsock is not None and tsock.fileno() != -1:
|
|
1850
|
+
return tsock
|
|
1851
|
+
lsock = getattr(self, "_last_sock", None)
|
|
1852
|
+
if lsock is not None and lsock.fileno() != -1:
|
|
1853
|
+
return lsock
|
|
1854
|
+
return None
|
|
1855
|
+
|
|
1856
|
+
@sock.setter
|
|
1857
|
+
def sock(self, value):
|
|
1858
|
+
if not getattr(self, "_initialized", False):
|
|
1859
|
+
self._override_sock = value
|
|
1860
|
+
return
|
|
1861
|
+
self._override_sock = value
|
|
1862
|
+
if value is None:
|
|
1863
|
+
self._last_sock = None
|
|
1864
|
+
if hasattr(self, "_local"):
|
|
1865
|
+
self._local.last_sock = None
|
|
1866
|
+
|
|
1867
|
+
def _connect_raw_socket(self, include_flowinfo=True, timeout=None):
|
|
1868
|
+
effective_timeout = self.timeout if timeout is None else timeout
|
|
1869
|
+
if self.proxy_info and socks is None:
|
|
1870
|
+
raise ProxiesUnavailableError("Proxy support missing but proxy use was requested!")
|
|
1871
|
+
if self.proxy_info and self.proxy_info.isgood() and self.proxy_info.applies_to(self.host):
|
|
1872
|
+
use_proxy = True
|
|
1873
|
+
(
|
|
1874
|
+
proxy_type,
|
|
1875
|
+
proxy_host,
|
|
1876
|
+
proxy_port,
|
|
1877
|
+
proxy_rdns,
|
|
1878
|
+
proxy_user,
|
|
1879
|
+
proxy_pass,
|
|
1880
|
+
proxy_headers,
|
|
1881
|
+
) = self.proxy_info.astuple()
|
|
1882
|
+
host = proxy_host
|
|
1883
|
+
port = proxy_port
|
|
1884
|
+
else:
|
|
1885
|
+
use_proxy = False
|
|
1886
|
+
host = self.host
|
|
1887
|
+
port = self.port
|
|
1888
|
+
proxy_type = None
|
|
1889
|
+
proxy_headers = None
|
|
1890
|
+
|
|
1891
|
+
socket_err = None
|
|
1892
|
+
sock = None
|
|
1893
|
+
|
|
1894
|
+
addrinfos = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
|
|
1895
|
+
if isinstance(host, str) and host.lower() == "localhost":
|
|
1896
|
+
addrinfos.sort(key=lambda r: 0 if r[0] == socket.AF_INET else 1)
|
|
1897
|
+
|
|
1898
|
+
for res in addrinfos:
|
|
1899
|
+
af, socktype, proto, canonname, sa = res
|
|
1900
|
+
try:
|
|
1901
|
+
if use_proxy:
|
|
1902
|
+
sock = socks.socksocket(af, socktype, proto)
|
|
1903
|
+
sock.setproxy(
|
|
1904
|
+
proxy_type,
|
|
1905
|
+
proxy_host,
|
|
1906
|
+
proxy_port,
|
|
1907
|
+
proxy_rdns,
|
|
1908
|
+
proxy_user,
|
|
1909
|
+
proxy_pass,
|
|
1910
|
+
)
|
|
1911
|
+
else:
|
|
1912
|
+
sock = socket.socket(af, socktype, proto)
|
|
1913
|
+
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
|
1914
|
+
if has_timeout(effective_timeout):
|
|
1915
|
+
sock.settimeout(effective_timeout)
|
|
1916
|
+
if self.debuglevel > 0:
|
|
1917
|
+
_debug_print(f"connect: ({self.host}, {self.port}) ************")
|
|
1918
|
+
if use_proxy:
|
|
1919
|
+
_debug_print(
|
|
1920
|
+
f"proxy: {(proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers)} ************"
|
|
1921
|
+
)
|
|
1922
|
+
target_addr = ((self.host, self.port) + sa[2:]) if include_flowinfo else (self.host, self.port)
|
|
1923
|
+
sock.connect(target_addr)
|
|
1924
|
+
except (socket.timeout, socket.gaierror):
|
|
1925
|
+
if sock:
|
|
1926
|
+
sock.close()
|
|
1927
|
+
raise
|
|
1928
|
+
except socket.error as e:
|
|
1929
|
+
if socket_err is None or "PySocks doesn't support IPv6" not in str(e):
|
|
1930
|
+
socket_err = e
|
|
1931
|
+
if self.debuglevel > 0:
|
|
1932
|
+
_debug_print(f"connect fail: ({self.host}, {self.port})")
|
|
1933
|
+
if use_proxy:
|
|
1934
|
+
_debug_print(
|
|
1935
|
+
f"proxy: {(proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers)}"
|
|
1936
|
+
)
|
|
1937
|
+
if sock:
|
|
1938
|
+
sock.close()
|
|
1939
|
+
sock = None
|
|
1940
|
+
continue
|
|
1941
|
+
break
|
|
1942
|
+
|
|
1943
|
+
if not sock:
|
|
1944
|
+
raise socket_err
|
|
1945
|
+
return sock
|
|
1946
|
+
|
|
1947
|
+
def connect(self):
|
|
1948
|
+
"""Connect to the host and port specified in __init__."""
|
|
1949
|
+
sock = self._connect_raw_socket(include_flowinfo=True, timeout=self.timeout)
|
|
1950
|
+
stream = _Httplib2SyncStream(sock, owner=self, enable_http2=False)
|
|
1951
|
+
self._record_sock(sock)
|
|
1952
|
+
self._backend.push_preconnected(stream)
|
|
1953
|
+
|
|
1954
|
+
def _clear_pending_outcome(self):
|
|
1955
|
+
pending = getattr(self._local, "pending_outcome", None)
|
|
1956
|
+
if pending is not None:
|
|
1957
|
+
self._local.pending_outcome = None
|
|
1958
|
+
hc_resp, _ = pending
|
|
1959
|
+
if hc_resp is not None:
|
|
1960
|
+
with contextlib.suppress(Exception):
|
|
1961
|
+
hc_resp.close()
|
|
1962
|
+
|
|
1963
|
+
def request(self, method, url, body=None, headers=None, *, encode_chunked=False):
|
|
1964
|
+
self._clear_pending_outcome()
|
|
1965
|
+
|
|
1966
|
+
raw_headers = []
|
|
1967
|
+
has_host = False
|
|
1968
|
+
has_content_length = False
|
|
1969
|
+
has_transfer_encoding = False
|
|
1970
|
+
|
|
1971
|
+
if headers:
|
|
1972
|
+
items = headers.items() if hasattr(headers, "items") else headers
|
|
1973
|
+
for k, v in items:
|
|
1974
|
+
kb = k.encode("iso-8859-1") if isinstance(k, str) else bytes(k)
|
|
1975
|
+
vb = v.encode("iso-8859-1") if isinstance(v, str) else bytes(v)
|
|
1976
|
+
if not kb or kb.startswith(b":") or any(b <= 0x20 or b == 0x7F or b == 0x3A for b in kb):
|
|
1977
|
+
raise http.client.HTTPException(f"Invalid header name {kb!r}")
|
|
1978
|
+
if any(b in (0x00, 0x0A, 0x0D) for b in vb):
|
|
1979
|
+
raise http.client.HTTPException(f"Invalid header value {vb!r}")
|
|
1980
|
+
kl = kb.lower()
|
|
1981
|
+
if kl == b"host":
|
|
1982
|
+
has_host = True
|
|
1983
|
+
elif kl == b"content-length":
|
|
1984
|
+
has_content_length = True
|
|
1985
|
+
elif kl == b"transfer-encoding":
|
|
1986
|
+
has_transfer_encoding = True
|
|
1987
|
+
raw_headers.append((kb, vb))
|
|
1988
|
+
|
|
1989
|
+
if not has_host:
|
|
1990
|
+
host_str = self.host
|
|
1991
|
+
if ":" in host_str and not host_str.startswith("["):
|
|
1992
|
+
host_str = f"[{host_str}]"
|
|
1993
|
+
default_port = 443 if self._is_https else 80
|
|
1994
|
+
if self.port and self.port != default_port:
|
|
1995
|
+
host_hdr = f"{host_str}:{self.port}".encode("ascii")
|
|
1996
|
+
else:
|
|
1997
|
+
host_hdr = host_str.encode("ascii")
|
|
1998
|
+
raw_headers.insert(0, (b"Host", host_hdr))
|
|
1999
|
+
|
|
2000
|
+
method_bytes = method.encode("ascii") if isinstance(method, str) else bytes(method)
|
|
2001
|
+
if not method_bytes or any(b <= 0x20 or b >= 0x7F for b in method_bytes):
|
|
2002
|
+
raise http.client.InvalidURL(f"Invalid method {method!r}")
|
|
2003
|
+
upper_method = method.upper() if isinstance(method, str) else method.decode("ascii").upper()
|
|
2004
|
+
if body is not None and hasattr(body, "read"):
|
|
2005
|
+
body = body.read()
|
|
2006
|
+
if isinstance(body, str):
|
|
2007
|
+
body = body.encode("utf-8")
|
|
2008
|
+
|
|
2009
|
+
has_nonempty_body = False
|
|
2010
|
+
if body is None:
|
|
2011
|
+
content_bytes = b""
|
|
2012
|
+
if upper_method in ("POST", "PUT", "PATCH") and not has_content_length and not has_transfer_encoding:
|
|
2013
|
+
raw_headers.append((b"Content-Length", b"0"))
|
|
2014
|
+
elif isinstance(body, (bytes, bytearray, memoryview)):
|
|
2015
|
+
content_bytes = bytes(body)
|
|
2016
|
+
has_nonempty_body = len(content_bytes) > 0
|
|
2017
|
+
if not has_content_length and not has_transfer_encoding:
|
|
2018
|
+
if has_nonempty_body or upper_method in ("POST", "PUT", "PATCH"):
|
|
2019
|
+
raw_headers.append((b"Content-Length", str(len(content_bytes)).encode("ascii")))
|
|
2020
|
+
else:
|
|
2021
|
+
chunks = [c.encode("utf-8") if isinstance(c, str) else bytes(c) for c in body]
|
|
2022
|
+
content_bytes = b"".join(chunks)
|
|
2023
|
+
has_nonempty_body = len(content_bytes) > 0
|
|
2024
|
+
if not has_content_length and not has_transfer_encoding:
|
|
2025
|
+
raw_headers.append((b"Content-Length", str(len(content_bytes)).encode("ascii")))
|
|
2026
|
+
|
|
2027
|
+
scheme_bytes = b"https" if self._is_https else b"http"
|
|
2028
|
+
host_bytes = self.host.encode("ascii")
|
|
2029
|
+
target_bytes = url.encode("ascii") if isinstance(url, str) else bytes(url)
|
|
2030
|
+
if not target_bytes:
|
|
2031
|
+
target_bytes = b"/"
|
|
2032
|
+
if any(b <= 0x20 or b == 0x7F for b in target_bytes):
|
|
2033
|
+
raise http.client.InvalidURL(f"URL can't contain control characters. {url!r}")
|
|
2034
|
+
|
|
2035
|
+
effective_timeout = socket.getdefaulttimeout() if self.timeout is None else self.timeout
|
|
2036
|
+
timeout_ext = {
|
|
2037
|
+
"connect": effective_timeout,
|
|
2038
|
+
"read": effective_timeout,
|
|
2039
|
+
"write": effective_timeout,
|
|
2040
|
+
"pool": effective_timeout,
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
hc_req = _HttpcoreRequest(
|
|
2044
|
+
method=method_bytes,
|
|
2045
|
+
url=_HttpcoreURL(
|
|
2046
|
+
scheme=scheme_bytes,
|
|
2047
|
+
host=host_bytes,
|
|
2048
|
+
port=self.port,
|
|
2049
|
+
target=target_bytes,
|
|
2050
|
+
),
|
|
2051
|
+
headers=raw_headers,
|
|
2052
|
+
content=content_bytes,
|
|
2053
|
+
extensions={
|
|
2054
|
+
"timeout": timeout_ext,
|
|
2055
|
+
"has_nonempty_body": has_nonempty_body,
|
|
2056
|
+
"request_sent": False,
|
|
2057
|
+
},
|
|
2058
|
+
)
|
|
2059
|
+
hc_req.method = method_bytes
|
|
2060
|
+
|
|
2061
|
+
if self.debuglevel > 0:
|
|
2062
|
+
is_h2 = False
|
|
2063
|
+
if self._is_https and getattr(self, "enable_http2", False):
|
|
2064
|
+
pool_conns = getattr(self.httpcore_pool, "_connections", [])
|
|
2065
|
+
for c in pool_conns:
|
|
2066
|
+
sub_c = getattr(c, "_connection", None)
|
|
2067
|
+
if isinstance(sub_c, _Httplib2HTTP2Connection):
|
|
2068
|
+
is_h2 = True
|
|
2069
|
+
break
|
|
2070
|
+
proto = "HTTP/2" if is_h2 else "HTTP/1.1"
|
|
2071
|
+
req_line = f"{upper_method} {target_bytes.decode('ascii', errors='replace')} {proto}\r\n"
|
|
2072
|
+
hdr_str = "".join(
|
|
2073
|
+
f"{k.decode('iso-8859-1') if isinstance(k, bytes) else k}: {v.decode('iso-8859-1') if isinstance(v, bytes) else v}\r\n"
|
|
2074
|
+
for k, v in raw_headers
|
|
2075
|
+
)
|
|
2076
|
+
_debug_print("send:", repr((req_line + hdr_str + "\r\n").encode("iso-8859-1")))
|
|
2077
|
+
if has_nonempty_body:
|
|
2078
|
+
_debug_print("send:", repr(content_bytes))
|
|
2079
|
+
|
|
2080
|
+
try:
|
|
2081
|
+
hc_resp = self.httpcore_pool.handle_request(hc_req)
|
|
2082
|
+
net_stream = hc_resp.extensions.get("network_stream")
|
|
2083
|
+
if net_stream is not None:
|
|
2084
|
+
sock_obj = net_stream.get_extra_info("socket")
|
|
2085
|
+
if sock_obj is not None:
|
|
2086
|
+
self._record_sock(sock_obj)
|
|
2087
|
+
self._local.pending_outcome = (hc_resp, upper_method)
|
|
2088
|
+
except Exception as exc:
|
|
2089
|
+
unwrapped = _unwrap_httpcore_exception(exc)
|
|
2090
|
+
if hc_req.extensions.get("request_sent"):
|
|
2091
|
+
self._local.pending_outcome = (None, unwrapped)
|
|
2092
|
+
else:
|
|
2093
|
+
raise unwrapped
|
|
2094
|
+
|
|
2095
|
+
def getresponse(self):
|
|
2096
|
+
pending = getattr(self._local, "pending_outcome", None)
|
|
2097
|
+
if pending is None:
|
|
2098
|
+
raise http.client.ResponseNotReady()
|
|
2099
|
+
self._local.pending_outcome = None
|
|
2100
|
+
hc_resp, second = pending
|
|
2101
|
+
if hc_resp is None:
|
|
2102
|
+
raise second
|
|
2103
|
+
return _Httplib2RawResponse(hc_resp, method=second, debuglevel=self.debuglevel)
|
|
2104
|
+
|
|
2105
|
+
def close(self):
|
|
2106
|
+
self._override_sock = None
|
|
2107
|
+
self._last_sock = None
|
|
2108
|
+
if hasattr(self, "_local"):
|
|
2109
|
+
self._local.last_sock = None
|
|
2110
|
+
self._clear_pending_outcome()
|
|
2111
|
+
if hasattr(self, "_backend"):
|
|
2112
|
+
self._backend.clear_preconnected()
|
|
2113
|
+
if hasattr(self, "httpcore_pool"):
|
|
2114
|
+
with contextlib.suppress(Exception):
|
|
2115
|
+
self.httpcore_pool.close()
|
|
2116
|
+
|
|
2117
|
+
def close_all(self):
|
|
2118
|
+
self.close()
|
|
2119
|
+
|
|
2120
|
+
|
|
2121
|
+
class HTTPSConnectionWithTimeout(HTTPConnectionWithTimeout, http.client.HTTPSConnection):
|
|
2122
|
+
"""This class allows communication via SSL using httpx/httpcore with HTTP/2 multiplexing.
|
|
2123
|
+
|
|
2124
|
+
All timeouts are in seconds. If None is passed for timeout then
|
|
2125
|
+
Python's default timeout for sockets will be used. See for example
|
|
2126
|
+
the docs of socket.setdefaulttimeout():
|
|
2127
|
+
http://docs.python.org/library/socket.html#socket.setdefaulttimeout
|
|
2128
|
+
"""
|
|
2129
|
+
|
|
2130
|
+
_is_https = True
|
|
2131
|
+
enable_http2 = True
|
|
2132
|
+
|
|
2133
|
+
def __init__(
|
|
2134
|
+
self,
|
|
2135
|
+
host,
|
|
2136
|
+
port=None,
|
|
2137
|
+
key_file=None,
|
|
2138
|
+
cert_file=None,
|
|
2139
|
+
timeout=None,
|
|
2140
|
+
proxy_info=None,
|
|
2141
|
+
ca_certs=None,
|
|
2142
|
+
disable_ssl_certificate_validation=False,
|
|
2143
|
+
tls_maximum_version=None,
|
|
2144
|
+
tls_minimum_version=None,
|
|
2145
|
+
key_password=None,
|
|
2146
|
+
enable_http2=None,
|
|
2147
|
+
pool_maxsize=DEFAULT_POOL_MAXSIZE,
|
|
2148
|
+
):
|
|
2149
|
+
self._initialized = False
|
|
2150
|
+
self._override_sock = None
|
|
2151
|
+
self._last_sock = None
|
|
2152
|
+
self._local = threading.local()
|
|
2153
|
+
self._pool_maxsize = pool_maxsize
|
|
2154
|
+
|
|
2155
|
+
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
|
|
2156
|
+
self.ca_certs = _resolve_ca_certs(ca_certs)
|
|
2157
|
+
self.tls_maximum_version = tls_maximum_version
|
|
2158
|
+
self.tls_minimum_version = tls_minimum_version
|
|
2159
|
+
self.key_file = key_file
|
|
2160
|
+
self.cert_file = cert_file
|
|
2161
|
+
self.key_password = key_password
|
|
2162
|
+
|
|
2163
|
+
if enable_http2 is None:
|
|
2164
|
+
enable_http2 = getattr(self.__class__, "enable_http2", ENABLE_HTTP2)
|
|
2165
|
+
effective_h2 = bool(enable_http2) and bool(ENABLE_HTTP2)
|
|
2166
|
+
self.enable_http2 = effective_h2
|
|
2167
|
+
|
|
2168
|
+
self.proxy_info = proxy_info
|
|
2169
|
+
if proxy_info and not isinstance(proxy_info, ProxyInfo):
|
|
2170
|
+
self.proxy_info = proxy_info("https")
|
|
2171
|
+
|
|
2172
|
+
context = _build_ssl_context(
|
|
2173
|
+
self.disable_ssl_certificate_validation,
|
|
2174
|
+
self.ca_certs,
|
|
2175
|
+
cert_file,
|
|
2176
|
+
key_file,
|
|
2177
|
+
maximum_version=tls_maximum_version,
|
|
2178
|
+
minimum_version=tls_minimum_version,
|
|
2179
|
+
key_password=key_password,
|
|
2180
|
+
enable_http2=effective_h2,
|
|
2181
|
+
)
|
|
2182
|
+
http.client.HTTPSConnection.__init__(
|
|
2183
|
+
self,
|
|
2184
|
+
host,
|
|
2185
|
+
port=port,
|
|
2186
|
+
timeout=timeout,
|
|
2187
|
+
context=context,
|
|
2188
|
+
)
|
|
2189
|
+
if isinstance(self.host, str) and self.host.startswith("[") and self.host.endswith("]"):
|
|
2190
|
+
self.host = self.host[1:-1]
|
|
2191
|
+
|
|
2192
|
+
self._backend = _Httplib2NetworkBackend(self)
|
|
2193
|
+
self.httpcore_pool = _Httplib2ConnectionPool(
|
|
2194
|
+
ssl_context=self._context,
|
|
2195
|
+
max_connections=pool_maxsize,
|
|
2196
|
+
max_keepalive_connections=pool_maxsize,
|
|
2197
|
+
keepalive_expiry=30.0,
|
|
2198
|
+
http1=True,
|
|
2199
|
+
http2=effective_h2,
|
|
2200
|
+
network_backend=self._backend,
|
|
2201
|
+
)
|
|
2202
|
+
self._initialized = True
|
|
2203
|
+
|
|
2204
|
+
def downgrade_to_http1(self):
|
|
2205
|
+
"""Downgrade connection to HTTP/1.1 by disabling HTTP/2 ALPN and recreating the connection pool."""
|
|
2206
|
+
self.enable_http2 = False
|
|
2207
|
+
self.close()
|
|
2208
|
+
if hasattr(self, "_context") and self._context is not None:
|
|
2209
|
+
with contextlib.suppress(NotImplementedError, ssl.SSLError):
|
|
2210
|
+
self._context.set_alpn_protocols(["http/1.1"])
|
|
2211
|
+
self._backend = _Httplib2NetworkBackend(self)
|
|
2212
|
+
self.httpcore_pool = _Httplib2ConnectionPool(
|
|
2213
|
+
ssl_context=self._context,
|
|
2214
|
+
max_connections=self._pool_maxsize,
|
|
2215
|
+
max_keepalive_connections=self._pool_maxsize,
|
|
2216
|
+
keepalive_expiry=30.0,
|
|
2217
|
+
http1=True,
|
|
2218
|
+
http2=False,
|
|
2219
|
+
network_backend=self._backend,
|
|
2220
|
+
)
|
|
2221
|
+
|
|
2222
|
+
def connect(self):
|
|
2223
|
+
"""Connect to a host on a given (SSL) port."""
|
|
2224
|
+
sock = None
|
|
2225
|
+
ssl_sock = None
|
|
2226
|
+
try:
|
|
2227
|
+
sock = self._connect_raw_socket(include_flowinfo=False, timeout=self.timeout)
|
|
2228
|
+
if self.enable_http2 and ENABLE_HTTP2:
|
|
2229
|
+
with contextlib.suppress(NotImplementedError):
|
|
2230
|
+
self._context.set_alpn_protocols(["h2", "http/1.1"])
|
|
2231
|
+
else:
|
|
2232
|
+
with contextlib.suppress(NotImplementedError):
|
|
2233
|
+
self._context.set_alpn_protocols(["http/1.1"])
|
|
2234
|
+
ssl_sock = self._context.wrap_socket(sock, server_hostname=self.host)
|
|
2235
|
+
except (ssl.SSLError, ssl.CertificateError):
|
|
2236
|
+
if sock:
|
|
2237
|
+
sock.close()
|
|
2238
|
+
if ssl_sock:
|
|
2239
|
+
ssl_sock.close()
|
|
2240
|
+
self._last_sock = None
|
|
2241
|
+
raise
|
|
2242
|
+
|
|
2243
|
+
stream = _Httplib2SyncStream(ssl_sock, owner=self, enable_http2=bool(self.enable_http2 and ENABLE_HTTP2))
|
|
2244
|
+
self._record_sock(ssl_sock)
|
|
2245
|
+
self._backend.push_preconnected(stream)
|
|
2246
|
+
|
|
2247
|
+
|
|
2248
|
+
SCHEME_TO_CONNECTION = {
|
|
2249
|
+
"http": HTTPConnectionWithTimeout,
|
|
2250
|
+
"https": HTTPSConnectionWithTimeout,
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
|
|
2254
|
+
_HTTP_LOCKS_GUARD = threading.RLock()
|
|
2255
|
+
|
|
2256
|
+
|
|
2257
|
+
def _get_http_lock(http_instance):
|
|
2258
|
+
lock = getattr(http_instance, "_lock", None)
|
|
2259
|
+
if lock is None:
|
|
2260
|
+
with _HTTP_LOCKS_GUARD:
|
|
2261
|
+
lock = getattr(http_instance, "_lock", None)
|
|
2262
|
+
if lock is None:
|
|
2263
|
+
lock = threading.RLock()
|
|
2264
|
+
http_instance._lock = lock
|
|
2265
|
+
return lock
|
|
2266
|
+
|
|
2267
|
+
|
|
2268
|
+
class Http:
|
|
2269
|
+
"""An HTTP client that handles:
|
|
2270
|
+
|
|
2271
|
+
- all methods
|
|
2272
|
+
- caching
|
|
2273
|
+
- ETags
|
|
2274
|
+
- compression,
|
|
2275
|
+
- HTTPS
|
|
2276
|
+
- Basic
|
|
2277
|
+
- Digest
|
|
2278
|
+
- WSSE
|
|
2279
|
+
|
|
2280
|
+
and more, powered by httpx/httpcore with HTTP/2 multiplexing and thread-safe connection pooling.
|
|
2281
|
+
"""
|
|
2282
|
+
|
|
2283
|
+
enable_http2 = True
|
|
2284
|
+
auto_downgrade_http2 = True
|
|
2285
|
+
pool_maxsize = DEFAULT_POOL_MAXSIZE
|
|
2286
|
+
|
|
2287
|
+
def __init__(
|
|
2288
|
+
self,
|
|
2289
|
+
cache=None,
|
|
2290
|
+
timeout=None,
|
|
2291
|
+
proxy_info=proxy_info_from_environment,
|
|
2292
|
+
ca_certs=None,
|
|
2293
|
+
disable_ssl_certificate_validation=False,
|
|
2294
|
+
tls_maximum_version=None,
|
|
2295
|
+
tls_minimum_version=None,
|
|
2296
|
+
decode_limit_hard=None,
|
|
2297
|
+
decode_limit_safe=None,
|
|
2298
|
+
decode_limit_ratio=None,
|
|
2299
|
+
decode_limit_chunk=None,
|
|
2300
|
+
enable_http2=None,
|
|
2301
|
+
auto_downgrade_http2=None,
|
|
2302
|
+
pool_maxsize=None,
|
|
2303
|
+
):
|
|
2304
|
+
self.proxy_info = proxy_info
|
|
2305
|
+
self.ca_certs = ca_certs
|
|
2306
|
+
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
|
|
2307
|
+
self.tls_maximum_version = tls_maximum_version
|
|
2308
|
+
self.tls_minimum_version = tls_minimum_version
|
|
2309
|
+
if enable_http2 is not None:
|
|
2310
|
+
self.enable_http2 = bool(enable_http2)
|
|
2311
|
+
else:
|
|
2312
|
+
self.enable_http2 = getattr(self.__class__, "enable_http2", ENABLE_HTTP2)
|
|
2313
|
+
|
|
2314
|
+
if auto_downgrade_http2 is not None:
|
|
2315
|
+
self.auto_downgrade_http2 = bool(auto_downgrade_http2)
|
|
2316
|
+
else:
|
|
2317
|
+
self.auto_downgrade_http2 = getattr(self.__class__, "auto_downgrade_http2", AUTO_DOWNGRADE_HTTP2)
|
|
2318
|
+
|
|
2319
|
+
if pool_maxsize is not None:
|
|
2320
|
+
self.pool_maxsize = pool_maxsize
|
|
2321
|
+
|
|
2322
|
+
self._http2_downgraded_hosts = set()
|
|
2323
|
+
|
|
2324
|
+
# Map domain name to an httplib connection
|
|
2325
|
+
self.connections = {}
|
|
2326
|
+
# The location of the cache, for now a directory
|
|
2327
|
+
# where cached responses are held.
|
|
2328
|
+
if cache and isinstance(cache, str):
|
|
2329
|
+
self.cache = FileCache(cache)
|
|
2330
|
+
else:
|
|
2331
|
+
self.cache = cache
|
|
2332
|
+
|
|
2333
|
+
# Name/password
|
|
2334
|
+
self.credentials = Credentials()
|
|
2335
|
+
|
|
2336
|
+
# Key/cert
|
|
2337
|
+
self.certificates = KeyCerts()
|
|
2338
|
+
|
|
2339
|
+
# authorization objects
|
|
2340
|
+
self.authorizations = []
|
|
2341
|
+
|
|
2342
|
+
# If set to False then no redirects are followed, even safe ones.
|
|
2343
|
+
self.follow_redirects = True
|
|
2344
|
+
|
|
2345
|
+
self.redirect_codes = REDIRECT_CODES
|
|
2346
|
+
|
|
2347
|
+
# Which HTTP methods do we apply optimistic concurrency to, i.e.
|
|
2348
|
+
# which methods get an "if-match:" etag header added to them.
|
|
2349
|
+
self.optimistic_concurrency_methods = ["PUT", "PATCH"]
|
|
2350
|
+
|
|
2351
|
+
self.safe_methods = list(SAFE_METHODS)
|
|
2352
|
+
|
|
2353
|
+
# If 'follow_redirects' is True, and this is set to True then
|
|
2354
|
+
# all redirecs are followed, including unsafe ones.
|
|
2355
|
+
self.follow_all_redirects = False
|
|
2356
|
+
|
|
2357
|
+
self.ignore_etag = False
|
|
2358
|
+
|
|
2359
|
+
self.force_exception_to_status_code = False
|
|
2360
|
+
|
|
2361
|
+
self.timeout = timeout
|
|
2362
|
+
|
|
2363
|
+
# Keep Authorization: headers on a redirect.
|
|
2364
|
+
self.forward_authorization_headers = False
|
|
2365
|
+
|
|
2366
|
+
limit_kwargs = dict(
|
|
2367
|
+
hard_limit=try_value_or_env(
|
|
2368
|
+
int, decode_limit_hard, "httplib2_decode_limit_hard"
|
|
2369
|
+
),
|
|
2370
|
+
safe_limit=try_value_or_env(
|
|
2371
|
+
int, decode_limit_safe, "httplib2_decode_limit_safe"
|
|
2372
|
+
),
|
|
2373
|
+
ratio=try_value_or_env(
|
|
2374
|
+
float, decode_limit_ratio, "httplib2_decode_limit_ratio"
|
|
2375
|
+
),
|
|
2376
|
+
chunk_size=try_value_or_env(
|
|
2377
|
+
int, decode_limit_chunk, "httplib2_decode_limit_chunk"
|
|
2378
|
+
),
|
|
2379
|
+
)
|
|
2380
|
+
self.limit_kwargs = {k: v for k, v in limit_kwargs.items() if v is not None}
|
|
2381
|
+
|
|
2382
|
+
def close(self):
|
|
2383
|
+
"""Close persistent connections, clear sensitive data."""
|
|
2384
|
+
with _get_http_lock(self):
|
|
2385
|
+
existing, self.connections = self.connections, {}
|
|
2386
|
+
for _, c in existing.items():
|
|
2387
|
+
getattr(c, "close_all", c.close)()
|
|
2388
|
+
self.certificates.clear()
|
|
2389
|
+
self.clear_credentials()
|
|
2390
|
+
|
|
2391
|
+
def force_http11(self):
|
|
2392
|
+
"""Force this Http instance to use HTTP/1.1 only and downgrade all active connections."""
|
|
2393
|
+
self.enable_http2 = False
|
|
2394
|
+
with _get_http_lock(self):
|
|
2395
|
+
for conn in self.connections.values():
|
|
2396
|
+
if hasattr(conn, "downgrade_to_http1"):
|
|
2397
|
+
conn.downgrade_to_http1()
|
|
2398
|
+
|
|
2399
|
+
def downgrade_to_http1(self):
|
|
2400
|
+
"""Alias for force_http11()."""
|
|
2401
|
+
self.force_http11()
|
|
2402
|
+
|
|
2403
|
+
def downgrade_host_to_http1(self, authority):
|
|
2404
|
+
"""Force connections to a specific host:port authority to downgrade to HTTP/1.1."""
|
|
2405
|
+
self._record_host_downgrade(authority)
|
|
2406
|
+
host_only = authority.split(":")[0]
|
|
2407
|
+
with _get_http_lock(self):
|
|
2408
|
+
for scheme in ("https", "http"):
|
|
2409
|
+
for key in (f"{scheme}:{authority}", f"{scheme}:{host_only}"):
|
|
2410
|
+
conn = self.connections.get(key)
|
|
2411
|
+
if conn is not None and hasattr(conn, "downgrade_to_http1"):
|
|
2412
|
+
conn.downgrade_to_http1()
|
|
2413
|
+
|
|
2414
|
+
def _record_host_downgrade(self, host_or_authority):
|
|
2415
|
+
self._http2_downgraded_hosts.add(host_or_authority)
|
|
2416
|
+
with _DOWNGRADE_LOCK:
|
|
2417
|
+
DOWNGRADED_HOSTS.add(host_or_authority)
|
|
2418
|
+
if ":" not in host_or_authority or host_or_authority.endswith(":443"):
|
|
2419
|
+
host_only = host_or_authority.split(":")[0]
|
|
2420
|
+
self._http2_downgraded_hosts.add(host_only)
|
|
2421
|
+
with _DOWNGRADE_LOCK:
|
|
2422
|
+
DOWNGRADED_HOSTS.add(host_only)
|
|
2423
|
+
|
|
2424
|
+
def _is_host_downgraded(self, host_or_authority):
|
|
2425
|
+
host_only = host_or_authority.split(":")[0]
|
|
2426
|
+
if host_or_authority in self._http2_downgraded_hosts or host_only in self._http2_downgraded_hosts:
|
|
2427
|
+
return True
|
|
2428
|
+
with _DOWNGRADE_LOCK:
|
|
2429
|
+
return host_or_authority in DOWNGRADED_HOSTS or host_only in DOWNGRADED_HOSTS
|
|
2430
|
+
|
|
2431
|
+
def __getstate__(self):
|
|
2432
|
+
state_dict = copy.copy(self.__dict__)
|
|
2433
|
+
if "request" in state_dict:
|
|
2434
|
+
del state_dict["request"]
|
|
2435
|
+
if "connections" in state_dict:
|
|
2436
|
+
del state_dict["connections"]
|
|
2437
|
+
if "_lock" in state_dict:
|
|
2438
|
+
del state_dict["_lock"]
|
|
2439
|
+
return state_dict
|
|
2440
|
+
|
|
2441
|
+
def __setstate__(self, state):
|
|
2442
|
+
self.__dict__.update(state)
|
|
2443
|
+
self.connections = {}
|
|
2444
|
+
|
|
2445
|
+
def _auth_from_challenge(self, host, request_uri, headers, response, content):
|
|
2446
|
+
"""A generator that creates Authorization objects
|
|
2447
|
+
that can be applied to requests.
|
|
2448
|
+
"""
|
|
2449
|
+
challenges = auth._parse_www_authenticate(response, "www-authenticate")
|
|
2450
|
+
for cred in self.credentials.iter(host):
|
|
2451
|
+
for scheme in AUTH_SCHEME_ORDER:
|
|
2452
|
+
if scheme in challenges:
|
|
2453
|
+
yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self)
|
|
2454
|
+
|
|
2455
|
+
def add_credentials(self, name, password, domain=""):
|
|
2456
|
+
"""Add a name and password that will be used
|
|
2457
|
+
any time a request requires authentication."""
|
|
2458
|
+
self.credentials.add(name, password, domain)
|
|
2459
|
+
|
|
2460
|
+
def add_certificate(self, key, cert, domain, password=None):
|
|
2461
|
+
"""Add a key and cert that will be used
|
|
2462
|
+
any time a request requires authentication."""
|
|
2463
|
+
self.certificates.add(key, cert, domain, password)
|
|
2464
|
+
|
|
2465
|
+
def clear_credentials(self):
|
|
2466
|
+
"""Remove all the names and passwords
|
|
2467
|
+
that are used for authentication"""
|
|
2468
|
+
self.credentials.clear()
|
|
2469
|
+
self.authorizations = []
|
|
2470
|
+
|
|
2471
|
+
def _conn_request(self, conn, request_uri, method, body, headers):
|
|
2472
|
+
i = 0
|
|
2473
|
+
seen_bad_status_line = False
|
|
2474
|
+
uses_pool = getattr(conn, "_uses_httpcore_pool", False)
|
|
2475
|
+
initial_body_pos = body.tell() if hasattr(body, "tell") else None
|
|
2476
|
+
while i < RETRIES:
|
|
2477
|
+
i += 1
|
|
2478
|
+
if initial_body_pos is not None and hasattr(body, "seek"):
|
|
2479
|
+
with contextlib.suppress(Exception):
|
|
2480
|
+
body.seek(initial_body_pos)
|
|
2481
|
+
try:
|
|
2482
|
+
if not uses_pool and conn.sock is None:
|
|
2483
|
+
conn.connect()
|
|
2484
|
+
conn.request(method, request_uri, body, headers)
|
|
2485
|
+
response = conn.getresponse()
|
|
2486
|
+
except socket.timeout:
|
|
2487
|
+
conn.close()
|
|
2488
|
+
raise
|
|
2489
|
+
except socket.gaierror:
|
|
2490
|
+
conn.close()
|
|
2491
|
+
raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
|
|
2492
|
+
except Exception as exc:
|
|
2493
|
+
can_downgrade = (
|
|
2494
|
+
getattr(conn, "_is_https", False)
|
|
2495
|
+
and getattr(conn, "enable_http2", False)
|
|
2496
|
+
and getattr(self, "auto_downgrade_http2", False)
|
|
2497
|
+
)
|
|
2498
|
+
if can_downgrade and _should_downgrade_on_error(exc):
|
|
2499
|
+
if getattr(conn, "debuglevel", 0) > 0 or getattr(self, "debuglevel", 0) > 0:
|
|
2500
|
+
_debug_print(
|
|
2501
|
+
f"auto-downgrade: HTTP/2 failed for {conn.host} ({type(exc).__name__}: {exc}), "
|
|
2502
|
+
f"falling back to HTTP/1.1 and retrying"
|
|
2503
|
+
)
|
|
2504
|
+
conn_authority = (
|
|
2505
|
+
f"{conn.host}:{conn.port}"
|
|
2506
|
+
if getattr(conn, "port", None) not in (None, 443)
|
|
2507
|
+
else conn.host
|
|
2508
|
+
)
|
|
2509
|
+
self._record_host_downgrade(conn_authority)
|
|
2510
|
+
conn.downgrade_to_http1()
|
|
2511
|
+
i = 0
|
|
2512
|
+
continue
|
|
2513
|
+
|
|
2514
|
+
if isinstance(exc, (http.client.BadStatusLine, http.client.ResponseNotReady)):
|
|
2515
|
+
if not seen_bad_status_line and i == 1:
|
|
2516
|
+
i = 0
|
|
2517
|
+
seen_bad_status_line = True
|
|
2518
|
+
conn.close()
|
|
2519
|
+
if not uses_pool:
|
|
2520
|
+
conn.connect()
|
|
2521
|
+
continue
|
|
2522
|
+
else:
|
|
2523
|
+
conn.close()
|
|
2524
|
+
raise exc
|
|
2525
|
+
|
|
2526
|
+
if isinstance(exc, socket.error):
|
|
2527
|
+
errno_ = _errno_from_exception(exc)
|
|
2528
|
+
if errno_ in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES:
|
|
2529
|
+
continue # retry on potentially transient errors
|
|
2530
|
+
conn.close()
|
|
2531
|
+
raise exc
|
|
2532
|
+
|
|
2533
|
+
if isinstance(exc, http.client.HTTPException):
|
|
2534
|
+
if conn.sock is None:
|
|
2535
|
+
if i < RETRIES - 1:
|
|
2536
|
+
conn.close()
|
|
2537
|
+
conn.connect()
|
|
2538
|
+
continue
|
|
2539
|
+
else:
|
|
2540
|
+
conn.close()
|
|
2541
|
+
raise exc
|
|
2542
|
+
if i < RETRIES - 1:
|
|
2543
|
+
conn.close()
|
|
2544
|
+
conn.connect()
|
|
2545
|
+
continue
|
|
2546
|
+
conn.close()
|
|
2547
|
+
raise exc
|
|
2548
|
+
|
|
2549
|
+
conn.close()
|
|
2550
|
+
raise exc
|
|
2551
|
+
else:
|
|
2552
|
+
content = b""
|
|
2553
|
+
if method == "HEAD":
|
|
2554
|
+
if hasattr(response, "close"):
|
|
2555
|
+
response.close()
|
|
2556
|
+
if getattr(response, "version", 11) != 20:
|
|
2557
|
+
conn.close()
|
|
2558
|
+
else:
|
|
2559
|
+
content = response.read()
|
|
2560
|
+
response = Response(response)
|
|
2561
|
+
if method != "HEAD":
|
|
2562
|
+
content = _decompressContent(response, content, self.limit_kwargs)
|
|
2563
|
+
|
|
2564
|
+
break
|
|
2565
|
+
return (response, content)
|
|
2566
|
+
|
|
2567
|
+
def _request(
|
|
2568
|
+
self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey,
|
|
2569
|
+
):
|
|
2570
|
+
"""Do the actual request using the connection object
|
|
2571
|
+
and also follow one level of redirects if necessary"""
|
|
2572
|
+
|
|
2573
|
+
auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)]
|
|
2574
|
+
auth_obj = auths and sorted(auths)[0][1] or None
|
|
2575
|
+
if auth_obj:
|
|
2576
|
+
auth_obj.request(method, request_uri, headers, body)
|
|
2577
|
+
|
|
2578
|
+
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
|
|
2579
|
+
|
|
2580
|
+
if auth_obj:
|
|
2581
|
+
if auth_obj.response(response, body):
|
|
2582
|
+
auth_obj.request(method, request_uri, headers, body)
|
|
2583
|
+
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
|
|
2584
|
+
response._stale_digest = 1
|
|
2585
|
+
|
|
2586
|
+
if response.status == 401:
|
|
2587
|
+
for authorization in self._auth_from_challenge(host, request_uri, headers, response, content):
|
|
2588
|
+
authorization.request(method, request_uri, headers, body)
|
|
2589
|
+
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
|
|
2590
|
+
if response.status != 401:
|
|
2591
|
+
self.authorizations.append(authorization)
|
|
2592
|
+
authorization.response(response, body)
|
|
2593
|
+
break
|
|
2594
|
+
|
|
2595
|
+
if self.follow_all_redirects or method in self.safe_methods or response.status in (303, 308):
|
|
2596
|
+
if self.follow_redirects and response.status in self.redirect_codes:
|
|
2597
|
+
if redirections:
|
|
2598
|
+
if "location" not in response and response.status != 300:
|
|
2599
|
+
raise RedirectMissingLocation(
|
|
2600
|
+
_("Redirected but the response is missing a Location: header."), response, content,
|
|
2601
|
+
)
|
|
2602
|
+
if "location" in response:
|
|
2603
|
+
location = response["location"]
|
|
2604
|
+
(scheme, authority, path, query, fragment) = parse_uri(location)
|
|
2605
|
+
if authority == None:
|
|
2606
|
+
response["location"] = urllib.parse.urljoin(absolute_uri, location)
|
|
2607
|
+
if response.status == 308 or (response.status == 301 and (method in self.safe_methods)):
|
|
2608
|
+
response["-x-permanent-redirect-url"] = response["location"]
|
|
2609
|
+
if "content-location" not in response:
|
|
2610
|
+
response["content-location"] = absolute_uri
|
|
2611
|
+
_updateCache(headers, response, content, self.cache, cachekey)
|
|
2612
|
+
if "if-none-match" in headers:
|
|
2613
|
+
del headers["if-none-match"]
|
|
2614
|
+
if "if-modified-since" in headers:
|
|
2615
|
+
del headers["if-modified-since"]
|
|
2616
|
+
if "authorization" in headers and not self.forward_authorization_headers:
|
|
2617
|
+
del headers["authorization"]
|
|
2618
|
+
if "location" in response:
|
|
2619
|
+
location = response["location"]
|
|
2620
|
+
old_response = copy.deepcopy(response)
|
|
2621
|
+
if "content-location" not in old_response:
|
|
2622
|
+
old_response["content-location"] = absolute_uri
|
|
2623
|
+
redirect_method = method
|
|
2624
|
+
if response.status in [302, 303]:
|
|
2625
|
+
redirect_method = "GET"
|
|
2626
|
+
body = None
|
|
2627
|
+
for body_hdr in ("content-length", "content-type", "transfer-encoding"):
|
|
2628
|
+
if body_hdr in headers:
|
|
2629
|
+
del headers[body_hdr]
|
|
2630
|
+
orig_scheme, orig_authority = parse_uri(absolute_uri)[:2]
|
|
2631
|
+
dest_scheme, dest_authority = parse_uri(location)[:2]
|
|
2632
|
+
if (
|
|
2633
|
+
"authorization" in headers
|
|
2634
|
+
and orig_scheme
|
|
2635
|
+
and dest_scheme
|
|
2636
|
+
and orig_scheme.lower() == "https"
|
|
2637
|
+
and dest_scheme.lower() == "http"
|
|
2638
|
+
):
|
|
2639
|
+
del headers["authorization"]
|
|
2640
|
+
if dest_authority and orig_authority and dest_authority.lower() != orig_authority.lower():
|
|
2641
|
+
if "cookie" in headers:
|
|
2642
|
+
del headers["cookie"]
|
|
2643
|
+
if "host" in headers:
|
|
2644
|
+
del headers["host"]
|
|
2645
|
+
(response, content) = self.request(
|
|
2646
|
+
location, method=redirect_method, body=body, headers=headers, redirections=redirections - 1,
|
|
2647
|
+
)
|
|
2648
|
+
response.previous = old_response
|
|
2649
|
+
else:
|
|
2650
|
+
raise RedirectLimit(
|
|
2651
|
+
"Redirected more times than redirection_limit allows.", response, content,
|
|
2652
|
+
)
|
|
2653
|
+
elif response.status in [200, 203] and method in self.safe_methods:
|
|
2654
|
+
if "content-location" not in response:
|
|
2655
|
+
response["content-location"] = absolute_uri
|
|
2656
|
+
_updateCache(headers, response, content, self.cache, cachekey)
|
|
2657
|
+
|
|
2658
|
+
return (response, content)
|
|
2659
|
+
|
|
2660
|
+
def _normalize_headers(self, headers):
|
|
2661
|
+
return _normalize_headers(headers)
|
|
2662
|
+
|
|
2663
|
+
def request(
|
|
2664
|
+
self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None,
|
|
2665
|
+
):
|
|
2666
|
+
"""Performs a single HTTP request."""
|
|
2667
|
+
conn_key = ""
|
|
2668
|
+
|
|
2669
|
+
try:
|
|
2670
|
+
if headers is None:
|
|
2671
|
+
headers = {}
|
|
2672
|
+
else:
|
|
2673
|
+
headers = self._normalize_headers(headers)
|
|
2674
|
+
|
|
2675
|
+
if "user-agent" not in headers:
|
|
2676
|
+
headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
|
|
2677
|
+
|
|
2678
|
+
uri = iri2uri(uri)
|
|
2679
|
+
# Prevent CWE-75 space injection to manipulate request via part of uri.
|
|
2680
|
+
# Prevent CWE-93 CRLF injection to modify headers via part of uri.
|
|
2681
|
+
uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A")
|
|
2682
|
+
|
|
2683
|
+
(scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
|
|
2684
|
+
|
|
2685
|
+
conn_key = scheme + ":" + authority
|
|
2686
|
+
with _get_http_lock(self):
|
|
2687
|
+
conn = self.connections.get(conn_key)
|
|
2688
|
+
is_downgraded = self._is_host_downgraded(authority)
|
|
2689
|
+
should_h2 = (
|
|
2690
|
+
bool(self.enable_http2)
|
|
2691
|
+
and bool(ENABLE_HTTP2)
|
|
2692
|
+
and not is_downgraded
|
|
2693
|
+
)
|
|
2694
|
+
if conn is not None and type(conn) is HTTPSConnectionWithTimeout:
|
|
2695
|
+
expected_ca_certs = _resolve_ca_certs(self.ca_certs)
|
|
2696
|
+
if (
|
|
2697
|
+
bool(conn.disable_ssl_certificate_validation) != bool(self.disable_ssl_certificate_validation)
|
|
2698
|
+
or conn.ca_certs != expected_ca_certs
|
|
2699
|
+
or conn.tls_minimum_version != self.tls_minimum_version
|
|
2700
|
+
or conn.tls_maximum_version != self.tls_maximum_version
|
|
2701
|
+
):
|
|
2702
|
+
conn.close()
|
|
2703
|
+
conn = None
|
|
2704
|
+
del self.connections[conn_key]
|
|
2705
|
+
if conn is not None:
|
|
2706
|
+
if getattr(conn, "_is_https", False) and getattr(conn, "enable_http2", False) and not should_h2:
|
|
2707
|
+
conn.downgrade_to_http1()
|
|
2708
|
+
if conn is None:
|
|
2709
|
+
if not connection_type:
|
|
2710
|
+
connection_type = SCHEME_TO_CONNECTION[scheme]
|
|
2711
|
+
certs = list(self.certificates.iter(authority))
|
|
2712
|
+
if issubclass(connection_type, HTTPSConnectionWithTimeout):
|
|
2713
|
+
extra_https_kwargs = {}
|
|
2714
|
+
if connection_type is HTTPSConnectionWithTimeout:
|
|
2715
|
+
extra_https_kwargs["enable_http2"] = should_h2
|
|
2716
|
+
extra_https_kwargs["pool_maxsize"] = self.pool_maxsize
|
|
2717
|
+
if certs:
|
|
2718
|
+
conn = self.connections[conn_key] = connection_type(
|
|
2719
|
+
authority,
|
|
2720
|
+
key_file=certs[0][0],
|
|
2721
|
+
cert_file=certs[0][1],
|
|
2722
|
+
timeout=self.timeout,
|
|
2723
|
+
proxy_info=self.proxy_info,
|
|
2724
|
+
ca_certs=self.ca_certs,
|
|
2725
|
+
disable_ssl_certificate_validation=self.disable_ssl_certificate_validation,
|
|
2726
|
+
tls_maximum_version=self.tls_maximum_version,
|
|
2727
|
+
tls_minimum_version=self.tls_minimum_version,
|
|
2728
|
+
key_password=certs[0][2],
|
|
2729
|
+
**extra_https_kwargs,
|
|
2730
|
+
)
|
|
2731
|
+
else:
|
|
2732
|
+
conn = self.connections[conn_key] = connection_type(
|
|
2733
|
+
authority,
|
|
2734
|
+
timeout=self.timeout,
|
|
2735
|
+
proxy_info=self.proxy_info,
|
|
2736
|
+
ca_certs=self.ca_certs,
|
|
2737
|
+
disable_ssl_certificate_validation=self.disable_ssl_certificate_validation,
|
|
2738
|
+
tls_maximum_version=self.tls_maximum_version,
|
|
2739
|
+
tls_minimum_version=self.tls_minimum_version,
|
|
2740
|
+
**extra_https_kwargs,
|
|
2741
|
+
)
|
|
2742
|
+
else:
|
|
2743
|
+
extra_http_kwargs = {}
|
|
2744
|
+
if connection_type is HTTPConnectionWithTimeout:
|
|
2745
|
+
extra_http_kwargs["pool_maxsize"] = self.pool_maxsize
|
|
2746
|
+
conn = self.connections[conn_key] = connection_type(
|
|
2747
|
+
authority,
|
|
2748
|
+
timeout=self.timeout,
|
|
2749
|
+
proxy_info=self.proxy_info,
|
|
2750
|
+
**extra_http_kwargs,
|
|
2751
|
+
)
|
|
2752
|
+
conn.set_debuglevel(debuglevel)
|
|
2753
|
+
|
|
2754
|
+
if "range" not in headers and "accept-encoding" not in headers:
|
|
2755
|
+
headers["accept-encoding"] = "gzip, deflate"
|
|
2756
|
+
|
|
2757
|
+
info = email.message.Message()
|
|
2758
|
+
cachekey = None
|
|
2759
|
+
cached_value = None
|
|
2760
|
+
if self.cache:
|
|
2761
|
+
cachekey = defrag_uri
|
|
2762
|
+
cached_value = self.cache.get(cachekey)
|
|
2763
|
+
if cached_value:
|
|
2764
|
+
try:
|
|
2765
|
+
info, content = cached_value.split(b"\r\n\r\n", 1)
|
|
2766
|
+
info = email.message_from_bytes(info)
|
|
2767
|
+
for k, v in info.items():
|
|
2768
|
+
if v.startswith("=?") and v.endswith("?="):
|
|
2769
|
+
info.replace_header(k, str(*email.header.decode_header(v)[0]))
|
|
2770
|
+
except (IndexError, ValueError):
|
|
2771
|
+
self.cache.delete(cachekey)
|
|
2772
|
+
cachekey = None
|
|
2773
|
+
cached_value = None
|
|
2774
|
+
|
|
2775
|
+
if (
|
|
2776
|
+
method in self.optimistic_concurrency_methods
|
|
2777
|
+
and self.cache
|
|
2778
|
+
and "etag" in info
|
|
2779
|
+
and not self.ignore_etag
|
|
2780
|
+
and "if-match" not in headers
|
|
2781
|
+
):
|
|
2782
|
+
headers["if-match"] = info["etag"]
|
|
2783
|
+
|
|
2784
|
+
if self.cache and cachekey and method not in self.safe_methods:
|
|
2785
|
+
self.cache.delete(cachekey)
|
|
2786
|
+
|
|
2787
|
+
if method in self.safe_methods and "vary" in info:
|
|
2788
|
+
vary = info["vary"]
|
|
2789
|
+
vary_headers = vary.lower().replace(" ", "").split(",")
|
|
2790
|
+
for header in vary_headers:
|
|
2791
|
+
key = "-varied-%s" % header
|
|
2792
|
+
value = info[key]
|
|
2793
|
+
if headers.get(header, None) != value:
|
|
2794
|
+
cached_value = None
|
|
2795
|
+
break
|
|
2796
|
+
|
|
2797
|
+
if (
|
|
2798
|
+
self.cache
|
|
2799
|
+
and cached_value
|
|
2800
|
+
and (method in self.safe_methods or info["status"] == "308")
|
|
2801
|
+
and "range" not in headers
|
|
2802
|
+
):
|
|
2803
|
+
redirect_method = method
|
|
2804
|
+
if info["status"] not in ("307", "308"):
|
|
2805
|
+
redirect_method = "GET"
|
|
2806
|
+
if "-x-permanent-redirect-url" in info:
|
|
2807
|
+
if redirections <= 0:
|
|
2808
|
+
raise RedirectLimit(
|
|
2809
|
+
"Redirected more times than redirection_limit allows.", {}, "",
|
|
2810
|
+
)
|
|
2811
|
+
(response, new_content) = self.request(
|
|
2812
|
+
info["-x-permanent-redirect-url"],
|
|
2813
|
+
method=redirect_method,
|
|
2814
|
+
headers=headers,
|
|
2815
|
+
redirections=redirections - 1,
|
|
2816
|
+
)
|
|
2817
|
+
response.previous = Response(info)
|
|
2818
|
+
response.previous.fromcache = True
|
|
2819
|
+
else:
|
|
2820
|
+
entry_disposition = _entry_disposition(info, headers)
|
|
2821
|
+
|
|
2822
|
+
if entry_disposition == "FRESH":
|
|
2823
|
+
response = Response(info)
|
|
2824
|
+
response.fromcache = True
|
|
2825
|
+
return (response, content)
|
|
2826
|
+
|
|
2827
|
+
if entry_disposition == "STALE":
|
|
2828
|
+
if "etag" in info and not self.ignore_etag and not "if-none-match" in headers:
|
|
2829
|
+
headers["if-none-match"] = info["etag"]
|
|
2830
|
+
if "last-modified" in info and not "last-modified" in headers:
|
|
2831
|
+
headers["if-modified-since"] = info["last-modified"]
|
|
2832
|
+
elif entry_disposition == "TRANSPARENT":
|
|
2833
|
+
pass
|
|
2834
|
+
|
|
2835
|
+
(response, new_content) = self._request(
|
|
2836
|
+
conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
|
|
2837
|
+
)
|
|
2838
|
+
|
|
2839
|
+
if response.status == 304 and method == "GET":
|
|
2840
|
+
for key in _get_end2end_headers(response):
|
|
2841
|
+
info[key] = response[key]
|
|
2842
|
+
merged_response = Response(info)
|
|
2843
|
+
if hasattr(response, "_stale_digest"):
|
|
2844
|
+
merged_response._stale_digest = response._stale_digest
|
|
2845
|
+
_updateCache(headers, merged_response, content, self.cache, cachekey)
|
|
2846
|
+
response = merged_response
|
|
2847
|
+
response.status = 200
|
|
2848
|
+
response.fromcache = True
|
|
2849
|
+
|
|
2850
|
+
elif response.status == 200:
|
|
2851
|
+
content = new_content
|
|
2852
|
+
else:
|
|
2853
|
+
self.cache.delete(cachekey)
|
|
2854
|
+
content = new_content
|
|
2855
|
+
else:
|
|
2856
|
+
cc = _parse_cache_control(headers)
|
|
2857
|
+
if "only-if-cached" in cc:
|
|
2858
|
+
info["status"] = "504"
|
|
2859
|
+
response = Response(info)
|
|
2860
|
+
content = b""
|
|
2861
|
+
else:
|
|
2862
|
+
(response, content) = self._request(
|
|
2863
|
+
conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
|
|
2864
|
+
)
|
|
2865
|
+
except Exception as e:
|
|
2866
|
+
is_timeout = isinstance(e, socket.timeout)
|
|
2867
|
+
if is_timeout:
|
|
2868
|
+
with _get_http_lock(self):
|
|
2869
|
+
conn = self.connections.pop(conn_key, None)
|
|
2870
|
+
if conn:
|
|
2871
|
+
conn.close()
|
|
2872
|
+
|
|
2873
|
+
if self.force_exception_to_status_code:
|
|
2874
|
+
if isinstance(e, HttpLib2ErrorWithResponse):
|
|
2875
|
+
response = e.response
|
|
2876
|
+
content = e.content
|
|
2877
|
+
response.status = 500
|
|
2878
|
+
response.reason = str(e)
|
|
2879
|
+
elif isinstance(e, socket.timeout):
|
|
2880
|
+
content = b"Request Timeout"
|
|
2881
|
+
response = Response({"content-type": "text/plain", "status": "408", "content-length": len(content),})
|
|
2882
|
+
response.reason = "Request Timeout"
|
|
2883
|
+
else:
|
|
2884
|
+
content = str(e).encode("utf-8")
|
|
2885
|
+
response = Response({"content-type": "text/plain", "status": "400", "content-length": len(content),})
|
|
2886
|
+
response.reason = "Bad Request"
|
|
2887
|
+
else:
|
|
2888
|
+
raise
|
|
2889
|
+
|
|
2890
|
+
return (response, content)
|
|
2891
|
+
|
|
2892
|
+
|
|
2893
|
+
class Response(dict):
|
|
2894
|
+
"""An object more like email.message than httplib.HTTPResponse."""
|
|
2895
|
+
|
|
2896
|
+
fromcache = False
|
|
2897
|
+
version = 11
|
|
2898
|
+
status = 200
|
|
2899
|
+
reason = "Ok"
|
|
2900
|
+
previous = None
|
|
2901
|
+
stream_id = None
|
|
2902
|
+
|
|
2903
|
+
def __init__(self, info):
|
|
2904
|
+
# info is an email.message, an http.client.HTTPResponse,
|
|
2905
|
+
# a _Httplib2RawResponse, or a mapping.
|
|
2906
|
+
if isinstance(info, (http.client.HTTPResponse, _Httplib2RawResponse)):
|
|
2907
|
+
headers_iter = info.headers.items() if hasattr(info, "headers") else info.getheaders()
|
|
2908
|
+
for key, value in headers_iter:
|
|
2909
|
+
key = key.lower()
|
|
2910
|
+
prev = self.get(key)
|
|
2911
|
+
if prev is not None:
|
|
2912
|
+
value = ", ".join((prev, value))
|
|
2913
|
+
self[key] = value
|
|
2914
|
+
self.status = info.status
|
|
2915
|
+
self["status"] = str(self.status)
|
|
2916
|
+
self.reason = info.reason
|
|
2917
|
+
self.version = info.version
|
|
2918
|
+
self.stream_id = getattr(info, "stream_id", None)
|
|
2919
|
+
elif isinstance(info, email.message.Message):
|
|
2920
|
+
for key, value in list(info.items()):
|
|
2921
|
+
self[key.lower()] = value
|
|
2922
|
+
self.status = int(self["status"])
|
|
2923
|
+
else:
|
|
2924
|
+
for key, value in info.items():
|
|
2925
|
+
self[key.lower()] = value
|
|
2926
|
+
self.status = int(self.get("status", self.status))
|
|
2927
|
+
|
|
2928
|
+
def __getattr__(self, name):
|
|
2929
|
+
if name == "dict":
|
|
2930
|
+
return self
|
|
2931
|
+
else:
|
|
2932
|
+
raise AttributeError(name)
|
|
2933
|
+
|
|
2934
|
+
|
|
2935
|
+
def try_value_or_env(to, value, env_key, default=None):
|
|
2936
|
+
candidates = (value, os.environ.get(env_key), os.environ.get(env_key.upper()))
|
|
2937
|
+
for x in candidates:
|
|
2938
|
+
if x is None:
|
|
2939
|
+
continue
|
|
2940
|
+
try:
|
|
2941
|
+
return to(x)
|
|
2942
|
+
except ValueError:
|
|
2943
|
+
pass
|
|
2944
|
+
return default
|