modern-httplib2 1.32.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2944 @@
1
+ """Small, fast, modern HTTP client library for Python 3.11+ powered by httpx."""
2
+
3
+ import base64
4
+ import calendar
5
+ import contextlib
6
+ import copy
7
+ import email
8
+ import email.feedparser
9
+ from email import header
10
+ import email.message
11
+ import email.utils
12
+ import errno
13
+ import functools
14
+ from gettext import gettext as _
15
+ from hashlib import md5 as _md5
16
+ from hashlib import sha1 as _sha
17
+ import hmac
18
+ import http.client
19
+ import os
20
+ import queue
21
+ import random
22
+ import re
23
+ import select
24
+ import socket
25
+ import ssl
26
+ import sys
27
+ import threading
28
+ import time
29
+ import urllib.parse
30
+ import zlib
31
+
32
+ if sys.version_info < (3, 11):
33
+ raise RuntimeError("modern-httplib2 requires Python 3.11 or newer")
34
+
35
+ from .decode import DecoderProtocol, DeflateDecoder, LimitDecoder, ZlibDecoder
36
+ from .patch import is_patched, patch, unpatch
37
+
38
+ __author__ = "Jay Lee"
39
+ __email__ = "jay0lee@gmail.com"
40
+ __copyright__ = "Copyright 2026, Jay Lee"
41
+ __contributors__ = ["Jay Lee"]
42
+ __license__ = "MIT"
43
+ __version__ = "1.32.0"
44
+
45
+ import httpcore
46
+ from httpcore._backends.base import (
47
+ NetworkBackend as _HttpcoreNetworkBackend,
48
+ NetworkStream as _HttpcoreNetworkStream,
49
+ )
50
+ from httpcore._exceptions import (
51
+ ConnectError as _HttpcoreConnectError,
52
+ ConnectTimeout as _HttpcoreConnectTimeout,
53
+ LocalProtocolError as _HttpcoreLocalProtocolError,
54
+ PoolTimeout as _HttpcorePoolTimeout,
55
+ ProtocolError as _HttpcoreProtocolError,
56
+ ProxyError as _HttpcoreProxyError,
57
+ ReadError as _HttpcoreReadError,
58
+ ReadTimeout as _HttpcoreReadTimeout,
59
+ RemoteProtocolError as _HttpcoreRemoteProtocolError,
60
+ TimeoutException as _HttpcoreTimeoutException,
61
+ WriteError as _HttpcoreWriteError,
62
+ WriteTimeout as _HttpcoreWriteTimeout,
63
+ )
64
+ from httpcore._models import (
65
+ Origin as _HttpcoreOrigin,
66
+ Request as _HttpcoreRequest,
67
+ Response as _HttpcoreResponse,
68
+ URL as _HttpcoreURL,
69
+ )
70
+ import httpcore._sync.connection as _httpcore_connection_mod
71
+ from httpcore._sync.connection import HTTPConnection as _HttpcoreHTTPConnection
72
+ from httpcore._sync.connection_pool import ConnectionPool as _HttpcoreConnectionPool
73
+ from httpcore._sync.http2 import HTTP2Connection as _HttpcoreHTTP2Connection
74
+ from httpcore._synchronization import Semaphore as _HttpcoreSemaphore
75
+ from httpcore._utils import is_socket_readable as _is_socket_readable
76
+ import httpx
77
+
78
+ try:
79
+ import socks
80
+ except ImportError:
81
+ socks = None
82
+
83
+ from . import auth, certs
84
+ from .error import *
85
+ from .iri2uri import iri2uri
86
+
87
+
88
+ def has_timeout(timeout):
89
+ return timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT
90
+
91
+
92
+ __all__ = [
93
+ "AUTO_DOWNGRADE_HTTP2",
94
+ "ENABLE_HTTP2",
95
+ "FailedToDecompressContent",
96
+ "Http",
97
+ "HttpLib2Error",
98
+ "ProxyInfo",
99
+ "REDIRECT_CODES",
100
+ "RETRIES",
101
+ "RedirectLimit",
102
+ "RedirectMissingLocation",
103
+ "Response",
104
+ "debug_print",
105
+ "debuglevel",
106
+ "disable_http2",
107
+ "enable_http2",
108
+ "force_http11",
109
+ "is_patched",
110
+ "patch",
111
+ "reset_downgraded_hosts",
112
+ "set_auto_downgrade_http2",
113
+ "set_enable_http2",
114
+ "unpatch",
115
+ "UnimplementedDigestAuthOptionError",
116
+ "UnimplementedHmacDigestAuthOptionError",
117
+ ]
118
+
119
+ # The httplib debug level, set to a non-zero value to get debug output
120
+ debuglevel = 0
121
+
122
+
123
+ def _debug_print(*args, **kwargs):
124
+ """Output debug messages, honoring custom print monkey-patches on httplib2, modern_httplib2, or http.client."""
125
+ httplib2_mod = sys.modules.get("httplib2") or sys.modules.get(__name__)
126
+ printer = getattr(httplib2_mod, "debug_print", None)
127
+ if printer is None or printer is _debug_print:
128
+ printer = getattr(httplib2_mod, "print", None)
129
+ if printer is None:
130
+ printer = getattr(http.client, "print", None)
131
+ if printer is None:
132
+ printer = print
133
+ str_args = tuple(str(a) if not isinstance(a, str) else a for a in args)
134
+ printer(*str_args, **kwargs)
135
+
136
+
137
+ debug_print = _debug_print
138
+
139
+
140
+ def _env_bool(var_name: str, default: bool) -> bool:
141
+ val = os.environ.get(var_name)
142
+ if val is None:
143
+ return default
144
+ val = val.strip().lower()
145
+ return val not in ("0", "false", "no", "off", "")
146
+
147
+
148
+ # Global flags to enable or disable HTTP/2 and auto-downgrade fallback
149
+ ENABLE_HTTP2 = _env_bool(
150
+ "MODERN_HTTPLIB2_ENABLE_HTTP2",
151
+ _env_bool("HTTPLIB2_ENABLE_HTTP2", True),
152
+ )
153
+
154
+ AUTO_DOWNGRADE_HTTP2 = _env_bool(
155
+ "MODERN_HTTPLIB2_AUTO_DOWNGRADE",
156
+ _env_bool("HTTPLIB2_AUTO_DOWNGRADE", True),
157
+ )
158
+
159
+ # Thread-safe set of hosts (authorities) that have been downgraded to HTTP/1.1
160
+ DOWNGRADED_HOSTS = set()
161
+ _DOWNGRADE_LOCK = threading.Lock()
162
+
163
+
164
+ def force_http11():
165
+ """Globally force all modern-httplib2 connections to HTTP/1.1."""
166
+ set_enable_http2(False)
167
+
168
+
169
+ def disable_http2():
170
+ """Globally disable HTTP/2 for all modern-httplib2 connections."""
171
+ set_enable_http2(False)
172
+
173
+
174
+ def enable_http2():
175
+ """Globally enable HTTP/2 for modern-httplib2 connections."""
176
+ set_enable_http2(True)
177
+
178
+
179
+ def set_enable_http2(enabled: bool):
180
+ """Globally enable or disable HTTP/2.
181
+
182
+ When disabled, ALPN will only advertise HTTP/1.1 and connections will use
183
+ standard HTTP/1.1 pooling.
184
+ """
185
+ global ENABLE_HTTP2
186
+ ENABLE_HTTP2 = bool(enabled)
187
+ Http.enable_http2 = bool(enabled)
188
+ HTTPSConnectionWithTimeout.enable_http2 = bool(enabled)
189
+
190
+
191
+ def set_auto_downgrade_http2(enabled: bool):
192
+ """Globally enable or disable automatic fallback to HTTP/1.1 when HTTP/2 fails."""
193
+ global AUTO_DOWNGRADE_HTTP2
194
+ AUTO_DOWNGRADE_HTTP2 = bool(enabled)
195
+ Http.auto_downgrade_http2 = bool(enabled)
196
+
197
+
198
+ def reset_downgraded_hosts():
199
+ """Reset the set of hosts that were automatically downgraded to HTTP/1.1."""
200
+ with _DOWNGRADE_LOCK:
201
+ DOWNGRADED_HOSTS.clear()
202
+
203
+
204
+ # A request will be tried 'RETRIES' times if it fails at the socket/connection level.
205
+ RETRIES = 2
206
+
207
+ # The number of redirections to follow before giving up.
208
+ # Note that only GET redirects are automatically followed.
209
+ # Will also honor 301 requests by saving that info and never
210
+ # requesting that URI again.
211
+ DEFAULT_MAX_REDIRECTS = 5
212
+
213
+ # Default connection pool size per (scheme, authority)
214
+ DEFAULT_POOL_MAXSIZE = 16
215
+
216
+ # Which headers are hop-by-hop headers by default
217
+ HOP_BY_HOP = [
218
+ "connection",
219
+ "keep-alive",
220
+ "proxy-authenticate",
221
+ "proxy-authorization",
222
+ "te",
223
+ "trailers",
224
+ "transfer-encoding",
225
+ "upgrade",
226
+ ]
227
+
228
+ # https://tools.ietf.org/html/rfc7231#section-8.1.3
229
+ SAFE_METHODS = ("GET", "HEAD", "OPTIONS", "TRACE")
230
+
231
+ # To change, assign to `Http().redirect_codes`
232
+ REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308))
233
+
234
+ CA_CERTS = certs.where()
235
+
236
+
237
+ def _resolve_ca_certs(ca_certs):
238
+ if ca_certs:
239
+ return ca_certs
240
+ if "HTTPLIB2_CA_CERTS" in os.environ:
241
+ return certs.where()
242
+ return CA_CERTS
243
+
244
+
245
+ DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLS_CLIENT
246
+
247
+
248
+ def _build_ssl_context(
249
+ disable_ssl_certificate_validation,
250
+ ca_certs,
251
+ cert_file=None,
252
+ key_file=None,
253
+ maximum_version=None,
254
+ minimum_version=None,
255
+ key_password=None,
256
+ enable_http2=False,
257
+ ):
258
+ context = ssl.SSLContext(DEFAULT_TLS_VERSION)
259
+ context.check_hostname = not disable_ssl_certificate_validation
260
+ context.verify_mode = ssl.CERT_NONE if disable_ssl_certificate_validation else ssl.CERT_REQUIRED
261
+
262
+ if maximum_version is not None:
263
+ context.maximum_version = (
264
+ getattr(ssl.TLSVersion, maximum_version)
265
+ if isinstance(maximum_version, str)
266
+ else maximum_version
267
+ )
268
+ if minimum_version is not None:
269
+ context.minimum_version = (
270
+ getattr(ssl.TLSVersion, minimum_version)
271
+ if isinstance(minimum_version, str)
272
+ else minimum_version
273
+ )
274
+
275
+ if not disable_ssl_certificate_validation:
276
+ context.load_verify_locations(ca_certs)
277
+
278
+ if cert_file:
279
+ context.load_cert_chain(cert_file, key_file, key_password)
280
+
281
+ if enable_http2 and ENABLE_HTTP2:
282
+ with contextlib.suppress(NotImplementedError, ssl.SSLError):
283
+ context.set_alpn_protocols(["h2", "http/1.1"])
284
+ else:
285
+ with contextlib.suppress(NotImplementedError, ssl.SSLError):
286
+ context.set_alpn_protocols(["http/1.1"])
287
+
288
+ return context
289
+
290
+
291
+ def _get_end2end_headers(response):
292
+ hopbyhop = set(HOP_BY_HOP)
293
+ hopbyhop.update(x.strip() for x in response.get("connection", "").split(","))
294
+ return [header for header in response if header not in hopbyhop]
295
+
296
+
297
+ def _errno_from_exception(e):
298
+ try:
299
+ return e.errno
300
+ except AttributeError:
301
+ pass
302
+
303
+ # socket.error and common wrap in .args
304
+ try:
305
+ if e.args:
306
+ return _errno_from_exception(e.args[0])
307
+ except AttributeError:
308
+ pass
309
+
310
+ # pysocks.ProxyError wraps in .socket_err
311
+ try:
312
+ if e.socket_err:
313
+ return _errno_from_exception(e.socket_err)
314
+ except AttributeError:
315
+ pass
316
+
317
+ return None
318
+
319
+
320
+ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
321
+
322
+
323
+ def parse_uri(uri):
324
+ """Parses a URI using the regex given in Appendix B of RFC 3986.
325
+
326
+ (scheme, authority, path, query, fragment) = parse_uri(uri)
327
+ """
328
+ groups = URI.match(uri).groups()
329
+ return (groups[1], groups[3], groups[4], groups[6], groups[8])
330
+
331
+
332
+ def urlnorm(uri):
333
+ scheme, authority, path, query, fragment = parse_uri(uri)
334
+ if not scheme or not authority:
335
+ raise RelativeURIError(f"Only absolute URIs are allowed. uri = {uri}")
336
+ authority = authority.lower()
337
+ scheme = scheme.lower()
338
+ if not path:
339
+ path = "/"
340
+ request_uri = f"{path}?{query}" if query else path
341
+ defrag_uri = f"{scheme}://{authority}{request_uri}"
342
+ return scheme, authority, request_uri, defrag_uri
343
+
344
+
345
+ # Cache filename construction
346
+ re_url_scheme = re.compile(r"^\w+://")
347
+ re_unsafe = re.compile(r"[^\w\-_.()=!]+", re.ASCII)
348
+
349
+
350
+ def safename(filename):
351
+ """Return a filename suitable for the cache.
352
+ Strips dangerous and common characters to create a filename we
353
+ can use to store the cache in.
354
+ """
355
+ if isinstance(filename, bytes):
356
+ filename_bytes = filename
357
+ filename = filename.decode("utf-8")
358
+ else:
359
+ filename_bytes = filename.encode("utf-8")
360
+ filemd5 = _md5(filename_bytes).hexdigest()
361
+ filename = re_url_scheme.sub("", filename)
362
+ filename = re_unsafe.sub("", filename)
363
+
364
+ # Limit length of filename to 90 chars (plus 1 comma + 32 md5 = 123 chars)
365
+ return f"{filename[:90]},{filemd5}"
366
+
367
+
368
+ NORMALIZE_SPACE = re.compile(r"(?:\r\n)?[ \t]+")
369
+
370
+
371
+ def _normalize_headers(headers):
372
+ return {
373
+ _convert_byte_str(key).lower(): NORMALIZE_SPACE.sub(_convert_byte_str(value), " ").strip()
374
+ for key, value in headers.items()
375
+ }
376
+
377
+
378
+ def _convert_byte_str(s):
379
+ if not isinstance(s, str):
380
+ return str(s, "utf-8")
381
+ return s
382
+
383
+
384
+ def _parse_cache_control(headers):
385
+ if "cache-control" not in headers:
386
+ return {}
387
+ retval = {}
388
+ for part in headers["cache-control"].split(","):
389
+ if "=" in part:
390
+ name, _, val = part.partition("=")
391
+ retval[name.strip().lower()] = val.strip().lower()
392
+ else:
393
+ retval[part.strip().lower()] = 1
394
+ return retval
395
+
396
+
397
+ # Whether to use a strict mode to parse WWW-Authenticate headers
398
+ # Might lead to bad results in case of ill-formed header value,
399
+ # so disabled by default, falling back to relaxed parsing.
400
+ # Set to true to turn on, useful for testing servers.
401
+ USE_WWW_AUTH_STRICT_PARSING = 0
402
+
403
+
404
+ def _entry_disposition(response_headers, request_headers):
405
+ """Determine freshness from the Date, Expires and Cache-Control headers."""
406
+ retval = "STALE"
407
+ cc = _parse_cache_control(request_headers)
408
+ cc_response = _parse_cache_control(response_headers)
409
+
410
+ if "pragma" in request_headers and request_headers["pragma"].lower().find("no-cache") != -1:
411
+ retval = "TRANSPARENT"
412
+ if "cache-control" not in request_headers:
413
+ request_headers["cache-control"] = "no-cache"
414
+ elif "no-cache" in cc:
415
+ retval = "TRANSPARENT"
416
+ elif "no-cache" in cc_response:
417
+ retval = "STALE"
418
+ elif "only-if-cached" in cc:
419
+ retval = "FRESH"
420
+ elif "date" in response_headers:
421
+ date = calendar.timegm(email.utils.parsedate_tz(response_headers["date"]))
422
+ now = time.time()
423
+ current_age = max(0, now - date)
424
+ if "max-age" in cc_response:
425
+ try:
426
+ freshness_lifetime = int(cc_response["max-age"])
427
+ except ValueError:
428
+ freshness_lifetime = 0
429
+ elif "expires" in response_headers:
430
+ expires = email.utils.parsedate_tz(response_headers["expires"])
431
+ if None == expires:
432
+ freshness_lifetime = 0
433
+ else:
434
+ freshness_lifetime = max(0, calendar.timegm(expires) - date)
435
+ else:
436
+ freshness_lifetime = 0
437
+ if "max-age" in cc:
438
+ try:
439
+ freshness_lifetime = int(cc["max-age"])
440
+ except ValueError:
441
+ freshness_lifetime = 0
442
+ if "min-fresh" in cc:
443
+ try:
444
+ min_fresh = int(cc["min-fresh"])
445
+ except ValueError:
446
+ min_fresh = 0
447
+ current_age += min_fresh
448
+ if freshness_lifetime > current_age:
449
+ retval = "FRESH"
450
+ return retval
451
+
452
+
453
+ def _decompressContent(response, new_content, limit_kwargs):
454
+ content = new_content
455
+ encoding_header = "content-encoding"
456
+ encoding = response.get(encoding_header, None)
457
+ limit_wrap = functools.partial(LimitDecoder, **limit_kwargs)
458
+ try:
459
+ if encoding in ["gzip", "deflate", "zlib"]:
460
+ try:
461
+ content = limit_wrap(ZlibDecoder()).consume_bytes(new_content, 0)
462
+ except (IOError, zlib.error):
463
+ content = limit_wrap(DeflateDecoder()).consume_bytes(new_content, 0)
464
+ response["content-length"] = str(len(content))
465
+ # Record the historical presence of the encoding in a way the won't interfere.
466
+ response["-content-encoding"] = response.pop(encoding_header)
467
+ except (IOError, zlib.error):
468
+ content = ""
469
+ raise FailedToDecompressContent(
470
+ _("Content purported to be compressed with %s but failed to decompress.")
471
+ % encoding,
472
+ response,
473
+ content,
474
+ )
475
+ return content
476
+
477
+
478
+ def _bind_write_headers(msg):
479
+ def _write_headers(self):
480
+ # Self refers to the Generator object.
481
+ for h, v in msg.items():
482
+ print("%s:" % h, end=" ", file=self._fp)
483
+ if isinstance(v, header.Header):
484
+ print(v.encode(maxlinelen=self._maxheaderlen), file=self._fp)
485
+ else:
486
+ # email.Header got lots of smarts, so use it.
487
+ headers = header.Header(v, maxlinelen=self._maxheaderlen, charset="utf-8", header_name=h)
488
+ print(headers.encode(), file=self._fp)
489
+ # A blank line always separates headers from body.
490
+ print(file=self._fp)
491
+
492
+ return _write_headers
493
+
494
+
495
+ def _updateCache(request_headers, response_headers, content, cache, cachekey):
496
+ if cachekey:
497
+ cc = _parse_cache_control(request_headers)
498
+ cc_response = _parse_cache_control(response_headers)
499
+ if "no-store" in cc or "no-store" in cc_response:
500
+ cache.delete(cachekey)
501
+ else:
502
+ info = email.message.Message()
503
+ for key, value in response_headers.items():
504
+ if key not in ["status", "content-encoding", "transfer-encoding"]:
505
+ info[key] = value
506
+
507
+ # Add annotations to the cache to indicate what headers
508
+ # are variant for this request.
509
+ vary = response_headers.get("vary", None)
510
+ if vary:
511
+ vary_headers = vary.lower().replace(" ", "").split(",")
512
+ for header in vary_headers:
513
+ key = "-varied-%s" % header
514
+ try:
515
+ info[key] = request_headers[header]
516
+ except KeyError:
517
+ pass
518
+
519
+ status = response_headers.status
520
+ if status == 304:
521
+ status = 200
522
+
523
+ status_header = "status: %d\r\n" % status
524
+
525
+ try:
526
+ header_str = info.as_string()
527
+ except UnicodeEncodeError:
528
+ setattr(info, "_write_headers", _bind_write_headers(info))
529
+ header_str = info.as_string()
530
+
531
+ header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str)
532
+ text = b"".join([status_header.encode("utf-8"), header_str.encode("utf-8"), content])
533
+
534
+ cache.set(cachekey, text)
535
+
536
+
537
+ def _cnonce():
538
+ dig = _md5(
539
+ ("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).encode("utf-8")
540
+ ).hexdigest()
541
+ return dig[:16]
542
+
543
+
544
+ def _wsse_username_token(cnonce, iso_now, password):
545
+ return (
546
+ base64.b64encode(_sha(f"{cnonce}{iso_now}{password}".encode("utf-8")).digest()).strip().decode("utf-8")
547
+ )
548
+
549
+
550
+ class Authentication:
551
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
552
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
553
+ self.path = path
554
+ self.host = host
555
+ self.credentials = credentials
556
+ self.http = http
557
+
558
+ def depth(self, request_uri):
559
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
560
+ return request_uri[len(self.path) :].count("/")
561
+
562
+ def inscope(self, host, request_uri):
563
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
564
+ return (host == self.host) and path.startswith(self.path)
565
+
566
+ def request(self, method, request_uri, headers, content):
567
+ """Modify the request headers to add the appropriate
568
+ Authorization header. Override this in sub-classes."""
569
+ pass
570
+
571
+ def response(self, response, content):
572
+ """Gives us a chance to update with new nonces
573
+ or such returned from the last authorized response.
574
+ Override this in sub-classes if necessary.
575
+
576
+ Return TRUE if the request is to be retried, for
577
+ example Digest may return stale=true.
578
+ """
579
+ return False
580
+
581
+ def __eq__(self, auth):
582
+ return False
583
+
584
+ def __ne__(self, auth):
585
+ return True
586
+
587
+ def __lt__(self, auth):
588
+ return True
589
+
590
+ def __gt__(self, auth):
591
+ return False
592
+
593
+ def __le__(self, auth):
594
+ return True
595
+
596
+ def __ge__(self, auth):
597
+ return False
598
+
599
+ def __bool__(self):
600
+ return True
601
+
602
+
603
+ class BasicAuthentication(Authentication):
604
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
605
+ super().__init__(credentials, host, request_uri, headers, response, content, http)
606
+
607
+ def request(self, method, request_uri, headers, content):
608
+ """Modify the request headers to add the appropriate
609
+ Authorization header."""
610
+ user, password = self.credentials
611
+ encoded = base64.b64encode(f"{user}:{password}".encode("utf-8")).strip().decode("utf-8")
612
+ headers["authorization"] = f"Basic {encoded}"
613
+
614
+
615
+ class DigestAuthentication(Authentication):
616
+ """Only do qop='auth' and MD5, since that
617
+ is all Apache currently implements"""
618
+
619
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
620
+ super().__init__(credentials, host, request_uri, headers, response, content, http)
621
+ self.challenge = auth._parse_www_authenticate(response, "www-authenticate")["digest"]
622
+ qop = self.challenge.get("qop", "auth")
623
+ self.challenge["qop"] = "auth" if "auth" in [x.strip() for x in qop.split()] else None
624
+ if self.challenge["qop"] is None:
625
+ raise UnimplementedDigestAuthOptionError(_(f"Unsupported value for qop: {qop}."))
626
+ self.challenge["algorithm"] = self.challenge.get("algorithm", "MD5").upper()
627
+ if self.challenge["algorithm"] != "MD5":
628
+ raise UnimplementedDigestAuthOptionError(
629
+ _(f"Unsupported value for algorithm: {self.challenge['algorithm']}.")
630
+ )
631
+ self.A1 = "".join([self.credentials[0], ":", self.challenge["realm"], ":", self.credentials[1]])
632
+ self.challenge["nc"] = 1
633
+
634
+ def request(self, method, request_uri, headers, content, cnonce=None):
635
+ """Modify the request headers"""
636
+ H = lambda x: _md5(x.encode("utf-8")).hexdigest()
637
+ KD = lambda s, d: H(f"{s}:{d}")
638
+ A2 = f"{method}:{request_uri}"
639
+ self.challenge["cnonce"] = cnonce or _cnonce()
640
+ request_digest = '"%s"' % KD(
641
+ H(self.A1),
642
+ f"{self.challenge['nonce']}:{self.challenge['nc']:08x}:{self.challenge['cnonce']}:{self.challenge['qop']}:{H(A2)}",
643
+ )
644
+ headers["authorization"] = (
645
+ f'Digest username="{self.credentials[0]}", realm="{self.challenge["realm"]}", '
646
+ f'nonce="{self.challenge["nonce"]}", uri="{request_uri}", '
647
+ f'algorithm={self.challenge["algorithm"]}, response={request_digest}, '
648
+ f'qop={self.challenge["qop"]}, nc={self.challenge["nc"]:08x}, '
649
+ f'cnonce="{self.challenge["cnonce"]}"'
650
+ )
651
+ if self.challenge.get("opaque"):
652
+ headers["authorization"] += f', opaque="{self.challenge["opaque"]}"'
653
+ self.challenge["nc"] += 1
654
+
655
+ def response(self, response, content):
656
+ if "authentication-info" not in response:
657
+ challenge = auth._parse_www_authenticate(response, "www-authenticate").get("digest", {})
658
+ if "true" == challenge.get("stale"):
659
+ self.challenge["nonce"] = challenge["nonce"]
660
+ self.challenge["nc"] = 1
661
+ return True
662
+ else:
663
+ updated_challenge = auth._parse_authentication_info(response, "authentication-info")
664
+
665
+ if "nextnonce" in updated_challenge:
666
+ self.challenge["nonce"] = updated_challenge["nextnonce"]
667
+ self.challenge["nc"] = 1
668
+ return False
669
+
670
+
671
+ class HmacDigestAuthentication(Authentication):
672
+ """HMAC Digest authentication implementation."""
673
+
674
+ __author__ = "Jay Lee"
675
+
676
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
677
+ super().__init__(credentials, host, request_uri, headers, response, content, http)
678
+ challenge = auth._parse_www_authenticate(response, "www-authenticate")
679
+ self.challenge = challenge["hmacdigest"]
680
+ self.challenge["reason"] = self.challenge.get("reason", "unauthorized")
681
+ if self.challenge["reason"] not in ["unauthorized", "integrity"]:
682
+ self.challenge["reason"] = "unauthorized"
683
+ self.challenge["salt"] = self.challenge.get("salt", "")
684
+ if not self.challenge.get("snonce"):
685
+ raise UnimplementedHmacDigestAuthOptionError(
686
+ _("The challenge doesn't contain a server nonce, or this one is empty.")
687
+ )
688
+ self.challenge["algorithm"] = self.challenge.get("algorithm", "HMAC-SHA-1")
689
+ if self.challenge["algorithm"] not in ["HMAC-SHA-1", "HMAC-MD5"]:
690
+ raise UnimplementedHmacDigestAuthOptionError(
691
+ _(f"Unsupported value for algorithm: {self.challenge['algorithm']}.")
692
+ )
693
+ self.challenge["pw-algorithm"] = self.challenge.get("pw-algorithm", "SHA-1")
694
+ if self.challenge["pw-algorithm"] not in ["SHA-1", "MD5"]:
695
+ raise UnimplementedHmacDigestAuthOptionError(
696
+ _(f"Unsupported value for pw-algorithm: {self.challenge['pw-algorithm']}.")
697
+ )
698
+ if self.challenge["algorithm"] == "HMAC-MD5":
699
+ self.hashmod = _md5
700
+ else:
701
+ self.hashmod = _sha
702
+ if self.challenge["pw-algorithm"] == "MD5":
703
+ self.pwhashmod = _md5
704
+ else:
705
+ self.pwhashmod = _sha
706
+ self.key = "".join(
707
+ [
708
+ self.credentials[0],
709
+ ":",
710
+ self.pwhashmod.new("".join([self.credentials[1], self.challenge["salt"]])).hexdigest().lower(),
711
+ ":",
712
+ self.challenge["realm"],
713
+ ]
714
+ )
715
+ self.key = self.pwhashmod.new(self.key).hexdigest().lower()
716
+
717
+ def request(self, method, request_uri, headers, content):
718
+ """Modify the request headers"""
719
+ keys = _get_end2end_headers(headers)
720
+ keylist = "".join([f"{k} " for k in keys])
721
+ headers_val = "".join([headers[k] for k in keys])
722
+ created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
723
+ cnonce = _cnonce()
724
+ request_digest = f"{method}:{request_uri}:{cnonce}:{self.challenge['snonce']}:{headers_val}"
725
+ request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
726
+ headers["authorization"] = (
727
+ f'HMACDigest username="{self.credentials[0]}", realm="{self.challenge["realm"]}", '
728
+ f'snonce="{self.challenge["snonce"]}", cnonce="{cnonce}", uri="{request_uri}", '
729
+ f'created="{created}", response="{request_digest}", headers="{keylist}"'
730
+ )
731
+
732
+ def response(self, response, content):
733
+ challenge = auth._parse_www_authenticate(response, "www-authenticate").get("hmacdigest", {})
734
+ return challenge.get("reason") in ["integrity", "stale"]
735
+
736
+
737
+ class WsseAuthentication(Authentication):
738
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
739
+ super().__init__(credentials, host, request_uri, headers, response, content, http)
740
+
741
+ def request(self, method, request_uri, headers, content):
742
+ """Modify the request headers to add the appropriate
743
+ Authorization header."""
744
+ headers["authorization"] = 'WSSE profile="UsernameToken"'
745
+ iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
746
+ cnonce = _cnonce()
747
+ password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
748
+ headers["X-WSSE"] = (
749
+ f'UsernameToken Username="{self.credentials[0]}", PasswordDigest="{password_digest}", '
750
+ f'Nonce="{cnonce}", Created="{iso_now}"'
751
+ )
752
+
753
+
754
+ class GoogleLoginAuthentication(Authentication):
755
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
756
+ from urllib.parse import urlencode
757
+
758
+ super().__init__(credentials, host, request_uri, headers, response, content, http)
759
+ challenge = auth._parse_www_authenticate(response, "www-authenticate")
760
+ service = challenge["googlelogin"].get("service", "xapi")
761
+ if service == "xapi" and request_uri.find("calendar") > 0:
762
+ service = "cl"
763
+
764
+ auth_dict = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers["user-agent"])
765
+ resp, content = self.http.request(
766
+ "https://www.google.com/accounts/ClientLogin",
767
+ method="POST",
768
+ body=urlencode(auth_dict),
769
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
770
+ )
771
+ lines = content.split("\n")
772
+ d = dict(tuple(line.split("=", 1)) for line in lines if line)
773
+ if resp.status == 403:
774
+ self.Auth = ""
775
+ else:
776
+ self.Auth = d["Auth"]
777
+
778
+ def request(self, method, request_uri, headers, content):
779
+ """Modify the request headers to add the appropriate
780
+ Authorization header."""
781
+ headers["authorization"] = f"GoogleLogin Auth={self.Auth}"
782
+
783
+
784
+ AUTH_SCHEME_CLASSES = {
785
+ "basic": BasicAuthentication,
786
+ "wsse": WsseAuthentication,
787
+ "digest": DigestAuthentication,
788
+ "hmacdigest": HmacDigestAuthentication,
789
+ "googlelogin": GoogleLoginAuthentication,
790
+ }
791
+
792
+ AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"]
793
+
794
+
795
+ class FileCache:
796
+ """Uses a local directory as a store for cached files.
797
+ Not really safe to use if multiple threads or processes are going to
798
+ be running on the same cache.
799
+ """
800
+
801
+ def __init__(self, cache, safe=safename):
802
+ self.cache = cache
803
+ self.safe = safe
804
+ os.makedirs(self.cache, exist_ok=True)
805
+
806
+ def get(self, key):
807
+ cache_full_path = os.path.join(self.cache, self.safe(key))
808
+ with contextlib.suppress(OSError):
809
+ with open(cache_full_path, "rb") as f:
810
+ return f.read()
811
+ return None
812
+
813
+ def set(self, key, value):
814
+ cache_full_path = os.path.join(self.cache, self.safe(key))
815
+ with open(cache_full_path, "wb") as f:
816
+ f.write(value)
817
+
818
+ def delete(self, key):
819
+ cache_full_path = os.path.join(self.cache, self.safe(key))
820
+ with contextlib.suppress(FileNotFoundError):
821
+ os.remove(cache_full_path)
822
+
823
+
824
+ class Credentials:
825
+ def __init__(self):
826
+ self.credentials = []
827
+
828
+ def add(self, name, password, domain=""):
829
+ self.credentials.append((domain.lower(), name, password))
830
+
831
+ def clear(self):
832
+ self.credentials = []
833
+
834
+ def iter(self, domain):
835
+ for cdomain, name, password in self.credentials:
836
+ if cdomain == "" or domain == cdomain:
837
+ yield (name, password)
838
+
839
+
840
+ class KeyCerts(Credentials):
841
+ """Identical to Credentials except that
842
+ name/password are mapped to key/cert."""
843
+
844
+ def add(self, key, cert, domain, password):
845
+ self.credentials.append((domain.lower(), key, cert, password))
846
+
847
+ def iter(self, domain):
848
+ for cdomain, key, cert, password in self.credentials:
849
+ if cdomain == "" or domain == cdomain:
850
+ yield (key, cert, password)
851
+
852
+
853
+ class AllHosts:
854
+ pass
855
+
856
+
857
+ class ProxyInfo:
858
+ """Collect information required to use a proxy."""
859
+
860
+ bypass_hosts = ()
861
+
862
+ def __init__(
863
+ self, proxy_type, proxy_host, proxy_port, proxy_rdns=True, proxy_user=None, proxy_pass=None, proxy_headers=None,
864
+ ):
865
+ if isinstance(proxy_user, bytes):
866
+ proxy_user = proxy_user.decode()
867
+ if isinstance(proxy_pass, bytes):
868
+ proxy_pass = proxy_pass.decode()
869
+ (
870
+ self.proxy_type,
871
+ self.proxy_host,
872
+ self.proxy_port,
873
+ self.proxy_rdns,
874
+ self.proxy_user,
875
+ self.proxy_pass,
876
+ self.proxy_headers,
877
+ ) = (
878
+ proxy_type,
879
+ proxy_host,
880
+ proxy_port,
881
+ proxy_rdns,
882
+ proxy_user,
883
+ proxy_pass,
884
+ proxy_headers,
885
+ )
886
+
887
+ def astuple(self):
888
+ return (
889
+ self.proxy_type,
890
+ self.proxy_host,
891
+ self.proxy_port,
892
+ self.proxy_rdns,
893
+ self.proxy_user,
894
+ self.proxy_pass,
895
+ self.proxy_headers,
896
+ )
897
+
898
+ def isgood(self):
899
+ return socks and (self.proxy_host is not None) and (self.proxy_port is not None)
900
+
901
+ def applies_to(self, hostname):
902
+ return not self.bypass_host(hostname)
903
+
904
+ def bypass_host(self, hostname):
905
+ """Has this host been excluded from the proxy config"""
906
+ if self.bypass_hosts is AllHosts:
907
+ return True
908
+
909
+ hostname = "." + hostname.lstrip(".")
910
+ for skip_name in self.bypass_hosts:
911
+ # *.suffix
912
+ if skip_name.startswith(".") and hostname.endswith(skip_name):
913
+ return True
914
+ # exact match
915
+ if hostname == "." + skip_name:
916
+ return True
917
+ return False
918
+
919
+ def __repr__(self):
920
+ return (
921
+ f"<ProxyInfo type={self.proxy_type} "
922
+ f"host:port={self.proxy_host}:{self.proxy_port} rdns={self.proxy_rdns} "
923
+ f"user={self.proxy_user} headers={self.proxy_headers}>"
924
+ )
925
+
926
+
927
+ def proxy_info_from_environment(method="http"):
928
+ """Read proxy info from the environment variables."""
929
+ if method not in ("http", "https"):
930
+ return
931
+
932
+ env_var = method + "_proxy"
933
+ url = os.environ.get(env_var, os.environ.get(env_var.upper()))
934
+ if not url:
935
+ return
936
+ return proxy_info_from_url(url, method, noproxy=None)
937
+
938
+
939
+ def proxy_info_from_url(url, method="http", noproxy=None):
940
+ """Construct a ProxyInfo from a URL (such as http_proxy env var)"""
941
+ url = urllib.parse.urlparse(url)
942
+
943
+ proxy_type = 3 # socks.PROXY_TYPE_HTTP
944
+ if url.scheme == "socks4":
945
+ proxy_type = 1 # socks.PROXY_TYPE_SOCKS4
946
+ elif url.scheme == "socks5" or url.scheme == "socks":
947
+ proxy_type = 2 # socks.PROXY_TYPE_SOCKS5
948
+ pi = ProxyInfo(
949
+ proxy_type=proxy_type,
950
+ proxy_host=url.hostname,
951
+ proxy_port=url.port or dict(https=443, http=80)[method],
952
+ proxy_user=url.username or None,
953
+ proxy_pass=url.password or None,
954
+ proxy_headers=None,
955
+ )
956
+
957
+ bypass_hosts = []
958
+ # If not given an explicit noproxy value, respect values in env vars.
959
+ if noproxy is None:
960
+ noproxy = os.environ.get("no_proxy", os.environ.get("NO_PROXY", ""))
961
+ # Special case: A single '*' character means all hosts should be bypassed.
962
+ if noproxy == "*":
963
+ bypass_hosts = AllHosts
964
+ elif noproxy.strip():
965
+ bypass_hosts = noproxy.split(",")
966
+ bypass_hosts = tuple(filter(bool, bypass_hosts)) # To exclude empty string.
967
+
968
+ pi.bypass_hosts = bypass_hosts
969
+ return pi
970
+
971
+
972
+ import h2.connection as _h2_connection
973
+ import h2.events as _h2_events
974
+ import h2.exceptions as _h2_exceptions
975
+ from httpcore._sync.http11 import HTTP11Connection as _HttpcoreHTTP11Connection
976
+ from httpcore._sync.http2 import (
977
+ HTTP2ConnectionByteStream as _HttpcoreHTTP2ConnectionByteStream,
978
+ HTTPConnectionState as _HttpcoreHTTPConnectionState,
979
+ has_body_headers as _httpcore_has_body_headers,
980
+ )
981
+
982
+
983
+ def _unwrap_httpcore_exception(exc):
984
+ """Translate httpcore/httpx exceptions into the stdlib/socket/http.client exceptions
985
+ expected by httplib2 callers and _conn_request retry logic.
986
+ """
987
+ if isinstance(
988
+ exc,
989
+ (
990
+ socket.timeout,
991
+ socket.gaierror,
992
+ ssl.SSLError,
993
+ ssl.CertificateError,
994
+ http.client.HTTPException,
995
+ HttpLib2Error,
996
+ ),
997
+ ):
998
+ return exc
999
+
1000
+ cause = getattr(exc, "__cause__", None) or getattr(exc, "__context__", None)
1001
+ if isinstance(
1002
+ cause,
1003
+ (
1004
+ socket.timeout,
1005
+ socket.gaierror,
1006
+ ssl.SSLError,
1007
+ ssl.CertificateError,
1008
+ http.client.HTTPException,
1009
+ HttpLib2Error,
1010
+ OSError,
1011
+ ),
1012
+ ):
1013
+ return cause
1014
+
1015
+ if isinstance(exc, _HttpcoreTimeoutException):
1016
+ return socket.timeout(str(exc))
1017
+
1018
+ if isinstance(exc, _HttpcoreRemoteProtocolError):
1019
+ msg = str(exc)
1020
+ if "Server disconnected" in msg or "ConnectionClosed" in msg:
1021
+ return http.client.RemoteDisconnected(msg)
1022
+ return http.client.BadStatusLine(msg)
1023
+
1024
+ if isinstance(exc, (_HttpcoreLocalProtocolError, _HttpcoreProtocolError)):
1025
+ return http.client.HTTPException(str(exc))
1026
+
1027
+ if isinstance(exc, (_HttpcoreConnectError, _HttpcoreReadError, _HttpcoreWriteError, _HttpcoreProxyError)):
1028
+ return OSError(str(exc))
1029
+
1030
+ return exc
1031
+
1032
+
1033
+ def _is_cert_verification_error(exc):
1034
+ """Check if exception represents an SSL certificate verification or TLS policy failure."""
1035
+ if isinstance(exc, (ssl.CertificateError, getattr(ssl, "SSLCertVerificationError", ()))):
1036
+ return True
1037
+ msg = str(exc).lower()
1038
+ non_downgrade_ssl_terms = (
1039
+ "certificate verify failed",
1040
+ "cert_has_expired",
1041
+ "certificate_unknown",
1042
+ "unknown_ca",
1043
+ "self_signed_cert",
1044
+ "self signed certificate",
1045
+ "hostname",
1046
+ "ip address mismatch",
1047
+ "wrong_version_number",
1048
+ "unsupported_protocol",
1049
+ "protocol_version",
1050
+ "no_ciphers_available",
1051
+ "no shared cipher",
1052
+ "handshake_failure",
1053
+ "insufficient_security",
1054
+ )
1055
+ return any(term in msg for term in non_downgrade_ssl_terms)
1056
+
1057
+
1058
+ def _should_downgrade_on_error(exc):
1059
+ """Determine whether an error on an HTTP/2 connection warrants automatic fallback to HTTP/1.1."""
1060
+ if _is_cert_verification_error(exc):
1061
+ return False
1062
+ if isinstance(exc, (socket.timeout, socket.gaierror, ConnectionRefusedError)):
1063
+ return False
1064
+ if isinstance(
1065
+ exc,
1066
+ (
1067
+ http.client.BadStatusLine,
1068
+ http.client.RemoteDisconnected,
1069
+ http.client.HTTPException,
1070
+ ConnectionResetError,
1071
+ ConnectionAbortedError,
1072
+ BrokenPipeError,
1073
+ ssl.SSLError,
1074
+ ),
1075
+ ):
1076
+ return True
1077
+ if "h2" in type(exc).__module__:
1078
+ return True
1079
+ if isinstance(exc, OSError):
1080
+ errno_ = _errno_from_exception(exc)
1081
+ if errno_ in (
1082
+ errno.ECONNREFUSED,
1083
+ errno.ENETUNREACH,
1084
+ errno.EHOSTUNREACH,
1085
+ errno.EADDRNOTAVAIL,
1086
+ errno.EACCES,
1087
+ errno.EPERM,
1088
+ ):
1089
+ return False
1090
+ if errno_ in (errno.ECONNRESET, errno.EPIPE, errno.ECONNABORTED, errno.EPROTO):
1091
+ return True
1092
+ return True
1093
+ msg = str(exc).lower()
1094
+ h2_terms = ("http/2", "h2", "protocol", "stream", "frame", "goaway", "rst_stream", "flow control")
1095
+ return any(term in msg for term in h2_terms)
1096
+
1097
+
1098
+ _HALF_CLOSED_SOCKS = []
1099
+
1100
+
1101
+ def _wait_socket(sock: socket.socket, for_write: bool, timeout: float | None) -> bool:
1102
+ try:
1103
+ if for_write:
1104
+ _, writable, _ = select.select([], [sock], [], timeout)
1105
+ return bool(writable)
1106
+ readable, _, _ = select.select([sock], [], [], timeout)
1107
+ return bool(readable)
1108
+ except ValueError:
1109
+ if sock.fileno() < 0:
1110
+ raise OSError(errno.EBADF, "Bad file descriptor")
1111
+ poller = select.poll()
1112
+ poller.register(sock, select.POLLOUT if for_write else select.POLLIN)
1113
+ timeout_ms = None if timeout is None else max(0, int(timeout * 1000))
1114
+ return bool(poller.poll(timeout_ms))
1115
+
1116
+
1117
+ class _Httplib2SyncStream(_HttpcoreNetworkStream):
1118
+ """Custom httpcore NetworkStream wrapping a raw/SSL/PySocks socket while preserving
1119
+ stdlib exception types, thread-safe concurrent SSLSocket read/write multiplexing,
1120
+ and half-closed socket semantics on read timeouts.
1121
+ """
1122
+
1123
+ def __init__(self, sock, owner=None, enable_http2=False):
1124
+ self._sock = sock
1125
+ self._owner = owner
1126
+ self._enable_http2 = enable_http2
1127
+ self._timed_out = False
1128
+ self._tls_started = isinstance(sock, ssl.SSLSocket)
1129
+ self._active_request = None
1130
+ self._ssl_lock = threading.Lock()
1131
+
1132
+ def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
1133
+ sock = self._sock
1134
+ if sock is None:
1135
+ return b""
1136
+ if self._active_request is not None:
1137
+ self._active_request.extensions["request_sent"] = True
1138
+ eff_timeout = timeout if has_timeout(timeout) else None
1139
+ if isinstance(sock, ssl.SSLSocket):
1140
+ deadline = (time.monotonic() + eff_timeout) if eff_timeout is not None else None
1141
+ while True:
1142
+ with self._ssl_lock:
1143
+ if self._sock is None:
1144
+ return b""
1145
+ has_pending = sock.pending() > 0
1146
+ if not has_pending:
1147
+ if deadline is not None:
1148
+ remaining = deadline - time.monotonic()
1149
+ if remaining <= 0:
1150
+ self._timed_out = True
1151
+ raise TimeoutError("timed out")
1152
+ else:
1153
+ remaining = None
1154
+ if not _wait_socket(sock, for_write=False, timeout=remaining):
1155
+ self._timed_out = True
1156
+ raise TimeoutError("timed out")
1157
+ with self._ssl_lock:
1158
+ if self._sock is None:
1159
+ return b""
1160
+ try:
1161
+ sock.settimeout(0.0)
1162
+ return sock.recv(max_bytes)
1163
+ except (ssl.SSLWantReadError, BlockingIOError):
1164
+ continue
1165
+ except ssl.SSLWantWriteError:
1166
+ pass
1167
+ except TimeoutError:
1168
+ self._timed_out = True
1169
+ raise
1170
+ finally:
1171
+ with contextlib.suppress(OSError):
1172
+ sock.settimeout(eff_timeout)
1173
+ if deadline is not None:
1174
+ remaining = deadline - time.monotonic()
1175
+ if remaining <= 0:
1176
+ self._timed_out = True
1177
+ raise TimeoutError("timed out")
1178
+ else:
1179
+ remaining = None
1180
+ if not _wait_socket(sock, for_write=True, timeout=remaining):
1181
+ self._timed_out = True
1182
+ raise TimeoutError("timed out")
1183
+ try:
1184
+ sock.settimeout(eff_timeout)
1185
+ return sock.recv(max_bytes)
1186
+ except TimeoutError:
1187
+ self._timed_out = True
1188
+ raise
1189
+
1190
+ def write(self, buffer: bytes, timeout: float | None = None) -> None:
1191
+ sock = self._sock
1192
+ if not buffer or sock is None:
1193
+ return
1194
+ eff_timeout = timeout if has_timeout(timeout) else None
1195
+ if isinstance(sock, ssl.SSLSocket):
1196
+ deadline = (time.monotonic() + eff_timeout) if eff_timeout is not None else None
1197
+ view = memoryview(buffer)
1198
+ sent_total = 0
1199
+ total_len = len(view)
1200
+ try:
1201
+ while sent_total < total_len:
1202
+ want_read = False
1203
+ with self._ssl_lock:
1204
+ if self._sock is None:
1205
+ raise OSError(errno.EBADF, "Socket closed")
1206
+ try:
1207
+ sock.settimeout(0.0)
1208
+ sent = sock.send(view[sent_total:])
1209
+ if sent > 0:
1210
+ sent_total += sent
1211
+ continue
1212
+ except (ssl.SSLWantWriteError, BlockingIOError):
1213
+ want_read = False
1214
+ except ssl.SSLWantReadError:
1215
+ want_read = True
1216
+ finally:
1217
+ with contextlib.suppress(OSError):
1218
+ sock.settimeout(eff_timeout)
1219
+ if deadline is not None:
1220
+ remaining = deadline - time.monotonic()
1221
+ if remaining <= 0:
1222
+ self._timed_out = True
1223
+ raise TimeoutError("timed out")
1224
+ else:
1225
+ remaining = None
1226
+ if not _wait_socket(sock, for_write=not want_read, timeout=remaining):
1227
+ self._timed_out = True
1228
+ raise TimeoutError("timed out")
1229
+ if self._active_request is not None:
1230
+ self._active_request.extensions["request_sent"] = True
1231
+ return
1232
+ except TimeoutError:
1233
+ self._timed_out = True
1234
+ raise
1235
+ except OSError as exc:
1236
+ if self._active_request is not None:
1237
+ self._active_request.extensions["request_sent"] = True
1238
+ raise _HttpcoreWriteError(str(exc)) from exc
1239
+ try:
1240
+ sock.settimeout(eff_timeout)
1241
+ sock.sendall(buffer)
1242
+ if self._active_request is not None:
1243
+ self._active_request.extensions["request_sent"] = True
1244
+ except TimeoutError:
1245
+ self._timed_out = True
1246
+ raise
1247
+ except OSError as exc:
1248
+ if self._active_request is not None:
1249
+ self._active_request.extensions["request_sent"] = True
1250
+ raise _HttpcoreWriteError(str(exc)) from exc
1251
+
1252
+ def close(self) -> None:
1253
+ with self._ssl_lock:
1254
+ sock = self._sock
1255
+ self._sock = None
1256
+ if sock is None:
1257
+ return
1258
+ if self._timed_out:
1259
+ self._timed_out = False
1260
+ with contextlib.suppress(OSError):
1261
+ sock.shutdown(socket.SHUT_WR)
1262
+ _HALF_CLOSED_SOCKS.append(sock)
1263
+ while len(_HALF_CLOSED_SOCKS) > 16:
1264
+ old_sock = _HALF_CLOSED_SOCKS.pop(0)
1265
+ with contextlib.suppress(OSError):
1266
+ old_sock.close()
1267
+ else:
1268
+ with contextlib.suppress(OSError):
1269
+ sock.close()
1270
+
1271
+ def start_tls(
1272
+ self,
1273
+ ssl_context: ssl.SSLContext,
1274
+ server_hostname: str | None = None,
1275
+ timeout: float | None = None,
1276
+ ) -> _HttpcoreNetworkStream:
1277
+ if self._tls_started:
1278
+ if (
1279
+ ssl_context.check_hostname
1280
+ and server_hostname is not None
1281
+ and isinstance(self._sock, ssl.SSLSocket)
1282
+ and getattr(self._sock, "server_hostname", None) not in (None, server_hostname)
1283
+ ):
1284
+ orig_host = getattr(self._sock, "server_hostname", None)
1285
+ self.close()
1286
+ raise ssl.CertificateError(
1287
+ f"Hostname mismatch on pre-connected TLS stream: expected {server_hostname!r}, got {orig_host!r}"
1288
+ )
1289
+ return self
1290
+ try:
1291
+ if self._enable_http2:
1292
+ with contextlib.suppress(NotImplementedError):
1293
+ ssl_context.set_alpn_protocols(["h2", "http/1.1"])
1294
+ else:
1295
+ with contextlib.suppress(NotImplementedError):
1296
+ ssl_context.set_alpn_protocols(["http/1.1"])
1297
+ self._sock.settimeout(timeout if has_timeout(timeout) else None)
1298
+ ssl_sock = ssl_context.wrap_socket(self._sock, server_hostname=server_hostname)
1299
+ self._sock = ssl_sock
1300
+ self._tls_started = True
1301
+ if self._owner is not None:
1302
+ self._owner._record_sock(ssl_sock)
1303
+ return self
1304
+ except Exception:
1305
+ self.close()
1306
+ raise
1307
+
1308
+ def get_extra_info(self, info: str):
1309
+ if self._sock is None:
1310
+ return None
1311
+ if info == "ssl_object" and isinstance(self._sock, ssl.SSLSocket):
1312
+ return self._sock._sslobj
1313
+ if info == "client_addr":
1314
+ return self._sock.getsockname()
1315
+ if info == "server_addr":
1316
+ return self._sock.getpeername()
1317
+ if info == "socket":
1318
+ return self._sock
1319
+ if info == "is_readable":
1320
+ return _is_socket_readable(self._sock)
1321
+ return None
1322
+
1323
+
1324
+ class _Httplib2NetworkBackend(_HttpcoreNetworkBackend):
1325
+ """Custom httpcore NetworkBackend that uses httplib2's socket/PySocks connection logic
1326
+ and supports pre-connected streams from explicit conn.connect() calls.
1327
+ """
1328
+
1329
+ def __init__(self, owner):
1330
+ self._owner = owner
1331
+ self._preconnected_streams = queue.Queue()
1332
+
1333
+ def push_preconnected(self, stream: _Httplib2SyncStream) -> None:
1334
+ self._preconnected_streams.put(stream)
1335
+
1336
+ def clear_preconnected(self) -> None:
1337
+ while True:
1338
+ try:
1339
+ stream = self._preconnected_streams.get_nowait()
1340
+ except queue.Empty:
1341
+ break
1342
+ with contextlib.suppress(Exception):
1343
+ stream.close()
1344
+
1345
+ def connect_tcp(
1346
+ self,
1347
+ host: str,
1348
+ port: int,
1349
+ timeout: float | None = None,
1350
+ local_address: str | None = None,
1351
+ socket_options=None,
1352
+ ) -> _HttpcoreNetworkStream:
1353
+ while True:
1354
+ try:
1355
+ stream = self._preconnected_streams.get_nowait()
1356
+ except queue.Empty:
1357
+ break
1358
+ if stream._sock is not None:
1359
+ return stream
1360
+
1361
+ sock = self._owner._connect_raw_socket(
1362
+ include_flowinfo=not self._owner._is_https,
1363
+ timeout=timeout,
1364
+ )
1365
+ stream = _Httplib2SyncStream(
1366
+ sock,
1367
+ owner=self._owner,
1368
+ enable_http2=getattr(self._owner, "enable_http2", False),
1369
+ )
1370
+ self._owner._record_sock(sock)
1371
+ return stream
1372
+
1373
+
1374
+ class _Httplib2HTTP2Connection(_HttpcoreHTTP2Connection):
1375
+ """High-concurrency thread-safe HTTP/2 connection subclass for httpcore.
1376
+
1377
+ Unlocks all local concurrent streams immediately on the initial connection flight,
1378
+ strips HTTP/1.1 connection-specific headers forbidden by RFC 9113, and serializes
1379
+ all h2 state machine mutations across concurrent worker threads.
1380
+ """
1381
+
1382
+ _FORBIDDEN_H2_HEADERS = frozenset(
1383
+ (
1384
+ b"host",
1385
+ b"transfer-encoding",
1386
+ b"connection",
1387
+ b"keep-alive",
1388
+ b"proxy-connection",
1389
+ b"upgrade",
1390
+ )
1391
+ )
1392
+
1393
+ def __init__(self, *args, **kwargs):
1394
+ super().__init__(*args, **kwargs)
1395
+ self._write_lock = threading.RLock()
1396
+
1397
+ def handle_request(self, request: _HttpcoreRequest) -> _HttpcoreResponse:
1398
+ with self._state_lock:
1399
+ if self._state in (_HttpcoreHTTPConnectionState.ACTIVE, _HttpcoreHTTPConnectionState.IDLE):
1400
+ self._request_count += 1
1401
+ self._state = _HttpcoreHTTPConnectionState.ACTIVE
1402
+ else:
1403
+ raise httpcore.ConnectionNotAvailable()
1404
+
1405
+ with self._init_lock:
1406
+ if not self._sent_connection_init:
1407
+ try:
1408
+ self._send_connection_init(request=request)
1409
+ except BaseException:
1410
+ self.close()
1411
+ raise
1412
+ self._sent_connection_init = True
1413
+ local_settings_max_streams = self._h2_state.local_settings.max_concurrent_streams
1414
+ self._max_streams = local_settings_max_streams
1415
+ self._max_streams_semaphore = _HttpcoreSemaphore(local_settings_max_streams)
1416
+
1417
+ self._max_streams_semaphore.acquire()
1418
+ stream_id = None
1419
+ try:
1420
+ with self._write_lock:
1421
+ if (
1422
+ self._state == _HttpcoreHTTPConnectionState.CLOSED
1423
+ or self._connection_error
1424
+ or self._connection_terminated
1425
+ ):
1426
+ self._request_count -= 1
1427
+ self._max_streams_semaphore.release()
1428
+ raise httpcore.ConnectionNotAvailable()
1429
+ try:
1430
+ stream_id = self._h2_state.get_next_available_stream_id()
1431
+ self._events[stream_id] = []
1432
+ except _h2_exceptions.NoAvailableStreamIDError:
1433
+ self._used_all_stream_ids = True
1434
+ self._request_count -= 1
1435
+ self._max_streams_semaphore.release()
1436
+ raise httpcore.ConnectionNotAvailable()
1437
+ self._send_request_headers(request=request, stream_id=stream_id)
1438
+
1439
+ self._send_request_body(request=request, stream_id=stream_id)
1440
+ status, headers = self._receive_response(request=request, stream_id=stream_id)
1441
+ return _HttpcoreResponse(
1442
+ status=status,
1443
+ headers=headers,
1444
+ content=_HttpcoreHTTP2ConnectionByteStream(self, request, stream_id=stream_id),
1445
+ extensions={
1446
+ "http_version": b"HTTP/2",
1447
+ "network_stream": self._network_stream,
1448
+ "stream_id": stream_id,
1449
+ },
1450
+ )
1451
+ except BaseException as exc:
1452
+ if stream_id is not None:
1453
+ self._response_closed(stream_id=stream_id)
1454
+ if isinstance(exc, _h2_exceptions.ProtocolError):
1455
+ if self._connection_terminated:
1456
+ raise _HttpcoreRemoteProtocolError(self._connection_terminated)
1457
+ raise _HttpcoreLocalProtocolError(exc)
1458
+ raise
1459
+
1460
+ def _send_request_headers(self, request: _HttpcoreRequest, stream_id: int) -> None:
1461
+ has_nonempty_body = request.extensions.get("has_nonempty_body")
1462
+ if has_nonempty_body is None:
1463
+ end_stream = not _httpcore_has_body_headers(request)
1464
+ else:
1465
+ end_stream = not has_nonempty_body
1466
+
1467
+ hosts = [v for k, v in request.headers if k.lower() == b"host"]
1468
+ authority = hosts[0] if hosts else request.url.origin.host
1469
+
1470
+ headers = [
1471
+ (b":method", request.method),
1472
+ (b":authority", authority),
1473
+ (b":scheme", request.url.scheme),
1474
+ (b":path", request.url.target),
1475
+ ]
1476
+ for k, v in request.headers:
1477
+ kl = k.lower()
1478
+ if kl.startswith(b":"):
1479
+ raise _HttpcoreLocalProtocolError(f"Illegal HTTP/2 pseudo-header in request headers: {k!r}")
1480
+ if kl in self._FORBIDDEN_H2_HEADERS:
1481
+ continue
1482
+ if kl == b"te" and v.lower() != b"trailers":
1483
+ continue
1484
+ headers.append((kl, v))
1485
+
1486
+ with self._write_lock:
1487
+ self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
1488
+ self._h2_state.increment_flow_control_window(2**24, stream_id=stream_id)
1489
+ self._write_outgoing_data(request)
1490
+ request.extensions["request_sent"] = True
1491
+
1492
+ def _send_request_body(self, request: _HttpcoreRequest, stream_id: int) -> None:
1493
+ has_nonempty_body = request.extensions.get("has_nonempty_body")
1494
+ if has_nonempty_body is False:
1495
+ return
1496
+ if has_nonempty_body is None and not _httpcore_has_body_headers(request):
1497
+ return
1498
+ for data in request.stream:
1499
+ self._send_stream_data(request, stream_id, data)
1500
+ self._send_end_stream(request, stream_id)
1501
+
1502
+ def _send_stream_data(self, request: _HttpcoreRequest, stream_id: int, data: bytes) -> None:
1503
+ while data:
1504
+ max_flow = self._wait_for_outgoing_flow(request, stream_id)
1505
+ chunk_size = min(len(data), max_flow)
1506
+ chunk, data = data[:chunk_size], data[chunk_size:]
1507
+ with self._write_lock:
1508
+ self._h2_state.send_data(stream_id, chunk)
1509
+ self._write_outgoing_data(request)
1510
+
1511
+ def _send_end_stream(self, request: _HttpcoreRequest, stream_id: int) -> None:
1512
+ with self._write_lock:
1513
+ self._h2_state.end_stream(stream_id)
1514
+ self._write_outgoing_data(request)
1515
+
1516
+ def _wait_for_outgoing_flow(self, request: _HttpcoreRequest, stream_id: int) -> int:
1517
+ with self._write_lock:
1518
+ local_flow = self._h2_state.local_flow_control_window(stream_id)
1519
+ max_frame_size = self._h2_state.max_outbound_frame_size
1520
+ flow = min(local_flow, max_frame_size)
1521
+ while flow == 0:
1522
+ self._receive_events(request)
1523
+ with self._write_lock:
1524
+ local_flow = self._h2_state.local_flow_control_window(stream_id)
1525
+ max_frame_size = self._h2_state.max_outbound_frame_size
1526
+ flow = min(local_flow, max_frame_size)
1527
+ return flow
1528
+
1529
+ def _receive_response_body(self, request: _HttpcoreRequest, stream_id: int):
1530
+ while True:
1531
+ event = self._receive_stream_event(request, stream_id)
1532
+ if isinstance(event, _h2_events.DataReceived):
1533
+ amount = event.flow_controlled_length
1534
+ with self._write_lock:
1535
+ self._h2_state.acknowledge_received_data(amount, stream_id)
1536
+ with contextlib.suppress(OSError, _HttpcoreWriteError):
1537
+ self._write_outgoing_data(request)
1538
+ yield event.data
1539
+ elif isinstance(event, _h2_events.StreamEnded):
1540
+ break
1541
+
1542
+ def _read_incoming_data(self, request: _HttpcoreRequest):
1543
+ timeouts = request.extensions.get("timeout", {})
1544
+ timeout = timeouts.get("read", None)
1545
+
1546
+ if self._read_exception is not None:
1547
+ raise self._read_exception
1548
+
1549
+ try:
1550
+ data = self._network_stream.read(self.READ_NUM_BYTES, timeout)
1551
+ if data == b"":
1552
+ raise _HttpcoreRemoteProtocolError("Server disconnected")
1553
+ except Exception as exc:
1554
+ self._read_exception = exc
1555
+ self._connection_error = True
1556
+ raise exc
1557
+
1558
+ with self._write_lock:
1559
+ events = self._h2_state.receive_data(data)
1560
+ return events
1561
+
1562
+ def close(self) -> None:
1563
+ with self._write_lock:
1564
+ with contextlib.suppress(Exception):
1565
+ self._h2_state.close_connection()
1566
+ self._state = _HttpcoreHTTPConnectionState.CLOSED
1567
+ self._network_stream.close()
1568
+
1569
+
1570
+ class _Httplib2HttpcoreConnection(_HttpcoreHTTPConnection):
1571
+ """Subclass of httpcore.HTTPConnection that instantiates _Httplib2HTTP2Connection
1572
+ when HTTP/2 is negotiated via ALPN.
1573
+ """
1574
+
1575
+ def handle_request(self, request: _HttpcoreRequest) -> _HttpcoreResponse:
1576
+ if not self.can_handle_request(request.url.origin):
1577
+ raise RuntimeError(f"Cannot handle request URL '{request.url.origin}' on '{self._origin}'")
1578
+
1579
+ try:
1580
+ with self._request_lock:
1581
+ if self._connection is None:
1582
+ stream = self._connect(request)
1583
+ ssl_object = stream.get_extra_info("ssl_object")
1584
+ http2_negotiated = (
1585
+ ssl_object is not None
1586
+ and ssl_object.selected_alpn_protocol() == "h2"
1587
+ )
1588
+ if http2_negotiated or (self._http2 and not self._http1):
1589
+ self._connection = _Httplib2HTTP2Connection(
1590
+ origin=self._origin,
1591
+ stream=stream,
1592
+ keepalive_expiry=self._keepalive_expiry,
1593
+ )
1594
+ else:
1595
+ self._connection = _HttpcoreHTTP11Connection(
1596
+ origin=self._origin,
1597
+ stream=stream,
1598
+ keepalive_expiry=self._keepalive_expiry,
1599
+ )
1600
+ except BaseException as exc:
1601
+ self._connect_failed = True
1602
+ raise exc
1603
+
1604
+ if isinstance(self._connection, _HttpcoreHTTP11Connection):
1605
+ stream = getattr(self._connection, "_network_stream", None)
1606
+ if isinstance(stream, _Httplib2SyncStream):
1607
+ stream._active_request = request
1608
+ return self._connection.handle_request(request)
1609
+
1610
+
1611
+ class _Httplib2ConnectionPool(_HttpcoreConnectionPool):
1612
+ """Subclass of httpcore.ConnectionPool that creates _Httplib2HttpcoreConnection instances."""
1613
+
1614
+ def create_connection(self, origin: _HttpcoreOrigin):
1615
+ return _Httplib2HttpcoreConnection(
1616
+ origin=origin,
1617
+ ssl_context=self._ssl_context,
1618
+ keepalive_expiry=self._keepalive_expiry,
1619
+ http1=self._http1,
1620
+ http2=self._http2,
1621
+ retries=self._retries,
1622
+ local_address=self._local_address,
1623
+ uds=self._uds,
1624
+ network_backend=self._network_backend,
1625
+ socket_options=self._socket_options,
1626
+ )
1627
+
1628
+ def _assign_requests_to_connections(self):
1629
+ closing_connections = []
1630
+ queued_requests = [request for request in self._requests if request.is_queued()]
1631
+
1632
+ for connection in list(self._connections):
1633
+ if connection.is_closed():
1634
+ self._connections.remove(connection)
1635
+ elif queued_requests and connection.has_expired():
1636
+ self._connections.remove(connection)
1637
+ closing_connections.append(connection)
1638
+ elif (
1639
+ connection.is_idle()
1640
+ and sum(1 for c in self._connections if c.is_idle()) > self._max_keepalive_connections
1641
+ ):
1642
+ self._connections.remove(connection)
1643
+ closing_connections.append(connection)
1644
+
1645
+ for pool_request in queued_requests:
1646
+ origin = pool_request.request.url.origin
1647
+ available_connections = [
1648
+ connection
1649
+ for connection in self._connections
1650
+ if connection.can_handle_request(origin) and connection.is_available()
1651
+ ]
1652
+ idle_connections = [
1653
+ connection for connection in self._connections if connection.is_idle()
1654
+ ]
1655
+
1656
+ if available_connections:
1657
+ connection = available_connections[0]
1658
+ pool_request.assign_to_connection(connection)
1659
+ elif len(self._connections) < self._max_connections:
1660
+ connection = self.create_connection(origin)
1661
+ self._connections.append(connection)
1662
+ pool_request.assign_to_connection(connection)
1663
+ elif idle_connections:
1664
+ connection = idle_connections[0]
1665
+ self._connections.remove(connection)
1666
+ closing_connections.append(connection)
1667
+ connection = self.create_connection(origin)
1668
+ self._connections.append(connection)
1669
+ pool_request.assign_to_connection(connection)
1670
+
1671
+ return closing_connections
1672
+
1673
+
1674
+ class _Httplib2RawResponse:
1675
+ """Adapter around httpcore.Response providing an http.client.HTTPResponse-compatible interface."""
1676
+
1677
+ def __init__(self, hc_resp: _HttpcoreResponse, method: str = "GET", debuglevel: int = 0):
1678
+ self._hc_resp = hc_resp
1679
+ self._method = method.upper()
1680
+ self.debuglevel = debuglevel
1681
+ self._stream_iter = iter(hc_resp.stream)
1682
+ self._buffer = b""
1683
+ self._closed = False
1684
+
1685
+ self.status = hc_resp.status
1686
+ http_ver = hc_resp.extensions.get("http_version", b"HTTP/1.1")
1687
+ if http_ver == b"HTTP/2":
1688
+ self.version = 20
1689
+ elif http_ver == b"HTTP/1.0":
1690
+ self.version = 10
1691
+ else:
1692
+ self.version = 11
1693
+
1694
+ reason_bytes = hc_resp.extensions.get("reason_phrase")
1695
+ if reason_bytes is not None:
1696
+ self.reason = (
1697
+ reason_bytes.decode("iso-8859-1")
1698
+ if isinstance(reason_bytes, bytes)
1699
+ else str(reason_bytes)
1700
+ )
1701
+ else:
1702
+ self.reason = http.client.responses.get(self.status, "OK")
1703
+
1704
+ self._headers_list = [
1705
+ (
1706
+ k.decode("iso-8859-1") if isinstance(k, bytes) else str(k),
1707
+ v.decode("iso-8859-1") if isinstance(v, bytes) else str(v),
1708
+ )
1709
+ for k, v in hc_resp.headers
1710
+ ]
1711
+
1712
+ self.stream_id = hc_resp.extensions.get("stream_id")
1713
+
1714
+ if self.debuglevel > 0:
1715
+ proto_str = http_ver.decode("ascii") if isinstance(http_ver, bytes) else str(http_ver)
1716
+ status_line = f"{proto_str} {self.status} {self.reason}\r\n"
1717
+ _debug_print("reply:", repr(status_line))
1718
+ if self.stream_id is not None:
1719
+ _debug_print("stream:", str(self.stream_id))
1720
+ for k, v in self._headers_list:
1721
+ _debug_print("header:", k + ":", v)
1722
+
1723
+ def set_debuglevel(self, level: int):
1724
+ self.debuglevel = level
1725
+
1726
+ def getheaders(self):
1727
+ return list(self._headers_list)
1728
+
1729
+ def getheader(self, name, default=None):
1730
+ target = name.lower()
1731
+ vals = [v for k, v in self._headers_list if k.lower() == target]
1732
+ if not vals:
1733
+ return default
1734
+ return ", ".join(vals)
1735
+
1736
+ def read(self, amt=None) -> bytes:
1737
+ if self._method == "HEAD":
1738
+ self.close()
1739
+ return b""
1740
+
1741
+ if amt is None:
1742
+ chunks = [self._buffer] if self._buffer else []
1743
+ self._buffer = b""
1744
+ if not self._closed:
1745
+ try:
1746
+ for chunk in self._stream_iter:
1747
+ if chunk:
1748
+ chunks.append(chunk)
1749
+ except _HttpcoreRemoteProtocolError as exc:
1750
+ if "peer closed connection without sending complete message body" not in str(exc):
1751
+ self.close()
1752
+ raise _unwrap_httpcore_exception(exc) from exc
1753
+ except Exception as exc:
1754
+ self.close()
1755
+ raise _unwrap_httpcore_exception(exc) from exc
1756
+ finally:
1757
+ self.close()
1758
+ return b"".join(chunks)
1759
+
1760
+ while len(self._buffer) < amt and not self._closed:
1761
+ try:
1762
+ chunk = next(self._stream_iter)
1763
+ if chunk:
1764
+ self._buffer += chunk
1765
+ except StopIteration:
1766
+ self.close()
1767
+ break
1768
+ except _HttpcoreRemoteProtocolError as exc:
1769
+ if "peer closed connection without sending complete message body" in str(exc):
1770
+ self.close()
1771
+ break
1772
+ self.close()
1773
+ raise _unwrap_httpcore_exception(exc) from exc
1774
+ except Exception as exc:
1775
+ self.close()
1776
+ raise _unwrap_httpcore_exception(exc) from exc
1777
+
1778
+ result, self._buffer = self._buffer[:amt], self._buffer[amt:]
1779
+ return result
1780
+
1781
+ def close(self) -> None:
1782
+ if not self._closed:
1783
+ self._closed = True
1784
+ with contextlib.suppress(Exception):
1785
+ self._hc_resp.close()
1786
+
1787
+
1788
+ class HTTPConnectionWithTimeout(http.client.HTTPConnection):
1789
+ """HTTPConnection subclass that supports timeouts and thread-safe httpx/httpcore pooling.
1790
+
1791
+ All timeouts are in seconds. If None is passed for timeout then
1792
+ Python's default timeout for sockets will be used. See for example
1793
+ the docs of socket.setdefaulttimeout():
1794
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
1795
+ """
1796
+
1797
+ _is_https = False
1798
+ _uses_httpcore_pool = True
1799
+ enable_http2 = False
1800
+
1801
+ @property
1802
+ def debuglevel(self):
1803
+ if hasattr(self, "_debuglevel"):
1804
+ return self._debuglevel
1805
+ return debuglevel or getattr(http.client.HTTPConnection, "debuglevel", 0)
1806
+
1807
+ @debuglevel.setter
1808
+ def debuglevel(self, val):
1809
+ self._debuglevel = val
1810
+
1811
+ def __init__(self, host, port=None, timeout=None, proxy_info=None, pool_maxsize=DEFAULT_POOL_MAXSIZE):
1812
+ self._initialized = False
1813
+ self._override_sock = None
1814
+ self._last_sock = None
1815
+ self._local = threading.local()
1816
+ self._pool_maxsize = pool_maxsize
1817
+
1818
+ super().__init__(host, port=port, timeout=timeout)
1819
+ if isinstance(self.host, str) and self.host.startswith("[") and self.host.endswith("]"):
1820
+ self.host = self.host[1:-1]
1821
+
1822
+ self.proxy_info = proxy_info
1823
+ if proxy_info and not isinstance(proxy_info, ProxyInfo):
1824
+ self.proxy_info = proxy_info("http")
1825
+
1826
+ self._backend = _Httplib2NetworkBackend(self)
1827
+ self.httpcore_pool = _Httplib2ConnectionPool(
1828
+ max_connections=pool_maxsize,
1829
+ max_keepalive_connections=pool_maxsize,
1830
+ keepalive_expiry=30.0,
1831
+ http1=True,
1832
+ http2=False,
1833
+ network_backend=self._backend,
1834
+ )
1835
+ self._initialized = True
1836
+
1837
+ def _record_sock(self, sock):
1838
+ self._last_sock = sock
1839
+ if hasattr(self, "_local"):
1840
+ self._local.last_sock = sock
1841
+
1842
+ @property
1843
+ def sock(self):
1844
+ if getattr(self, "_override_sock", None) is not None:
1845
+ return self._override_sock
1846
+ local_obj = getattr(self, "_local", None)
1847
+ if local_obj is not None:
1848
+ tsock = getattr(local_obj, "last_sock", None)
1849
+ if tsock is not None and tsock.fileno() != -1:
1850
+ return tsock
1851
+ lsock = getattr(self, "_last_sock", None)
1852
+ if lsock is not None and lsock.fileno() != -1:
1853
+ return lsock
1854
+ return None
1855
+
1856
+ @sock.setter
1857
+ def sock(self, value):
1858
+ if not getattr(self, "_initialized", False):
1859
+ self._override_sock = value
1860
+ return
1861
+ self._override_sock = value
1862
+ if value is None:
1863
+ self._last_sock = None
1864
+ if hasattr(self, "_local"):
1865
+ self._local.last_sock = None
1866
+
1867
+ def _connect_raw_socket(self, include_flowinfo=True, timeout=None):
1868
+ effective_timeout = self.timeout if timeout is None else timeout
1869
+ if self.proxy_info and socks is None:
1870
+ raise ProxiesUnavailableError("Proxy support missing but proxy use was requested!")
1871
+ if self.proxy_info and self.proxy_info.isgood() and self.proxy_info.applies_to(self.host):
1872
+ use_proxy = True
1873
+ (
1874
+ proxy_type,
1875
+ proxy_host,
1876
+ proxy_port,
1877
+ proxy_rdns,
1878
+ proxy_user,
1879
+ proxy_pass,
1880
+ proxy_headers,
1881
+ ) = self.proxy_info.astuple()
1882
+ host = proxy_host
1883
+ port = proxy_port
1884
+ else:
1885
+ use_proxy = False
1886
+ host = self.host
1887
+ port = self.port
1888
+ proxy_type = None
1889
+ proxy_headers = None
1890
+
1891
+ socket_err = None
1892
+ sock = None
1893
+
1894
+ addrinfos = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
1895
+ if isinstance(host, str) and host.lower() == "localhost":
1896
+ addrinfos.sort(key=lambda r: 0 if r[0] == socket.AF_INET else 1)
1897
+
1898
+ for res in addrinfos:
1899
+ af, socktype, proto, canonname, sa = res
1900
+ try:
1901
+ if use_proxy:
1902
+ sock = socks.socksocket(af, socktype, proto)
1903
+ sock.setproxy(
1904
+ proxy_type,
1905
+ proxy_host,
1906
+ proxy_port,
1907
+ proxy_rdns,
1908
+ proxy_user,
1909
+ proxy_pass,
1910
+ )
1911
+ else:
1912
+ sock = socket.socket(af, socktype, proto)
1913
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
1914
+ if has_timeout(effective_timeout):
1915
+ sock.settimeout(effective_timeout)
1916
+ if self.debuglevel > 0:
1917
+ _debug_print(f"connect: ({self.host}, {self.port}) ************")
1918
+ if use_proxy:
1919
+ _debug_print(
1920
+ f"proxy: {(proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers)} ************"
1921
+ )
1922
+ target_addr = ((self.host, self.port) + sa[2:]) if include_flowinfo else (self.host, self.port)
1923
+ sock.connect(target_addr)
1924
+ except (socket.timeout, socket.gaierror):
1925
+ if sock:
1926
+ sock.close()
1927
+ raise
1928
+ except socket.error as e:
1929
+ if socket_err is None or "PySocks doesn't support IPv6" not in str(e):
1930
+ socket_err = e
1931
+ if self.debuglevel > 0:
1932
+ _debug_print(f"connect fail: ({self.host}, {self.port})")
1933
+ if use_proxy:
1934
+ _debug_print(
1935
+ f"proxy: {(proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers)}"
1936
+ )
1937
+ if sock:
1938
+ sock.close()
1939
+ sock = None
1940
+ continue
1941
+ break
1942
+
1943
+ if not sock:
1944
+ raise socket_err
1945
+ return sock
1946
+
1947
+ def connect(self):
1948
+ """Connect to the host and port specified in __init__."""
1949
+ sock = self._connect_raw_socket(include_flowinfo=True, timeout=self.timeout)
1950
+ stream = _Httplib2SyncStream(sock, owner=self, enable_http2=False)
1951
+ self._record_sock(sock)
1952
+ self._backend.push_preconnected(stream)
1953
+
1954
+ def _clear_pending_outcome(self):
1955
+ pending = getattr(self._local, "pending_outcome", None)
1956
+ if pending is not None:
1957
+ self._local.pending_outcome = None
1958
+ hc_resp, _ = pending
1959
+ if hc_resp is not None:
1960
+ with contextlib.suppress(Exception):
1961
+ hc_resp.close()
1962
+
1963
+ def request(self, method, url, body=None, headers=None, *, encode_chunked=False):
1964
+ self._clear_pending_outcome()
1965
+
1966
+ raw_headers = []
1967
+ has_host = False
1968
+ has_content_length = False
1969
+ has_transfer_encoding = False
1970
+
1971
+ if headers:
1972
+ items = headers.items() if hasattr(headers, "items") else headers
1973
+ for k, v in items:
1974
+ kb = k.encode("iso-8859-1") if isinstance(k, str) else bytes(k)
1975
+ vb = v.encode("iso-8859-1") if isinstance(v, str) else bytes(v)
1976
+ if not kb or kb.startswith(b":") or any(b <= 0x20 or b == 0x7F or b == 0x3A for b in kb):
1977
+ raise http.client.HTTPException(f"Invalid header name {kb!r}")
1978
+ if any(b in (0x00, 0x0A, 0x0D) for b in vb):
1979
+ raise http.client.HTTPException(f"Invalid header value {vb!r}")
1980
+ kl = kb.lower()
1981
+ if kl == b"host":
1982
+ has_host = True
1983
+ elif kl == b"content-length":
1984
+ has_content_length = True
1985
+ elif kl == b"transfer-encoding":
1986
+ has_transfer_encoding = True
1987
+ raw_headers.append((kb, vb))
1988
+
1989
+ if not has_host:
1990
+ host_str = self.host
1991
+ if ":" in host_str and not host_str.startswith("["):
1992
+ host_str = f"[{host_str}]"
1993
+ default_port = 443 if self._is_https else 80
1994
+ if self.port and self.port != default_port:
1995
+ host_hdr = f"{host_str}:{self.port}".encode("ascii")
1996
+ else:
1997
+ host_hdr = host_str.encode("ascii")
1998
+ raw_headers.insert(0, (b"Host", host_hdr))
1999
+
2000
+ method_bytes = method.encode("ascii") if isinstance(method, str) else bytes(method)
2001
+ if not method_bytes or any(b <= 0x20 or b >= 0x7F for b in method_bytes):
2002
+ raise http.client.InvalidURL(f"Invalid method {method!r}")
2003
+ upper_method = method.upper() if isinstance(method, str) else method.decode("ascii").upper()
2004
+ if body is not None and hasattr(body, "read"):
2005
+ body = body.read()
2006
+ if isinstance(body, str):
2007
+ body = body.encode("utf-8")
2008
+
2009
+ has_nonempty_body = False
2010
+ if body is None:
2011
+ content_bytes = b""
2012
+ if upper_method in ("POST", "PUT", "PATCH") and not has_content_length and not has_transfer_encoding:
2013
+ raw_headers.append((b"Content-Length", b"0"))
2014
+ elif isinstance(body, (bytes, bytearray, memoryview)):
2015
+ content_bytes = bytes(body)
2016
+ has_nonempty_body = len(content_bytes) > 0
2017
+ if not has_content_length and not has_transfer_encoding:
2018
+ if has_nonempty_body or upper_method in ("POST", "PUT", "PATCH"):
2019
+ raw_headers.append((b"Content-Length", str(len(content_bytes)).encode("ascii")))
2020
+ else:
2021
+ chunks = [c.encode("utf-8") if isinstance(c, str) else bytes(c) for c in body]
2022
+ content_bytes = b"".join(chunks)
2023
+ has_nonempty_body = len(content_bytes) > 0
2024
+ if not has_content_length and not has_transfer_encoding:
2025
+ raw_headers.append((b"Content-Length", str(len(content_bytes)).encode("ascii")))
2026
+
2027
+ scheme_bytes = b"https" if self._is_https else b"http"
2028
+ host_bytes = self.host.encode("ascii")
2029
+ target_bytes = url.encode("ascii") if isinstance(url, str) else bytes(url)
2030
+ if not target_bytes:
2031
+ target_bytes = b"/"
2032
+ if any(b <= 0x20 or b == 0x7F for b in target_bytes):
2033
+ raise http.client.InvalidURL(f"URL can't contain control characters. {url!r}")
2034
+
2035
+ effective_timeout = socket.getdefaulttimeout() if self.timeout is None else self.timeout
2036
+ timeout_ext = {
2037
+ "connect": effective_timeout,
2038
+ "read": effective_timeout,
2039
+ "write": effective_timeout,
2040
+ "pool": effective_timeout,
2041
+ }
2042
+
2043
+ hc_req = _HttpcoreRequest(
2044
+ method=method_bytes,
2045
+ url=_HttpcoreURL(
2046
+ scheme=scheme_bytes,
2047
+ host=host_bytes,
2048
+ port=self.port,
2049
+ target=target_bytes,
2050
+ ),
2051
+ headers=raw_headers,
2052
+ content=content_bytes,
2053
+ extensions={
2054
+ "timeout": timeout_ext,
2055
+ "has_nonempty_body": has_nonempty_body,
2056
+ "request_sent": False,
2057
+ },
2058
+ )
2059
+ hc_req.method = method_bytes
2060
+
2061
+ if self.debuglevel > 0:
2062
+ is_h2 = False
2063
+ if self._is_https and getattr(self, "enable_http2", False):
2064
+ pool_conns = getattr(self.httpcore_pool, "_connections", [])
2065
+ for c in pool_conns:
2066
+ sub_c = getattr(c, "_connection", None)
2067
+ if isinstance(sub_c, _Httplib2HTTP2Connection):
2068
+ is_h2 = True
2069
+ break
2070
+ proto = "HTTP/2" if is_h2 else "HTTP/1.1"
2071
+ req_line = f"{upper_method} {target_bytes.decode('ascii', errors='replace')} {proto}\r\n"
2072
+ hdr_str = "".join(
2073
+ f"{k.decode('iso-8859-1') if isinstance(k, bytes) else k}: {v.decode('iso-8859-1') if isinstance(v, bytes) else v}\r\n"
2074
+ for k, v in raw_headers
2075
+ )
2076
+ _debug_print("send:", repr((req_line + hdr_str + "\r\n").encode("iso-8859-1")))
2077
+ if has_nonempty_body:
2078
+ _debug_print("send:", repr(content_bytes))
2079
+
2080
+ try:
2081
+ hc_resp = self.httpcore_pool.handle_request(hc_req)
2082
+ net_stream = hc_resp.extensions.get("network_stream")
2083
+ if net_stream is not None:
2084
+ sock_obj = net_stream.get_extra_info("socket")
2085
+ if sock_obj is not None:
2086
+ self._record_sock(sock_obj)
2087
+ self._local.pending_outcome = (hc_resp, upper_method)
2088
+ except Exception as exc:
2089
+ unwrapped = _unwrap_httpcore_exception(exc)
2090
+ if hc_req.extensions.get("request_sent"):
2091
+ self._local.pending_outcome = (None, unwrapped)
2092
+ else:
2093
+ raise unwrapped
2094
+
2095
+ def getresponse(self):
2096
+ pending = getattr(self._local, "pending_outcome", None)
2097
+ if pending is None:
2098
+ raise http.client.ResponseNotReady()
2099
+ self._local.pending_outcome = None
2100
+ hc_resp, second = pending
2101
+ if hc_resp is None:
2102
+ raise second
2103
+ return _Httplib2RawResponse(hc_resp, method=second, debuglevel=self.debuglevel)
2104
+
2105
+ def close(self):
2106
+ self._override_sock = None
2107
+ self._last_sock = None
2108
+ if hasattr(self, "_local"):
2109
+ self._local.last_sock = None
2110
+ self._clear_pending_outcome()
2111
+ if hasattr(self, "_backend"):
2112
+ self._backend.clear_preconnected()
2113
+ if hasattr(self, "httpcore_pool"):
2114
+ with contextlib.suppress(Exception):
2115
+ self.httpcore_pool.close()
2116
+
2117
+ def close_all(self):
2118
+ self.close()
2119
+
2120
+
2121
+ class HTTPSConnectionWithTimeout(HTTPConnectionWithTimeout, http.client.HTTPSConnection):
2122
+ """This class allows communication via SSL using httpx/httpcore with HTTP/2 multiplexing.
2123
+
2124
+ All timeouts are in seconds. If None is passed for timeout then
2125
+ Python's default timeout for sockets will be used. See for example
2126
+ the docs of socket.setdefaulttimeout():
2127
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
2128
+ """
2129
+
2130
+ _is_https = True
2131
+ enable_http2 = True
2132
+
2133
+ def __init__(
2134
+ self,
2135
+ host,
2136
+ port=None,
2137
+ key_file=None,
2138
+ cert_file=None,
2139
+ timeout=None,
2140
+ proxy_info=None,
2141
+ ca_certs=None,
2142
+ disable_ssl_certificate_validation=False,
2143
+ tls_maximum_version=None,
2144
+ tls_minimum_version=None,
2145
+ key_password=None,
2146
+ enable_http2=None,
2147
+ pool_maxsize=DEFAULT_POOL_MAXSIZE,
2148
+ ):
2149
+ self._initialized = False
2150
+ self._override_sock = None
2151
+ self._last_sock = None
2152
+ self._local = threading.local()
2153
+ self._pool_maxsize = pool_maxsize
2154
+
2155
+ self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
2156
+ self.ca_certs = _resolve_ca_certs(ca_certs)
2157
+ self.tls_maximum_version = tls_maximum_version
2158
+ self.tls_minimum_version = tls_minimum_version
2159
+ self.key_file = key_file
2160
+ self.cert_file = cert_file
2161
+ self.key_password = key_password
2162
+
2163
+ if enable_http2 is None:
2164
+ enable_http2 = getattr(self.__class__, "enable_http2", ENABLE_HTTP2)
2165
+ effective_h2 = bool(enable_http2) and bool(ENABLE_HTTP2)
2166
+ self.enable_http2 = effective_h2
2167
+
2168
+ self.proxy_info = proxy_info
2169
+ if proxy_info and not isinstance(proxy_info, ProxyInfo):
2170
+ self.proxy_info = proxy_info("https")
2171
+
2172
+ context = _build_ssl_context(
2173
+ self.disable_ssl_certificate_validation,
2174
+ self.ca_certs,
2175
+ cert_file,
2176
+ key_file,
2177
+ maximum_version=tls_maximum_version,
2178
+ minimum_version=tls_minimum_version,
2179
+ key_password=key_password,
2180
+ enable_http2=effective_h2,
2181
+ )
2182
+ http.client.HTTPSConnection.__init__(
2183
+ self,
2184
+ host,
2185
+ port=port,
2186
+ timeout=timeout,
2187
+ context=context,
2188
+ )
2189
+ if isinstance(self.host, str) and self.host.startswith("[") and self.host.endswith("]"):
2190
+ self.host = self.host[1:-1]
2191
+
2192
+ self._backend = _Httplib2NetworkBackend(self)
2193
+ self.httpcore_pool = _Httplib2ConnectionPool(
2194
+ ssl_context=self._context,
2195
+ max_connections=pool_maxsize,
2196
+ max_keepalive_connections=pool_maxsize,
2197
+ keepalive_expiry=30.0,
2198
+ http1=True,
2199
+ http2=effective_h2,
2200
+ network_backend=self._backend,
2201
+ )
2202
+ self._initialized = True
2203
+
2204
+ def downgrade_to_http1(self):
2205
+ """Downgrade connection to HTTP/1.1 by disabling HTTP/2 ALPN and recreating the connection pool."""
2206
+ self.enable_http2 = False
2207
+ self.close()
2208
+ if hasattr(self, "_context") and self._context is not None:
2209
+ with contextlib.suppress(NotImplementedError, ssl.SSLError):
2210
+ self._context.set_alpn_protocols(["http/1.1"])
2211
+ self._backend = _Httplib2NetworkBackend(self)
2212
+ self.httpcore_pool = _Httplib2ConnectionPool(
2213
+ ssl_context=self._context,
2214
+ max_connections=self._pool_maxsize,
2215
+ max_keepalive_connections=self._pool_maxsize,
2216
+ keepalive_expiry=30.0,
2217
+ http1=True,
2218
+ http2=False,
2219
+ network_backend=self._backend,
2220
+ )
2221
+
2222
+ def connect(self):
2223
+ """Connect to a host on a given (SSL) port."""
2224
+ sock = None
2225
+ ssl_sock = None
2226
+ try:
2227
+ sock = self._connect_raw_socket(include_flowinfo=False, timeout=self.timeout)
2228
+ if self.enable_http2 and ENABLE_HTTP2:
2229
+ with contextlib.suppress(NotImplementedError):
2230
+ self._context.set_alpn_protocols(["h2", "http/1.1"])
2231
+ else:
2232
+ with contextlib.suppress(NotImplementedError):
2233
+ self._context.set_alpn_protocols(["http/1.1"])
2234
+ ssl_sock = self._context.wrap_socket(sock, server_hostname=self.host)
2235
+ except (ssl.SSLError, ssl.CertificateError):
2236
+ if sock:
2237
+ sock.close()
2238
+ if ssl_sock:
2239
+ ssl_sock.close()
2240
+ self._last_sock = None
2241
+ raise
2242
+
2243
+ stream = _Httplib2SyncStream(ssl_sock, owner=self, enable_http2=bool(self.enable_http2 and ENABLE_HTTP2))
2244
+ self._record_sock(ssl_sock)
2245
+ self._backend.push_preconnected(stream)
2246
+
2247
+
2248
+ SCHEME_TO_CONNECTION = {
2249
+ "http": HTTPConnectionWithTimeout,
2250
+ "https": HTTPSConnectionWithTimeout,
2251
+ }
2252
+
2253
+
2254
+ _HTTP_LOCKS_GUARD = threading.RLock()
2255
+
2256
+
2257
+ def _get_http_lock(http_instance):
2258
+ lock = getattr(http_instance, "_lock", None)
2259
+ if lock is None:
2260
+ with _HTTP_LOCKS_GUARD:
2261
+ lock = getattr(http_instance, "_lock", None)
2262
+ if lock is None:
2263
+ lock = threading.RLock()
2264
+ http_instance._lock = lock
2265
+ return lock
2266
+
2267
+
2268
+ class Http:
2269
+ """An HTTP client that handles:
2270
+
2271
+ - all methods
2272
+ - caching
2273
+ - ETags
2274
+ - compression,
2275
+ - HTTPS
2276
+ - Basic
2277
+ - Digest
2278
+ - WSSE
2279
+
2280
+ and more, powered by httpx/httpcore with HTTP/2 multiplexing and thread-safe connection pooling.
2281
+ """
2282
+
2283
+ enable_http2 = True
2284
+ auto_downgrade_http2 = True
2285
+ pool_maxsize = DEFAULT_POOL_MAXSIZE
2286
+
2287
+ def __init__(
2288
+ self,
2289
+ cache=None,
2290
+ timeout=None,
2291
+ proxy_info=proxy_info_from_environment,
2292
+ ca_certs=None,
2293
+ disable_ssl_certificate_validation=False,
2294
+ tls_maximum_version=None,
2295
+ tls_minimum_version=None,
2296
+ decode_limit_hard=None,
2297
+ decode_limit_safe=None,
2298
+ decode_limit_ratio=None,
2299
+ decode_limit_chunk=None,
2300
+ enable_http2=None,
2301
+ auto_downgrade_http2=None,
2302
+ pool_maxsize=None,
2303
+ ):
2304
+ self.proxy_info = proxy_info
2305
+ self.ca_certs = ca_certs
2306
+ self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
2307
+ self.tls_maximum_version = tls_maximum_version
2308
+ self.tls_minimum_version = tls_minimum_version
2309
+ if enable_http2 is not None:
2310
+ self.enable_http2 = bool(enable_http2)
2311
+ else:
2312
+ self.enable_http2 = getattr(self.__class__, "enable_http2", ENABLE_HTTP2)
2313
+
2314
+ if auto_downgrade_http2 is not None:
2315
+ self.auto_downgrade_http2 = bool(auto_downgrade_http2)
2316
+ else:
2317
+ self.auto_downgrade_http2 = getattr(self.__class__, "auto_downgrade_http2", AUTO_DOWNGRADE_HTTP2)
2318
+
2319
+ if pool_maxsize is not None:
2320
+ self.pool_maxsize = pool_maxsize
2321
+
2322
+ self._http2_downgraded_hosts = set()
2323
+
2324
+ # Map domain name to an httplib connection
2325
+ self.connections = {}
2326
+ # The location of the cache, for now a directory
2327
+ # where cached responses are held.
2328
+ if cache and isinstance(cache, str):
2329
+ self.cache = FileCache(cache)
2330
+ else:
2331
+ self.cache = cache
2332
+
2333
+ # Name/password
2334
+ self.credentials = Credentials()
2335
+
2336
+ # Key/cert
2337
+ self.certificates = KeyCerts()
2338
+
2339
+ # authorization objects
2340
+ self.authorizations = []
2341
+
2342
+ # If set to False then no redirects are followed, even safe ones.
2343
+ self.follow_redirects = True
2344
+
2345
+ self.redirect_codes = REDIRECT_CODES
2346
+
2347
+ # Which HTTP methods do we apply optimistic concurrency to, i.e.
2348
+ # which methods get an "if-match:" etag header added to them.
2349
+ self.optimistic_concurrency_methods = ["PUT", "PATCH"]
2350
+
2351
+ self.safe_methods = list(SAFE_METHODS)
2352
+
2353
+ # If 'follow_redirects' is True, and this is set to True then
2354
+ # all redirecs are followed, including unsafe ones.
2355
+ self.follow_all_redirects = False
2356
+
2357
+ self.ignore_etag = False
2358
+
2359
+ self.force_exception_to_status_code = False
2360
+
2361
+ self.timeout = timeout
2362
+
2363
+ # Keep Authorization: headers on a redirect.
2364
+ self.forward_authorization_headers = False
2365
+
2366
+ limit_kwargs = dict(
2367
+ hard_limit=try_value_or_env(
2368
+ int, decode_limit_hard, "httplib2_decode_limit_hard"
2369
+ ),
2370
+ safe_limit=try_value_or_env(
2371
+ int, decode_limit_safe, "httplib2_decode_limit_safe"
2372
+ ),
2373
+ ratio=try_value_or_env(
2374
+ float, decode_limit_ratio, "httplib2_decode_limit_ratio"
2375
+ ),
2376
+ chunk_size=try_value_or_env(
2377
+ int, decode_limit_chunk, "httplib2_decode_limit_chunk"
2378
+ ),
2379
+ )
2380
+ self.limit_kwargs = {k: v for k, v in limit_kwargs.items() if v is not None}
2381
+
2382
+ def close(self):
2383
+ """Close persistent connections, clear sensitive data."""
2384
+ with _get_http_lock(self):
2385
+ existing, self.connections = self.connections, {}
2386
+ for _, c in existing.items():
2387
+ getattr(c, "close_all", c.close)()
2388
+ self.certificates.clear()
2389
+ self.clear_credentials()
2390
+
2391
+ def force_http11(self):
2392
+ """Force this Http instance to use HTTP/1.1 only and downgrade all active connections."""
2393
+ self.enable_http2 = False
2394
+ with _get_http_lock(self):
2395
+ for conn in self.connections.values():
2396
+ if hasattr(conn, "downgrade_to_http1"):
2397
+ conn.downgrade_to_http1()
2398
+
2399
+ def downgrade_to_http1(self):
2400
+ """Alias for force_http11()."""
2401
+ self.force_http11()
2402
+
2403
+ def downgrade_host_to_http1(self, authority):
2404
+ """Force connections to a specific host:port authority to downgrade to HTTP/1.1."""
2405
+ self._record_host_downgrade(authority)
2406
+ host_only = authority.split(":")[0]
2407
+ with _get_http_lock(self):
2408
+ for scheme in ("https", "http"):
2409
+ for key in (f"{scheme}:{authority}", f"{scheme}:{host_only}"):
2410
+ conn = self.connections.get(key)
2411
+ if conn is not None and hasattr(conn, "downgrade_to_http1"):
2412
+ conn.downgrade_to_http1()
2413
+
2414
+ def _record_host_downgrade(self, host_or_authority):
2415
+ self._http2_downgraded_hosts.add(host_or_authority)
2416
+ with _DOWNGRADE_LOCK:
2417
+ DOWNGRADED_HOSTS.add(host_or_authority)
2418
+ if ":" not in host_or_authority or host_or_authority.endswith(":443"):
2419
+ host_only = host_or_authority.split(":")[0]
2420
+ self._http2_downgraded_hosts.add(host_only)
2421
+ with _DOWNGRADE_LOCK:
2422
+ DOWNGRADED_HOSTS.add(host_only)
2423
+
2424
+ def _is_host_downgraded(self, host_or_authority):
2425
+ host_only = host_or_authority.split(":")[0]
2426
+ if host_or_authority in self._http2_downgraded_hosts or host_only in self._http2_downgraded_hosts:
2427
+ return True
2428
+ with _DOWNGRADE_LOCK:
2429
+ return host_or_authority in DOWNGRADED_HOSTS or host_only in DOWNGRADED_HOSTS
2430
+
2431
+ def __getstate__(self):
2432
+ state_dict = copy.copy(self.__dict__)
2433
+ if "request" in state_dict:
2434
+ del state_dict["request"]
2435
+ if "connections" in state_dict:
2436
+ del state_dict["connections"]
2437
+ if "_lock" in state_dict:
2438
+ del state_dict["_lock"]
2439
+ return state_dict
2440
+
2441
+ def __setstate__(self, state):
2442
+ self.__dict__.update(state)
2443
+ self.connections = {}
2444
+
2445
+ def _auth_from_challenge(self, host, request_uri, headers, response, content):
2446
+ """A generator that creates Authorization objects
2447
+ that can be applied to requests.
2448
+ """
2449
+ challenges = auth._parse_www_authenticate(response, "www-authenticate")
2450
+ for cred in self.credentials.iter(host):
2451
+ for scheme in AUTH_SCHEME_ORDER:
2452
+ if scheme in challenges:
2453
+ yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self)
2454
+
2455
+ def add_credentials(self, name, password, domain=""):
2456
+ """Add a name and password that will be used
2457
+ any time a request requires authentication."""
2458
+ self.credentials.add(name, password, domain)
2459
+
2460
+ def add_certificate(self, key, cert, domain, password=None):
2461
+ """Add a key and cert that will be used
2462
+ any time a request requires authentication."""
2463
+ self.certificates.add(key, cert, domain, password)
2464
+
2465
+ def clear_credentials(self):
2466
+ """Remove all the names and passwords
2467
+ that are used for authentication"""
2468
+ self.credentials.clear()
2469
+ self.authorizations = []
2470
+
2471
+ def _conn_request(self, conn, request_uri, method, body, headers):
2472
+ i = 0
2473
+ seen_bad_status_line = False
2474
+ uses_pool = getattr(conn, "_uses_httpcore_pool", False)
2475
+ initial_body_pos = body.tell() if hasattr(body, "tell") else None
2476
+ while i < RETRIES:
2477
+ i += 1
2478
+ if initial_body_pos is not None and hasattr(body, "seek"):
2479
+ with contextlib.suppress(Exception):
2480
+ body.seek(initial_body_pos)
2481
+ try:
2482
+ if not uses_pool and conn.sock is None:
2483
+ conn.connect()
2484
+ conn.request(method, request_uri, body, headers)
2485
+ response = conn.getresponse()
2486
+ except socket.timeout:
2487
+ conn.close()
2488
+ raise
2489
+ except socket.gaierror:
2490
+ conn.close()
2491
+ raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
2492
+ except Exception as exc:
2493
+ can_downgrade = (
2494
+ getattr(conn, "_is_https", False)
2495
+ and getattr(conn, "enable_http2", False)
2496
+ and getattr(self, "auto_downgrade_http2", False)
2497
+ )
2498
+ if can_downgrade and _should_downgrade_on_error(exc):
2499
+ if getattr(conn, "debuglevel", 0) > 0 or getattr(self, "debuglevel", 0) > 0:
2500
+ _debug_print(
2501
+ f"auto-downgrade: HTTP/2 failed for {conn.host} ({type(exc).__name__}: {exc}), "
2502
+ f"falling back to HTTP/1.1 and retrying"
2503
+ )
2504
+ conn_authority = (
2505
+ f"{conn.host}:{conn.port}"
2506
+ if getattr(conn, "port", None) not in (None, 443)
2507
+ else conn.host
2508
+ )
2509
+ self._record_host_downgrade(conn_authority)
2510
+ conn.downgrade_to_http1()
2511
+ i = 0
2512
+ continue
2513
+
2514
+ if isinstance(exc, (http.client.BadStatusLine, http.client.ResponseNotReady)):
2515
+ if not seen_bad_status_line and i == 1:
2516
+ i = 0
2517
+ seen_bad_status_line = True
2518
+ conn.close()
2519
+ if not uses_pool:
2520
+ conn.connect()
2521
+ continue
2522
+ else:
2523
+ conn.close()
2524
+ raise exc
2525
+
2526
+ if isinstance(exc, socket.error):
2527
+ errno_ = _errno_from_exception(exc)
2528
+ if errno_ in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES:
2529
+ continue # retry on potentially transient errors
2530
+ conn.close()
2531
+ raise exc
2532
+
2533
+ if isinstance(exc, http.client.HTTPException):
2534
+ if conn.sock is None:
2535
+ if i < RETRIES - 1:
2536
+ conn.close()
2537
+ conn.connect()
2538
+ continue
2539
+ else:
2540
+ conn.close()
2541
+ raise exc
2542
+ if i < RETRIES - 1:
2543
+ conn.close()
2544
+ conn.connect()
2545
+ continue
2546
+ conn.close()
2547
+ raise exc
2548
+
2549
+ conn.close()
2550
+ raise exc
2551
+ else:
2552
+ content = b""
2553
+ if method == "HEAD":
2554
+ if hasattr(response, "close"):
2555
+ response.close()
2556
+ if getattr(response, "version", 11) != 20:
2557
+ conn.close()
2558
+ else:
2559
+ content = response.read()
2560
+ response = Response(response)
2561
+ if method != "HEAD":
2562
+ content = _decompressContent(response, content, self.limit_kwargs)
2563
+
2564
+ break
2565
+ return (response, content)
2566
+
2567
+ def _request(
2568
+ self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey,
2569
+ ):
2570
+ """Do the actual request using the connection object
2571
+ and also follow one level of redirects if necessary"""
2572
+
2573
+ auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)]
2574
+ auth_obj = auths and sorted(auths)[0][1] or None
2575
+ if auth_obj:
2576
+ auth_obj.request(method, request_uri, headers, body)
2577
+
2578
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers)
2579
+
2580
+ if auth_obj:
2581
+ if auth_obj.response(response, body):
2582
+ auth_obj.request(method, request_uri, headers, body)
2583
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers)
2584
+ response._stale_digest = 1
2585
+
2586
+ if response.status == 401:
2587
+ for authorization in self._auth_from_challenge(host, request_uri, headers, response, content):
2588
+ authorization.request(method, request_uri, headers, body)
2589
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers)
2590
+ if response.status != 401:
2591
+ self.authorizations.append(authorization)
2592
+ authorization.response(response, body)
2593
+ break
2594
+
2595
+ if self.follow_all_redirects or method in self.safe_methods or response.status in (303, 308):
2596
+ if self.follow_redirects and response.status in self.redirect_codes:
2597
+ if redirections:
2598
+ if "location" not in response and response.status != 300:
2599
+ raise RedirectMissingLocation(
2600
+ _("Redirected but the response is missing a Location: header."), response, content,
2601
+ )
2602
+ if "location" in response:
2603
+ location = response["location"]
2604
+ (scheme, authority, path, query, fragment) = parse_uri(location)
2605
+ if authority == None:
2606
+ response["location"] = urllib.parse.urljoin(absolute_uri, location)
2607
+ if response.status == 308 or (response.status == 301 and (method in self.safe_methods)):
2608
+ response["-x-permanent-redirect-url"] = response["location"]
2609
+ if "content-location" not in response:
2610
+ response["content-location"] = absolute_uri
2611
+ _updateCache(headers, response, content, self.cache, cachekey)
2612
+ if "if-none-match" in headers:
2613
+ del headers["if-none-match"]
2614
+ if "if-modified-since" in headers:
2615
+ del headers["if-modified-since"]
2616
+ if "authorization" in headers and not self.forward_authorization_headers:
2617
+ del headers["authorization"]
2618
+ if "location" in response:
2619
+ location = response["location"]
2620
+ old_response = copy.deepcopy(response)
2621
+ if "content-location" not in old_response:
2622
+ old_response["content-location"] = absolute_uri
2623
+ redirect_method = method
2624
+ if response.status in [302, 303]:
2625
+ redirect_method = "GET"
2626
+ body = None
2627
+ for body_hdr in ("content-length", "content-type", "transfer-encoding"):
2628
+ if body_hdr in headers:
2629
+ del headers[body_hdr]
2630
+ orig_scheme, orig_authority = parse_uri(absolute_uri)[:2]
2631
+ dest_scheme, dest_authority = parse_uri(location)[:2]
2632
+ if (
2633
+ "authorization" in headers
2634
+ and orig_scheme
2635
+ and dest_scheme
2636
+ and orig_scheme.lower() == "https"
2637
+ and dest_scheme.lower() == "http"
2638
+ ):
2639
+ del headers["authorization"]
2640
+ if dest_authority and orig_authority and dest_authority.lower() != orig_authority.lower():
2641
+ if "cookie" in headers:
2642
+ del headers["cookie"]
2643
+ if "host" in headers:
2644
+ del headers["host"]
2645
+ (response, content) = self.request(
2646
+ location, method=redirect_method, body=body, headers=headers, redirections=redirections - 1,
2647
+ )
2648
+ response.previous = old_response
2649
+ else:
2650
+ raise RedirectLimit(
2651
+ "Redirected more times than redirection_limit allows.", response, content,
2652
+ )
2653
+ elif response.status in [200, 203] and method in self.safe_methods:
2654
+ if "content-location" not in response:
2655
+ response["content-location"] = absolute_uri
2656
+ _updateCache(headers, response, content, self.cache, cachekey)
2657
+
2658
+ return (response, content)
2659
+
2660
+ def _normalize_headers(self, headers):
2661
+ return _normalize_headers(headers)
2662
+
2663
+ def request(
2664
+ self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None,
2665
+ ):
2666
+ """Performs a single HTTP request."""
2667
+ conn_key = ""
2668
+
2669
+ try:
2670
+ if headers is None:
2671
+ headers = {}
2672
+ else:
2673
+ headers = self._normalize_headers(headers)
2674
+
2675
+ if "user-agent" not in headers:
2676
+ headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
2677
+
2678
+ uri = iri2uri(uri)
2679
+ # Prevent CWE-75 space injection to manipulate request via part of uri.
2680
+ # Prevent CWE-93 CRLF injection to modify headers via part of uri.
2681
+ uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A")
2682
+
2683
+ (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
2684
+
2685
+ conn_key = scheme + ":" + authority
2686
+ with _get_http_lock(self):
2687
+ conn = self.connections.get(conn_key)
2688
+ is_downgraded = self._is_host_downgraded(authority)
2689
+ should_h2 = (
2690
+ bool(self.enable_http2)
2691
+ and bool(ENABLE_HTTP2)
2692
+ and not is_downgraded
2693
+ )
2694
+ if conn is not None and type(conn) is HTTPSConnectionWithTimeout:
2695
+ expected_ca_certs = _resolve_ca_certs(self.ca_certs)
2696
+ if (
2697
+ bool(conn.disable_ssl_certificate_validation) != bool(self.disable_ssl_certificate_validation)
2698
+ or conn.ca_certs != expected_ca_certs
2699
+ or conn.tls_minimum_version != self.tls_minimum_version
2700
+ or conn.tls_maximum_version != self.tls_maximum_version
2701
+ ):
2702
+ conn.close()
2703
+ conn = None
2704
+ del self.connections[conn_key]
2705
+ if conn is not None:
2706
+ if getattr(conn, "_is_https", False) and getattr(conn, "enable_http2", False) and not should_h2:
2707
+ conn.downgrade_to_http1()
2708
+ if conn is None:
2709
+ if not connection_type:
2710
+ connection_type = SCHEME_TO_CONNECTION[scheme]
2711
+ certs = list(self.certificates.iter(authority))
2712
+ if issubclass(connection_type, HTTPSConnectionWithTimeout):
2713
+ extra_https_kwargs = {}
2714
+ if connection_type is HTTPSConnectionWithTimeout:
2715
+ extra_https_kwargs["enable_http2"] = should_h2
2716
+ extra_https_kwargs["pool_maxsize"] = self.pool_maxsize
2717
+ if certs:
2718
+ conn = self.connections[conn_key] = connection_type(
2719
+ authority,
2720
+ key_file=certs[0][0],
2721
+ cert_file=certs[0][1],
2722
+ timeout=self.timeout,
2723
+ proxy_info=self.proxy_info,
2724
+ ca_certs=self.ca_certs,
2725
+ disable_ssl_certificate_validation=self.disable_ssl_certificate_validation,
2726
+ tls_maximum_version=self.tls_maximum_version,
2727
+ tls_minimum_version=self.tls_minimum_version,
2728
+ key_password=certs[0][2],
2729
+ **extra_https_kwargs,
2730
+ )
2731
+ else:
2732
+ conn = self.connections[conn_key] = connection_type(
2733
+ authority,
2734
+ timeout=self.timeout,
2735
+ proxy_info=self.proxy_info,
2736
+ ca_certs=self.ca_certs,
2737
+ disable_ssl_certificate_validation=self.disable_ssl_certificate_validation,
2738
+ tls_maximum_version=self.tls_maximum_version,
2739
+ tls_minimum_version=self.tls_minimum_version,
2740
+ **extra_https_kwargs,
2741
+ )
2742
+ else:
2743
+ extra_http_kwargs = {}
2744
+ if connection_type is HTTPConnectionWithTimeout:
2745
+ extra_http_kwargs["pool_maxsize"] = self.pool_maxsize
2746
+ conn = self.connections[conn_key] = connection_type(
2747
+ authority,
2748
+ timeout=self.timeout,
2749
+ proxy_info=self.proxy_info,
2750
+ **extra_http_kwargs,
2751
+ )
2752
+ conn.set_debuglevel(debuglevel)
2753
+
2754
+ if "range" not in headers and "accept-encoding" not in headers:
2755
+ headers["accept-encoding"] = "gzip, deflate"
2756
+
2757
+ info = email.message.Message()
2758
+ cachekey = None
2759
+ cached_value = None
2760
+ if self.cache:
2761
+ cachekey = defrag_uri
2762
+ cached_value = self.cache.get(cachekey)
2763
+ if cached_value:
2764
+ try:
2765
+ info, content = cached_value.split(b"\r\n\r\n", 1)
2766
+ info = email.message_from_bytes(info)
2767
+ for k, v in info.items():
2768
+ if v.startswith("=?") and v.endswith("?="):
2769
+ info.replace_header(k, str(*email.header.decode_header(v)[0]))
2770
+ except (IndexError, ValueError):
2771
+ self.cache.delete(cachekey)
2772
+ cachekey = None
2773
+ cached_value = None
2774
+
2775
+ if (
2776
+ method in self.optimistic_concurrency_methods
2777
+ and self.cache
2778
+ and "etag" in info
2779
+ and not self.ignore_etag
2780
+ and "if-match" not in headers
2781
+ ):
2782
+ headers["if-match"] = info["etag"]
2783
+
2784
+ if self.cache and cachekey and method not in self.safe_methods:
2785
+ self.cache.delete(cachekey)
2786
+
2787
+ if method in self.safe_methods and "vary" in info:
2788
+ vary = info["vary"]
2789
+ vary_headers = vary.lower().replace(" ", "").split(",")
2790
+ for header in vary_headers:
2791
+ key = "-varied-%s" % header
2792
+ value = info[key]
2793
+ if headers.get(header, None) != value:
2794
+ cached_value = None
2795
+ break
2796
+
2797
+ if (
2798
+ self.cache
2799
+ and cached_value
2800
+ and (method in self.safe_methods or info["status"] == "308")
2801
+ and "range" not in headers
2802
+ ):
2803
+ redirect_method = method
2804
+ if info["status"] not in ("307", "308"):
2805
+ redirect_method = "GET"
2806
+ if "-x-permanent-redirect-url" in info:
2807
+ if redirections <= 0:
2808
+ raise RedirectLimit(
2809
+ "Redirected more times than redirection_limit allows.", {}, "",
2810
+ )
2811
+ (response, new_content) = self.request(
2812
+ info["-x-permanent-redirect-url"],
2813
+ method=redirect_method,
2814
+ headers=headers,
2815
+ redirections=redirections - 1,
2816
+ )
2817
+ response.previous = Response(info)
2818
+ response.previous.fromcache = True
2819
+ else:
2820
+ entry_disposition = _entry_disposition(info, headers)
2821
+
2822
+ if entry_disposition == "FRESH":
2823
+ response = Response(info)
2824
+ response.fromcache = True
2825
+ return (response, content)
2826
+
2827
+ if entry_disposition == "STALE":
2828
+ if "etag" in info and not self.ignore_etag and not "if-none-match" in headers:
2829
+ headers["if-none-match"] = info["etag"]
2830
+ if "last-modified" in info and not "last-modified" in headers:
2831
+ headers["if-modified-since"] = info["last-modified"]
2832
+ elif entry_disposition == "TRANSPARENT":
2833
+ pass
2834
+
2835
+ (response, new_content) = self._request(
2836
+ conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
2837
+ )
2838
+
2839
+ if response.status == 304 and method == "GET":
2840
+ for key in _get_end2end_headers(response):
2841
+ info[key] = response[key]
2842
+ merged_response = Response(info)
2843
+ if hasattr(response, "_stale_digest"):
2844
+ merged_response._stale_digest = response._stale_digest
2845
+ _updateCache(headers, merged_response, content, self.cache, cachekey)
2846
+ response = merged_response
2847
+ response.status = 200
2848
+ response.fromcache = True
2849
+
2850
+ elif response.status == 200:
2851
+ content = new_content
2852
+ else:
2853
+ self.cache.delete(cachekey)
2854
+ content = new_content
2855
+ else:
2856
+ cc = _parse_cache_control(headers)
2857
+ if "only-if-cached" in cc:
2858
+ info["status"] = "504"
2859
+ response = Response(info)
2860
+ content = b""
2861
+ else:
2862
+ (response, content) = self._request(
2863
+ conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
2864
+ )
2865
+ except Exception as e:
2866
+ is_timeout = isinstance(e, socket.timeout)
2867
+ if is_timeout:
2868
+ with _get_http_lock(self):
2869
+ conn = self.connections.pop(conn_key, None)
2870
+ if conn:
2871
+ conn.close()
2872
+
2873
+ if self.force_exception_to_status_code:
2874
+ if isinstance(e, HttpLib2ErrorWithResponse):
2875
+ response = e.response
2876
+ content = e.content
2877
+ response.status = 500
2878
+ response.reason = str(e)
2879
+ elif isinstance(e, socket.timeout):
2880
+ content = b"Request Timeout"
2881
+ response = Response({"content-type": "text/plain", "status": "408", "content-length": len(content),})
2882
+ response.reason = "Request Timeout"
2883
+ else:
2884
+ content = str(e).encode("utf-8")
2885
+ response = Response({"content-type": "text/plain", "status": "400", "content-length": len(content),})
2886
+ response.reason = "Bad Request"
2887
+ else:
2888
+ raise
2889
+
2890
+ return (response, content)
2891
+
2892
+
2893
+ class Response(dict):
2894
+ """An object more like email.message than httplib.HTTPResponse."""
2895
+
2896
+ fromcache = False
2897
+ version = 11
2898
+ status = 200
2899
+ reason = "Ok"
2900
+ previous = None
2901
+ stream_id = None
2902
+
2903
+ def __init__(self, info):
2904
+ # info is an email.message, an http.client.HTTPResponse,
2905
+ # a _Httplib2RawResponse, or a mapping.
2906
+ if isinstance(info, (http.client.HTTPResponse, _Httplib2RawResponse)):
2907
+ headers_iter = info.headers.items() if hasattr(info, "headers") else info.getheaders()
2908
+ for key, value in headers_iter:
2909
+ key = key.lower()
2910
+ prev = self.get(key)
2911
+ if prev is not None:
2912
+ value = ", ".join((prev, value))
2913
+ self[key] = value
2914
+ self.status = info.status
2915
+ self["status"] = str(self.status)
2916
+ self.reason = info.reason
2917
+ self.version = info.version
2918
+ self.stream_id = getattr(info, "stream_id", None)
2919
+ elif isinstance(info, email.message.Message):
2920
+ for key, value in list(info.items()):
2921
+ self[key.lower()] = value
2922
+ self.status = int(self["status"])
2923
+ else:
2924
+ for key, value in info.items():
2925
+ self[key.lower()] = value
2926
+ self.status = int(self.get("status", self.status))
2927
+
2928
+ def __getattr__(self, name):
2929
+ if name == "dict":
2930
+ return self
2931
+ else:
2932
+ raise AttributeError(name)
2933
+
2934
+
2935
+ def try_value_or_env(to, value, env_key, default=None):
2936
+ candidates = (value, os.environ.get(env_key), os.environ.get(env_key.upper()))
2937
+ for x in candidates:
2938
+ if x is None:
2939
+ continue
2940
+ try:
2941
+ return to(x)
2942
+ except ValueError:
2943
+ pass
2944
+ return default