python-proxy-headers 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

python_proxy_headers/aiohttp_proxy.py

@@ -6,6 +6,7 @@ from aiohttp.client import ClientSession
 from aiohttp.helpers import reify
 from aiohttp import hdrs
 from multidict import CIMultiDict, CIMultiDictProxy
+from .header_utils import merge_proxy_response_headers
 
 class ProxyTCPConnector(TCPConnector):
 	async def _create_proxy_connection(self, req: ClientRequest, traces, timeout):
@@ -143,13 +144,21 @@ class ProxyClientRequest(ClientRequest):
 		return resp
 
 class ProxyClientResponse(ClientResponse):
+	@reify
+	def proxy_headers(self):
+		"""Headers from the proxy CONNECT response, not merged into origin headers."""
+		raw = getattr(self, "_proxy_headers", None)
+		if raw:
+			return CIMultiDictProxy(CIMultiDict(raw))
+		return CIMultiDictProxy(CIMultiDict())
+
 	@reify
 	def headers(self):
 		proxy_headers = getattr(self, '_proxy_headers', None)
 
 		if proxy_headers:
 			headers = CIMultiDict(self._headers)
-			headers.extend(proxy_headers)
+			merge_proxy_response_headers(headers, proxy_headers)
 			return CIMultiDictProxy(headers)
 		else:
 			return self._headers

python_proxy_headers/autoscraper_proxy.py

@@ -35,6 +35,7 @@ except ImportError:
         "Install it with: pip install autoscraper"
     )
 
+from .header_utils import validate_headers
 from .requests_adapter import ProxySession
 
 
@@ -71,7 +72,7 @@ class ProxyAutoScraper(AutoScraper):
         stack_list: Optional[List] = None
     ):
         super().__init__(stack_list=stack_list)
-        self._proxy_headers = proxy_headers or {}
+        self._proxy_headers = validate_headers(proxy_headers)
         self._session: Optional[ProxySession] = None
     
     def _get_session(self) -> ProxySession:
@@ -90,7 +91,7 @@ class ProxyAutoScraper(AutoScraper):
         Args:
             proxy_headers: New proxy headers to use
         """
-        self._proxy_headers = proxy_headers
+        self._proxy_headers = validate_headers(proxy_headers)
         if self._session is not None:
             self._session.close()
             self._session = None

python_proxy_headers/cloudscraper_proxy.py

@@ -27,6 +27,7 @@ except ImportError:
         "Install it with: pip install cloudscraper"
     )
 
+from .header_utils import validate_headers
 from .urllib3_proxy_manager import proxy_from_url
 
 
@@ -41,8 +42,13 @@ class CipherSuiteProxyHeaderAdapter(CipherSuiteAdapter):
     """
     
     def __init__(self, proxy_headers: Optional[Dict[str, str]] = None, **kwargs):
-        self._proxy_headers = proxy_headers or {}
+        self._proxy_headers = validate_headers(proxy_headers)
         super().__init__(**kwargs)
+
+    def build_response(self, req, resp):
+        response = super().build_response(req, resp)
+        response.proxy_headers = getattr(resp, "proxy_headers", {}) or {}
+        return response
     
     def proxy_manager_for(self, proxy, **proxy_kwargs):
         """
@@ -103,7 +109,7 @@ class ProxyCloudScraper(cloudscraper.CloudScraper):
     """
     
     def __init__(self, proxy_headers: Optional[Dict[str, str]] = None, **kwargs):
-        self._proxy_headers = proxy_headers or {}
+        self._proxy_headers = validate_headers(proxy_headers)
         
         # Call parent init
         super().__init__(**kwargs)
@@ -142,7 +148,7 @@ class ProxyCloudScraper(cloudscraper.CloudScraper):
         Args:
             proxy_headers: New proxy headers to use
         """
-        self._proxy_headers = proxy_headers
+        self._proxy_headers = validate_headers(proxy_headers)
         
         # Remount adapters with new headers
         self.mount(

python_proxy_headers/header_utils.py

@@ -0,0 +1,203 @@
+"""Validation and safe merging for proxy CONNECT headers.
+
+CONNECT requests are raw HTTP and must not interpolate CR, LF, or NUL.
+CONNECT response headers are not origin HTTPS headers: hop-by-hop and
+security-sensitive names are not copied onto the origin response, and
+existing origin headers are never overwritten.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Sequence, Tuple, Union
+from urllib.parse import urlparse
+
+RawHeaders = Union[Mapping[Any, Any], Sequence[Tuple[Any, Any]], None]
+
+# Never copy these from a CONNECT response onto the origin response.
+BLOCKED_PROXY_RESPONSE_HEADERS = frozenset({
+	"age",
+	"authorization",
+	"cache-control",
+	"clear-site-data",
+	"connection",
+	"content-disposition",
+	"content-encoding",
+	"content-language",
+	"content-length",
+	"content-location",
+	"content-range",
+	"content-security-policy",
+	"content-security-policy-report-only",
+	"content-type",
+	"cookie",
+	"date",
+	"etag",
+	"expires",
+	"host",
+	"keep-alive",
+	"last-modified",
+	"link",
+	"location",
+	"pragma",
+	"proxy-agent",
+	"proxy-authenticate",
+	"proxy-authorization",
+	"proxy-connection",
+	"refresh",
+	"server",
+	"set-cookie",
+	"set-cookie2",
+	"strict-transport-security",
+	"te",
+	"trailer",
+	"transfer-encoding",
+	"upgrade",
+	"vary",
+	"via",
+	"warning",
+	"www-authenticate",
+	"x-content-type-options",
+	"x-frame-options",
+	"x-xss-protection",
+})
+
+
+def _as_str(value: Any) -> str:
+	if isinstance(value, bytes):
+		return value.decode("latin-1")
+	return str(value)
+
+
+def validate_header_name(name: Any) -> str:
+	"""Return a header name, or raise ValueError if it is not a single token."""
+	name_s = _as_str(name)
+	if not name_s:
+		raise ValueError("Header name must not be empty")
+	for char in name_s:
+		code = ord(char)
+		if char in "()<>@,;:\\\"/[]?={} \t:" or code <= 32 or code == 127:
+			raise ValueError(f"Invalid header name {name_s!r}")
+	return name_s
+
+
+def validate_header_value(value: Any) -> str:
+	"""Return a header value, or raise ValueError if it contains CR, LF, or NUL."""
+	value_s = _as_str(value)
+	if any(char in value_s for char in "\r\n\x00"):
+		raise ValueError(f"Invalid header value {value_s!r}")
+	return value_s
+
+
+def validate_headers(headers: Optional[Mapping[Any, Any]]) -> Dict[str, str]:
+	"""Validate a mapping of HTTP header names and values.
+
+	Raises:
+		ValueError: If any name or value contains CR, LF, NUL, or other
+			characters illegal in a single header line.
+	"""
+	if not headers:
+		return {}
+	validated: Dict[str, str] = {}
+	for name, value in headers.items():
+		validated[validate_header_name(name)] = validate_header_value(value)
+	return validated
+
+
+def validate_http_method(method: Any) -> str:
+	"""Return an HTTP method token, or raise ValueError if it is unsafe.
+
+	Methods must be a single token (no spaces, slashes, or control characters)
+	so they cannot smuggle extra request lines through CURLOPT_CUSTOMREQUEST.
+	"""
+	method_s = _as_str(method).upper()
+	if not method_s:
+		raise ValueError("HTTP method must not be empty")
+	for char in method_s:
+		code = ord(char)
+		if char in "()<>@,;:\\\"/[]?={} \t" or code <= 32 or code == 127:
+			raise ValueError(f"Invalid HTTP method {method_s!r}")
+	return method_s
+
+
+def validate_http_url(url: Any) -> str:
+	"""Return an http(s) URL, or raise ValueError for other schemes.
+
+	High-level HTTP helpers must not pass caller URLs to protocol-agile
+	backends such as libcurl, which otherwise honor ``file://`` and similar
+	schemes (and ignore an HTTP proxy for those schemes).
+	"""
+	url_s = _as_str(url)
+	if not url_s or any(char in url_s for char in "\r\n\x00"):
+		raise ValueError(f"Invalid URL {url_s!r}")
+	parsed = urlparse(url_s)
+	scheme = parsed.scheme.lower()
+	if scheme not in ("http", "https"):
+		raise ValueError(
+			f"Unsupported URL scheme {parsed.scheme!r}; only http and https are allowed"
+		)
+	if not parsed.netloc:
+		raise ValueError(f"URL must include a host: {url_s!r}")
+	return url_s
+
+
+def _header_items(headers: RawHeaders) -> List[Tuple[Any, Any]]:
+	if not headers:
+		return []
+	if isinstance(headers, (list, tuple)):
+		return list(headers)
+	if hasattr(headers, "items"):
+		return list(headers.items())
+	return []
+
+
+def origin_has_header(origin_headers: RawHeaders, name: str) -> bool:
+	"""Return True if origin_headers already contains ``name`` (case-insensitive)."""
+	lowered = _as_str(name).lower()
+	for key, _value in _header_items(origin_headers):
+		if _as_str(key).lower() == lowered:
+			return True
+	return False
+
+
+def is_mergeable_proxy_header(name: Any) -> bool:
+	"""Return True if a CONNECT response header may be copied onto origin headers."""
+	lowered = _as_str(name).lower()
+	if lowered in BLOCKED_PROXY_RESPONSE_HEADERS:
+		return False
+	if lowered.startswith("access-control-"):
+		return False
+	return True
+
+
+def snapshot_headers(headers: RawHeaders) -> Dict[str, str]:
+	"""Copy headers into a plain dict of strings."""
+	snapshot: Dict[str, str] = {}
+	for name, value in _header_items(headers):
+		snapshot[_as_str(name)] = _as_str(value)
+	return snapshot
+
+
+def merge_proxy_response_headers(
+	origin_headers: MutableMapping[Any, Any],
+	proxy_headers: RawHeaders,
+) -> None:
+	"""Copy safe CONNECT headers onto origin headers without overwriting."""
+	for name, value in _header_items(proxy_headers):
+		name_s = _as_str(name)
+		if not is_mergeable_proxy_header(name_s):
+			continue
+		if origin_has_header(origin_headers, name_s):
+			continue
+		origin_headers[name_s] = _as_str(value)
+
+
+def filter_connect_headers(
+	origin_headers: RawHeaders,
+	connect_headers: RawHeaders,
+) -> List[Tuple[Any, Any]]:
+	"""Return CONNECT header pairs that are safe to merge into origin headers."""
+	extra: List[Tuple[Any, Any]] = []
+	for name, value in _header_items(connect_headers):
+		if is_mergeable_proxy_header(name) and not origin_has_header(origin_headers, name):
+			extra.append((name, value))
+	return extra

python_proxy_headers/httpx_proxy.py

@@ -9,6 +9,7 @@ from httpcore._ssl import default_ssl_context
 from httpcore._trace import Trace
 from httpx import AsyncHTTPTransport, HTTPTransport, Client
 from httpx._config import DEFAULT_LIMITS, DEFAULT_TIMEOUT_CONFIG, Proxy, create_ssl_context
+from .header_utils import filter_connect_headers
 
 class ProxyTunnelHTTPConnection(TunnelHTTPConnection):
 	# Unfortunately the only way to get connect_response.headers into the Response
@@ -90,10 +91,16 @@ class ProxyTunnelHTTPConnection(TunnelHTTPConnection):
 						keepalive_expiry=self._keepalive_expiry,
 					)
 
+				self._connect_response_headers = connect_response.headers
 				self._connected = True
-		# this is the only modification
+		# Merge safe CONNECT headers without overwriting origin values
 		response = self._connection.handle_request(request)
-		response.headers = merge_headers(response.headers, connect_response.headers)
+		response.headers = merge_headers(
+			response.headers,
+			filter_connect_headers(
+				response.headers, getattr(self, "_connect_response_headers", ())
+			),
+		)
 		return response
 
 class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection):
@@ -174,10 +181,16 @@ class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection):
 						keepalive_expiry=self._keepalive_expiry,
 					)
 
+				self._connect_response_headers = connect_response.headers
 				self._connected = True
-		# this is the only modification
+		# Merge safe CONNECT headers without overwriting origin values
 		response = await self._connection.handle_async_request(request)
-		response.headers = merge_headers(response.headers, connect_response.headers)
+		response.headers = merge_headers(
+			response.headers,
+			filter_connect_headers(
+				response.headers, getattr(self, "_connect_response_headers", ())
+			),
+		)
 		return response
 
 class HTTPProxyHeaders(HTTPProxy):

python_proxy_headers/pycurl_proxy.py

@@ -38,6 +38,8 @@ from io import BytesIO
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Tuple
 
+from .header_utils import validate_headers, validate_http_method, validate_http_url
+
 try:
     import pycurl
 except ImportError:
@@ -67,7 +69,8 @@ def set_proxy_headers(curl, headers: Dict[str, str]) -> None:
     """
     if not headers:
         return
-    
+
+    headers = validate_headers(headers)
     header_list = [f"{k}: {v}" for k, v in headers.items()]
     
     # Set CURLOPT_PROXYHEADER
@@ -260,6 +263,18 @@ class Response:
             raise Exception(f"HTTP Error {self.status_code}")
 
 
+def _restrict_http_protocols(curl) -> None:
+    """Limit libcurl to HTTP(S) so file:// and similar schemes cannot run."""
+    http_https = pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS
+    curl.setopt(pycurl.PROTOCOLS, http_https)
+    redir_protocols = getattr(pycurl, "REDIR_PROTOCOLS", None)
+    if redir_protocols is not None:
+        try:
+            curl.setopt(redir_protocols, http_https)
+        except pycurl.error:
+            pass
+
+
 def request(
     method: str,
     url: str,
@@ -275,7 +290,7 @@ def request(
     
     Args:
         method: HTTP method (GET, POST, etc.)
-        url: Target URL
+        url: Target URL (http and https only)
         proxy: Proxy URL (e.g., 'http://user:pass@proxy:8080')
         proxy_headers: Headers to send to the proxy
         headers: Headers to send to the origin server
@@ -285,17 +300,25 @@ def request(
         
     Returns:
         Response object with body, headers, and proxy_headers
+
+    Raises:
+        ValueError: If the URL scheme is not http/https, the method is not a
+            single token, or a header name/value contains CR, LF, or NUL.
     """
+    url = validate_http_url(url)
+    method = validate_http_method(method)
+    headers = validate_headers(headers)
+
     c = pycurl.Curl()
     body = BytesIO()
     capture = HeaderCapture(c)
     
     try:
+        _restrict_http_protocols(c)
         c.setopt(pycurl.URL, url)
         c.setopt(pycurl.WRITEFUNCTION, body.write)
         
         # HTTP method
-        method = method.upper()
         if method == 'GET':
             c.setopt(pycurl.HTTPGET, 1)
         elif method == 'POST':

python_proxy_headers/requests_adapter.py

@@ -1,11 +1,17 @@
 from requests.adapters import HTTPAdapter
 from requests.sessions import Session
+from .header_utils import validate_headers
 from .urllib3_proxy_manager import proxy_from_url
 
 class HTTPProxyHeaderAdapter(HTTPAdapter):
 	def __init__(self, proxy_headers=None):
 		super().__init__()
-		self._proxy_headers = proxy_headers or {}
+		self._proxy_headers = validate_headers(proxy_headers)
+
+	def build_response(self, req, resp):
+		response = super().build_response(req, resp)
+		response.proxy_headers = getattr(resp, "proxy_headers", {}) or {}
+		return response
 	
 	def proxy_manager_for(self, proxy, **proxy_kwargs):
 		"""Return urllib3 ProxyManager for the given proxy.

python_proxy_headers/urllib3_proxy_manager.py

@@ -6,6 +6,14 @@ from urllib3.poolmanager import ProxyManager
 from urllib3.util.request import make_headers
 from urllib3.util.url import parse_url
 
+from .header_utils import (
+	merge_proxy_response_headers,
+	snapshot_headers,
+	validate_header_name,
+	validate_header_value,
+	validate_headers,
+)
+
 if sys.version_info < (3, 12, 0):
 	#####################################
 	### copied from python3.12 source ###
@@ -77,6 +85,8 @@ class HTTPSProxyConnection(HTTPSConnection):
 				self._http_vsn_str.encode("ascii"))
 			headers = [connect]
 			for header, value in self._tunnel_headers.items():
+				validate_header_name(header)
+				validate_header_value(value)
 				headers.append(f"{header}: {value}\r\n".encode("latin-1"))
 			headers.append(b"\r\n")
 			# Making a single send() call instead of one per line encourages
@@ -125,7 +135,9 @@ class HTTPSProxyConnectionPool(HTTPSConnectionPool):
 	
 	def urlopen(self, *args, **kwargs):
 		response = super().urlopen(*args, **kwargs)
-		response.headers.update(self._proxy_response_headers)
+		proxy_headers = getattr(self, "_proxy_response_headers", None)
+		response.proxy_headers = snapshot_headers(proxy_headers)
+		merge_proxy_response_headers(response.headers, proxy_headers)
 		return response
 
 class ProxyHeaderManager(ProxyManager):
@@ -137,15 +149,14 @@ class ProxyHeaderManager(ProxyManager):
 		if proxy_url is None and args:
 			proxy_url = args[0]
 		proxy_headers = kwargs.get("proxy_headers")
-		merged = dict(proxy_headers or {})
+		merged = validate_headers(dict(proxy_headers or {}))
 		if isinstance(proxy_url, str):
 			parsed = parse_url(proxy_url)
 			if parsed.auth and not any(
 				k.lower() == "proxy-authorization" for k in merged
 			):
 				merged.update(make_headers(proxy_basic_auth=parsed.auth))
-		if merged != dict(proxy_headers or {}):
-			kwargs["proxy_headers"] = merged
+		kwargs["proxy_headers"] = merged
 		super().__init__(*args, **kwargs)
 		self.pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSProxyConnectionPool}
 

{python_proxy_headers-0.2.2.dist-info → python_proxy_headers-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-proxy-headers
-Version: 0.2.2
+Version: 0.2.4
 Summary: Handle custom proxy headers for http & https requests in various python libraries
 Author-email: ProxyMesh <support@proxymesh.com>
 Project-URL: Homepage, https://github.com/proxymesh/python-proxy-headers

python_proxy_headers-0.2.4.dist-info/RECORD

@@ -0,0 +1,14 @@
+python_proxy_headers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+python_proxy_headers/aiohttp_proxy.py,sha256=c6-MS6ldiLS5rxPmunrk9Z8-BkXryJOe6HNxbi3amo8,5256
+python_proxy_headers/autoscraper_proxy.py,sha256=wQoTE2akpbY2nXZWQvxmP5Ik2k8vSVd6ErYlz24YWCM,11415
+python_proxy_headers/cloudscraper_proxy.py,sha256=kdzqPHX_9Bf05EeV9jalbA5uqGEKdKXanyBV0y-N85E,8058
+python_proxy_headers/header_utils.py,sha256=llAy606dwqEES4N60SLzSlB3ijdpcJVm7fnoNwdLtVY,5880
+python_proxy_headers/httpx_proxy.py,sha256=HxaKs-MWbxS6Jhxp0356OUvkT5Vun4Bab6G-NmfdhC0,12232
+python_proxy_headers/pycurl_proxy.py,sha256=Ogan8WT4cC5odifLbPXfct1twGy0JYbSmEkoRGwk9nM,12602
+python_proxy_headers/requests_adapter.py,sha256=ln3cXaWCHHMgL4uNlNrp9MDsocsFL68FuZ_Kyuj4U8U,2540
+python_proxy_headers/urllib3_proxy_manager.py,sha256=Yi_OlG5eZtQSPhj8-Tw7Y5hKmwCj8pApZn6fJARoOD8,5516
+python_proxy_headers-0.2.4.dist-info/licenses/LICENSE,sha256=i_H6fvudjqZOKkZFCIHGmbPHIxszjCAT90UJSY2OM0U,1066
+python_proxy_headers-0.2.4.dist-info/METADATA,sha256=T_Sy6rc2N8MsSIJXAAgYhb30n0utvdhlTEsxslW48r8,6288
+python_proxy_headers-0.2.4.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+python_proxy_headers-0.2.4.dist-info/top_level.txt,sha256=Bhm-Qc6vn0DAd2Li8ZBn6KLkls2zJnM4-CtXHvIiRh8,21
+python_proxy_headers-0.2.4.dist-info/RECORD,,

{python_proxy_headers-0.2.2.dist-info → python_proxy_headers-0.2.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

python_proxy_headers-0.2.2.dist-info/RECORD

@@ -1,13 +0,0 @@
-python_proxy_headers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-python_proxy_headers/aiohttp_proxy.py,sha256=OyEruj3CCrFmlEikmmOXGQsVJnrAfK-TMzycP-sLH_Y,4920
-python_proxy_headers/autoscraper_proxy.py,sha256=g51K71xOah_uNtMIXEXggtd6T8ol7vAukg7oaWVvhEA,11342
-python_proxy_headers/cloudscraper_proxy.py,sha256=6xF58QywHV4IYlC_KtXIWQTXZjE8CeEa5-GfqcSGZ3U,7780
-python_proxy_headers/httpx_proxy.py,sha256=Q8tDlfl4u3fjsZZDA1zj1pbN7fiAwJ3ctFGojbqtaEo,11829
-python_proxy_headers/pycurl_proxy.py,sha256=_JAt_o5gX7o0sRLOQXG3421mEehzBm-rhUl4c9IG5HQ,11738
-python_proxy_headers/requests_adapter.py,sha256=CfGEEYc0eaKKNA11VbhZzv9qFVEL1uMSzAUNvidIyEU,2313
-python_proxy_headers/urllib3_proxy_manager.py,sha256=ryT7QaCPb7lyQtHaP3v6MKfBerurBYrdRj4i-41ORT4,5196
-python_proxy_headers-0.2.2.dist-info/licenses/LICENSE,sha256=i_H6fvudjqZOKkZFCIHGmbPHIxszjCAT90UJSY2OM0U,1066
-python_proxy_headers-0.2.2.dist-info/METADATA,sha256=ltXQg143IYn7qLMgAjQihF4UJCUNurgBjSJfZPWcD1U,6288
-python_proxy_headers-0.2.2.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-python_proxy_headers-0.2.2.dist-info/top_level.txt,sha256=Bhm-Qc6vn0DAd2Li8ZBn6KLkls2zJnM4-CtXHvIiRh8,21
-python_proxy_headers-0.2.2.dist-info/RECORD,,