pip 25.1.1__py3-none-any.whl → 25.3__py3-none-any.whl

pip/_internal/network/download.py

@@ -1,35 +1,42 @@
 """Download files with progress indicators."""
 
+from __future__ import annotations
+
 import email.message
 import logging
 import mimetypes
 import os
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
 from http import HTTPStatus
-from typing import BinaryIO, Iterable, Optional, Tuple
+from typing import BinaryIO
 
+from pip._vendor.requests import PreparedRequest
 from pip._vendor.requests.models import Response
+from pip._vendor.urllib3 import HTTPResponse as URLlib3Response
+from pip._vendor.urllib3._collections import HTTPHeaderDict
 from pip._vendor.urllib3.exceptions import ReadTimeoutError
 
-from pip._internal.cli.progress_bars import get_download_progress_renderer
+from pip._internal.cli.progress_bars import BarType, get_download_progress_renderer
 from pip._internal.exceptions import IncompleteDownloadError, NetworkConnectionError
 from pip._internal.models.index import PyPI
 from pip._internal.models.link import Link
-from pip._internal.network.cache import is_from_cache
-from pip._internal.network.session import PipSession
+from pip._internal.network.cache import SafeFileCache, is_from_cache
+from pip._internal.network.session import CacheControlAdapter, PipSession
 from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
 from pip._internal.utils.misc import format_size, redact_auth_from_url, splitext
 
 logger = logging.getLogger(__name__)
 
 
-def _get_http_response_size(resp: Response) -> Optional[int]:
+def _get_http_response_size(resp: Response) -> int | None:
     try:
         return int(resp.headers["content-length"])
     except (ValueError, KeyError, TypeError):
         return None
 
 
-def _get_http_response_etag_or_last_modified(resp: Response) -> Optional[str]:
+def _get_http_response_etag_or_last_modified(resp: Response) -> str | None:
     """
     Return either the ETag or Last-Modified header (or None if neither exists).
     The return value can be used in an If-Range header.
@@ -37,12 +44,12 @@ def _get_http_response_etag_or_last_modified(resp: Response) -> Optional[str]:
     return resp.headers.get("etag", resp.headers.get("last-modified"))
 
 
-def _prepare_download(
+def _log_download(
     resp: Response,
     link: Link,
-    progress_bar: str,
-    total_length: Optional[int],
-    range_start: Optional[int] = 0,
+    progress_bar: BarType,
+    total_length: int | None,
+    range_start: int | None = 0,
 ) -> Iterable[bytes]:
     if link.netloc == PyPI.file_storage_domain:
         url = link.show_url
@@ -119,7 +126,7 @@ def _get_http_response_filename(resp: Response, link: Link) -> str:
     content_disposition = resp.headers.get("content-disposition")
     if content_disposition:
         filename = parse_content_disposition(content_disposition, filename)
-    ext: Optional[str] = splitext(filename)[1]
+    ext: str | None = splitext(filename)[1]
     if not ext:
         ext = mimetypes.guess_extension(resp.headers.get("content-type", ""))
         if ext:
@@ -131,35 +138,35 @@ def _get_http_response_filename(resp: Response, link: Link) -> str:
     return filename
 
 
-def _http_get_download(
-    session: PipSession,
-    link: Link,
-    range_start: Optional[int] = 0,
-    if_range: Optional[str] = None,
-) -> Response:
-    target_url = link.url.split("#", 1)[0]
-    headers = HEADERS.copy()
-    # request a partial download
-    if range_start:
-        headers["Range"] = f"bytes={range_start}-"
-    # make sure the file hasn't changed
-    if if_range:
-        headers["If-Range"] = if_range
-    try:
-        resp = session.get(target_url, headers=headers, stream=True)
-        raise_for_status(resp)
-    except NetworkConnectionError as e:
-        assert e.response is not None
-        logger.critical("HTTP error %s while getting %s", e.response.status_code, link)
-        raise
-    return resp
+@dataclass
+class _FileDownload:
+    """Stores the state of a single link download."""
+
+    link: Link
+    output_file: BinaryIO
+    size: int | None
+    bytes_received: int = 0
+    reattempts: int = 0
+
+    def is_incomplete(self) -> bool:
+        return bool(self.size is not None and self.bytes_received < self.size)
+
+    def write_chunk(self, data: bytes) -> None:
+        self.bytes_received += len(data)
+        self.output_file.write(data)
+
+    def reset_file(self) -> None:
+        """Delete any saved data and reset progress to zero."""
+        self.output_file.seek(0)
+        self.output_file.truncate()
+        self.bytes_received = 0
 
 
 class Downloader:
     def __init__(
         self,
         session: PipSession,
-        progress_bar: str,
+        progress_bar: BarType,
         resume_retries: int,
     ) -> None:
         assert (
@@ -169,146 +176,167 @@ class Downloader:
         self._progress_bar = progress_bar
         self._resume_retries = resume_retries
 
-    def __call__(self, link: Link, location: str) -> Tuple[str, str]:
-        """Download the file given by link into location."""
-        resp = _http_get_download(self._session, link)
-        # NOTE: The original download size needs to be passed down everywhere
-        # so if the download is resumed (with a HTTP Range request) the progress
-        # bar will report the right size.
-        total_length = _get_http_response_size(resp)
-        content_type = resp.headers.get("Content-Type", "")
+    def batch(
+        self, links: Iterable[Link], location: str
+    ) -> Iterable[tuple[Link, tuple[str, str]]]:
+        """Convenience method to download multiple links."""
+        for link in links:
+            filepath, content_type = self(link, location)
+            yield link, (filepath, content_type)
 
-        filename = _get_http_response_filename(resp, link)
-        filepath = os.path.join(location, filename)
+    def __call__(self, link: Link, location: str) -> tuple[str, str]:
+        """Download a link and save it under location."""
+        resp = self._http_get(link)
+        download_size = _get_http_response_size(resp)
 
+        filepath = os.path.join(location, _get_http_response_filename(resp, link))
         with open(filepath, "wb") as content_file:
-            bytes_received = self._process_response(
-                resp, link, content_file, 0, total_length
-            )
-            # If possible, check for an incomplete download and attempt resuming.
-            if total_length and bytes_received < total_length:
-                self._attempt_resume(
-                    resp, link, content_file, total_length, bytes_received
-                )
+            download = _FileDownload(link, content_file, download_size)
+            self._process_response(download, resp)
+            if download.is_incomplete():
+                self._attempt_resumes_or_redownloads(download, resp)
 
+        content_type = resp.headers.get("Content-Type", "")
         return filepath, content_type
 
-    def _process_response(
-        self,
-        resp: Response,
-        link: Link,
-        content_file: BinaryIO,
-        bytes_received: int,
-        total_length: Optional[int],
-    ) -> int:
-        """Process the response and write the chunks to the file."""
-        chunks = _prepare_download(
-            resp, link, self._progress_bar, total_length, range_start=bytes_received
-        )
-        return self._write_chunks_to_file(
-            chunks, content_file, allow_partial=bool(total_length)
+    def _process_response(self, download: _FileDownload, resp: Response) -> None:
+        """Download and save chunks from a response."""
+        chunks = _log_download(
+            resp,
+            download.link,
+            self._progress_bar,
+            download.size,
+            range_start=download.bytes_received,
         )
-
-    def _write_chunks_to_file(
-        self, chunks: Iterable[bytes], content_file: BinaryIO, *, allow_partial: bool
-    ) -> int:
-        """Write the chunks to the file and return the number of bytes received."""
-        bytes_received = 0
         try:
             for chunk in chunks:
-                bytes_received += len(chunk)
-                content_file.write(chunk)
+                download.write_chunk(chunk)
         except ReadTimeoutError as e:
-            # If partial downloads are OK (the download will be retried), don't bail.
-            if not allow_partial:
+            # If the download size is not known, then give up downloading the file.
+            if download.size is None:
                 raise e
 
-            # Ensuring bytes_received is returned to attempt resume
             logger.warning("Connection timed out while downloading.")
 
-        return bytes_received
-
-    def _attempt_resume(
-        self,
-        resp: Response,
-        link: Link,
-        content_file: BinaryIO,
-        total_length: Optional[int],
-        bytes_received: int,
+    def _attempt_resumes_or_redownloads(
+        self, download: _FileDownload, first_resp: Response
     ) -> None:
-        """Attempt to resume the download if connection was dropped."""
-        etag_or_last_modified = _get_http_response_etag_or_last_modified(resp)
-
-        attempts_left = self._resume_retries
-        while total_length and attempts_left and bytes_received < total_length:
-            attempts_left -= 1
+        """Attempt to resume/restart the download if connection was dropped."""
 
+        while download.reattempts < self._resume_retries and download.is_incomplete():
+            assert download.size is not None
+            download.reattempts += 1
             logger.warning(
                 "Attempting to resume incomplete download (%s/%s, attempt %d)",
-                format_size(bytes_received),
-                format_size(total_length),
-                (self._resume_retries - attempts_left),
+                format_size(download.bytes_received),
+                format_size(download.size),
+                download.reattempts,
             )
 
             try:
-                # Try to resume the download using a HTTP range request.
-                resume_resp = _http_get_download(
-                    self._session,
-                    link,
-                    range_start=bytes_received,
-                    if_range=etag_or_last_modified,
-                )
-
+                resume_resp = self._http_get_resume(download, should_match=first_resp)
                 # Fallback: if the server responded with 200 (i.e., the file has
                 # since been modified or range requests are unsupported) or any
                 # other unexpected status, restart the download from the beginning.
                 must_restart = resume_resp.status_code != HTTPStatus.PARTIAL_CONTENT
                 if must_restart:
-                    bytes_received, total_length, etag_or_last_modified = (
-                        self._reset_download_state(resume_resp, content_file)
-                    )
+                    download.reset_file()
+                    download.size = _get_http_response_size(resume_resp)
+                    first_resp = resume_resp
 
-                bytes_received += self._process_response(
-                    resume_resp, link, content_file, bytes_received, total_length
-                )
+                self._process_response(download, resume_resp)
             except (ConnectionError, ReadTimeoutError, OSError):
                 continue
 
         # No more resume attempts. Raise an error if the download is still incomplete.
-        if total_length and bytes_received < total_length:
-            os.remove(content_file.name)
-            raise IncompleteDownloadError(
-                link, bytes_received, total_length, retries=self._resume_retries
-            )
+        if download.is_incomplete():
+            os.remove(download.output_file.name)
+            raise IncompleteDownloadError(download)
 
-    def _reset_download_state(
-        self,
-        resp: Response,
-        content_file: BinaryIO,
-    ) -> Tuple[int, Optional[int], Optional[str]]:
-        """Reset the download state to restart downloading from the beginning."""
-        content_file.seek(0)
-        content_file.truncate()
-        bytes_received = 0
-        total_length = _get_http_response_size(resp)
-        etag_or_last_modified = _get_http_response_etag_or_last_modified(resp)
+        # If we successfully completed the download via resume, manually cache it
+        # as a complete response to enable future caching
+        if download.reattempts > 0:
+            self._cache_resumed_download(download, first_resp)
 
-        return bytes_received, total_length, etag_or_last_modified
+    def _cache_resumed_download(
+        self, download: _FileDownload, original_response: Response
+    ) -> None:
+        """
+        Manually cache a file that was successfully downloaded via resume retries.
+
+        cachecontrol doesn't cache 206 (Partial Content) responses, since they
+        are not complete files. This method manually adds the final file to the
+        cache as though it was downloaded in a single request, so that future
+        requests can use the cache.
+        """
+        url = download.link.url_without_fragment
+        adapter = self._session.get_adapter(url)
+
+        # Check if the adapter is the CacheControlAdapter (i.e. caching is enabled)
+        if not isinstance(adapter, CacheControlAdapter):
+            logger.debug(
+                "Skipping resume download caching: no cache controller for %s", url
+            )
+            return
+
+        # Check SafeFileCache is being used
+        assert isinstance(
+            adapter.cache, SafeFileCache
+        ), "separate body cache not in use!"
+
+        synthetic_request = PreparedRequest()
+        synthetic_request.prepare(method="GET", url=url, headers={})
+
+        synthetic_response_headers = HTTPHeaderDict()
+        for key, value in original_response.headers.items():
+            if key.lower() not in ["content-range", "content-length"]:
+                synthetic_response_headers[key] = value
+        synthetic_response_headers["content-length"] = str(download.size)
+
+        synthetic_response = URLlib3Response(
+            body="",
+            headers=synthetic_response_headers,
+            status=200,
+            preload_content=False,
+        )
 
+        # Save metadata and then stream the file contents to cache.
+        cache_url = adapter.controller.cache_url(url)
+        metadata_blob = adapter.controller.serializer.dumps(
+            synthetic_request, synthetic_response, b""
+        )
+        adapter.cache.set(cache_url, metadata_blob)
+        download.output_file.flush()
+        with open(download.output_file.name, "rb") as f:
+            adapter.cache.set_body_from_io(cache_url, f)
 
-class BatchDownloader:
-    def __init__(
-        self,
-        session: PipSession,
-        progress_bar: str,
-        resume_retries: int,
-    ) -> None:
-        self._downloader = Downloader(session, progress_bar, resume_retries)
+        logger.debug(
+            "Cached resumed download as complete response for future use: %s", url
+        )
 
-    def __call__(
-        self, links: Iterable[Link], location: str
-    ) -> Iterable[Tuple[Link, Tuple[str, str]]]:
-        """Download the files given by links into location."""
-        for link in links:
-            filepath, content_type = self._downloader(link, location)
-            yield link, (filepath, content_type)
+    def _http_get_resume(
+        self, download: _FileDownload, should_match: Response
+    ) -> Response:
+        """Issue a HTTP range request to resume the download."""
+        # To better understand the download resumption logic, see the mdn web docs:
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Range_requests
+        headers = HEADERS.copy()
+        headers["Range"] = f"bytes={download.bytes_received}-"
+        # If possible, use a conditional range request to avoid corrupted
+        # downloads caused by the remote file changing in-between.
+        if identifier := _get_http_response_etag_or_last_modified(should_match):
+            headers["If-Range"] = identifier
+        return self._http_get(download.link, headers)
+
+    def _http_get(self, link: Link, headers: Mapping[str, str] = HEADERS) -> Response:
+        target_url = link.url_without_fragment
+        try:
+            resp = self._session.get(target_url, headers=headers, stream=True)
+            raise_for_status(resp)
+        except NetworkConnectionError as e:
+            assert e.response is not None
+            logger.critical(
+                "HTTP error %s while getting %s", e.response.status_code, link
+            )
+            raise
+        return resp

pip/_internal/network/lazy_wheel.py

@@ -1,14 +1,17 @@
 """Lazy ZIP over HTTP"""
 
+from __future__ import annotations
+
 __all__ = ["HTTPRangeRequestUnsupported", "dist_from_wheel_url"]
 
 from bisect import bisect_left, bisect_right
+from collections.abc import Generator
 from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
-from typing import Any, Dict, Generator, List, Optional, Tuple
+from typing import Any
 from zipfile import BadZipFile, ZipFile
 
-from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.utils import NormalizedName
 from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
 
 from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution
@@ -20,7 +23,9 @@ class HTTPRangeRequestUnsupported(Exception):
     pass
 
 
-def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistribution:
+def dist_from_wheel_url(
+    name: NormalizedName, url: str, session: PipSession
+) -> BaseDistribution:
     """Return a distribution object from the given wheel URL.
 
     This uses HTTP range requests to only fetch the portion of the wheel
@@ -34,7 +39,7 @@ def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistrib
         wheel = MemoryWheel(zf.name, zf)  # type: ignore
         # After context manager exit, wheel.name
         # is an invalid file by intention.
-        return get_wheel_distribution(wheel, canonicalize_name(name))
+        return get_wheel_distribution(wheel, name)
 
 
 class LazyZipOverHTTP:
@@ -56,8 +61,8 @@ class LazyZipOverHTTP:
         self._length = int(head.headers["Content-Length"])
         self._file = NamedTemporaryFile()
         self.truncate(self._length)
-        self._left: List[int] = []
-        self._right: List[int] = []
+        self._left: list[int] = []
+        self._right: list[int] = []
         if "bytes" not in head.headers.get("Accept-Ranges", "none"):
             raise HTTPRangeRequestUnsupported("range request is not supported")
         self._check_zip()
@@ -117,7 +122,7 @@ class LazyZipOverHTTP:
         """Return the current position."""
         return self._file.tell()
 
-    def truncate(self, size: Optional[int] = None) -> int:
+    def truncate(self, size: int | None = None) -> int:
         """Resize the stream to the given size in bytes.
 
         If size is unspecified resize to the current position.
@@ -131,7 +136,7 @@ class LazyZipOverHTTP:
         """Return False."""
         return False
 
-    def __enter__(self) -> "LazyZipOverHTTP":
+    def __enter__(self) -> LazyZipOverHTTP:
         self._file.__enter__()
         return self
 
@@ -166,7 +171,7 @@ class LazyZipOverHTTP:
                     break
 
     def _stream_response(
-        self, start: int, end: int, base_headers: Dict[str, str] = HEADERS
+        self, start: int, end: int, base_headers: dict[str, str] = HEADERS
     ) -> Response:
         """Return HTTP response to a range request from start to end."""
         headers = base_headers.copy()
@@ -177,7 +182,7 @@ class LazyZipOverHTTP:
 
     def _merge(
         self, start: int, end: int, left: int, right: int
-    ) -> Generator[Tuple[int, int], None, None]:
+    ) -> Generator[tuple[int, int], None, None]:
         """Return a generator of intervals to be fetched.
 
         Args:

pip/_internal/network/session.py

@@ -2,6 +2,8 @@
 network request configuration and behavior.
 """
 
+from __future__ import annotations
+
 import email.utils
 import functools
 import io
@@ -16,16 +18,11 @@ import subprocess
 import sys
 import urllib.parse
 import warnings
+from collections.abc import Generator, Mapping, Sequence
 from typing import (
     TYPE_CHECKING,
     Any,
-    Dict,
-    Generator,
-    List,
-    Mapping,
     Optional,
-    Sequence,
-    Tuple,
     Union,
 )
 
@@ -54,18 +51,19 @@ if TYPE_CHECKING:
     from ssl import SSLContext
 
     from pip._vendor.urllib3.poolmanager import PoolManager
+    from pip._vendor.urllib3.proxymanager import ProxyManager
 
 
 logger = logging.getLogger(__name__)
 
-SecureOrigin = Tuple[str, str, Optional[Union[int, str]]]
+SecureOrigin = tuple[str, str, Optional[Union[int, str]]]
 
 
 # Ignore warning raised when using --trusted-host.
 warnings.filterwarnings("ignore", category=InsecureRequestWarning)
 
 
-SECURE_ORIGINS: List[SecureOrigin] = [
+SECURE_ORIGINS: list[SecureOrigin] = [
     # protocol, hostname, port
     # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
     ("https", "*", "*"),
@@ -112,7 +110,7 @@ def user_agent() -> str:
     """
     Return a string representing the user agent.
     """
-    data: Dict[str, Any] = {
+    data: dict[str, Any] = {
         "installer": {"name": "pip", "version": __version__},
         "python": platform.python_version(),
         "implementation": {
@@ -140,7 +138,7 @@ def user_agent() -> str:
         from pip._vendor import distro
 
         linux_distribution = distro.name(), distro.version(), distro.codename()
-        distro_infos: Dict[str, Any] = dict(
+        distro_infos: dict[str, Any] = dict(
             filter(
                 lambda x: x[1],
                 zip(["name", "version", "id"], linux_distribution),
@@ -214,10 +212,10 @@ class LocalFSAdapter(BaseAdapter):
         self,
         request: PreparedRequest,
         stream: bool = False,
-        timeout: Optional[Union[float, Tuple[float, float]]] = None,
-        verify: Union[bool, str] = True,
-        cert: Optional[Union[str, Tuple[str, str]]] = None,
-        proxies: Optional[Mapping[str, str]] = None,
+        timeout: float | tuple[float, float] | None = None,
+        verify: bool | str = True,
+        cert: str | tuple[str, str] | None = None,
+        proxies: Mapping[str, str] | None = None,
     ) -> Response:
         pathname = url_to_path(request.url)
 
@@ -264,7 +262,7 @@ class _SSLContextAdapterMixin:
     def __init__(
         self,
         *,
-        ssl_context: Optional["SSLContext"] = None,
+        ssl_context: SSLContext | None = None,
         **kwargs: Any,
     ) -> None:
         self._ssl_context = ssl_context
@@ -276,7 +274,7 @@ class _SSLContextAdapterMixin:
         maxsize: int,
         block: bool = DEFAULT_POOLBLOCK,
         **pool_kwargs: Any,
-    ) -> "PoolManager":
+    ) -> PoolManager:
         if self._ssl_context is not None:
             pool_kwargs.setdefault("ssl_context", self._ssl_context)
         return super().init_poolmanager(  # type: ignore[misc]
@@ -286,6 +284,13 @@ class _SSLContextAdapterMixin:
             **pool_kwargs,
         )
 
+    def proxy_manager_for(self, proxy: str, **proxy_kwargs: Any) -> ProxyManager:
+        # Proxy manager replaces the pool manager, so inject our SSL
+        # context here too. https://github.com/pypa/pip/issues/13288
+        if self._ssl_context is not None:
+            proxy_kwargs.setdefault("ssl_context", self._ssl_context)
+        return super().proxy_manager_for(proxy, **proxy_kwargs)  # type: ignore[misc]
+
 
 class HTTPAdapter(_SSLContextAdapterMixin, _BaseHTTPAdapter):
     pass
@@ -300,8 +305,8 @@ class InsecureHTTPAdapter(HTTPAdapter):
         self,
         conn: ConnectionPool,
         url: str,
-        verify: Union[bool, str],
-        cert: Optional[Union[str, Tuple[str, str]]],
+        verify: bool | str,
+        cert: str | tuple[str, str] | None,
     ) -> None:
         super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
 
@@ -311,23 +316,23 @@ class InsecureCacheControlAdapter(CacheControlAdapter):
         self,
         conn: ConnectionPool,
         url: str,
-        verify: Union[bool, str],
-        cert: Optional[Union[str, Tuple[str, str]]],
+        verify: bool | str,
+        cert: str | tuple[str, str] | None,
     ) -> None:
         super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
 
 
 class PipSession(requests.Session):
-    timeout: Optional[int] = None
+    timeout: int | None = None
 
     def __init__(
         self,
         *args: Any,
         retries: int = 0,
-        cache: Optional[str] = None,
+        cache: str | None = None,
         trusted_hosts: Sequence[str] = (),
-        index_urls: Optional[List[str]] = None,
-        ssl_context: Optional["SSLContext"] = None,
+        index_urls: list[str] | None = None,
+        ssl_context: SSLContext | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -338,7 +343,7 @@ class PipSession(requests.Session):
 
         # Namespace the attribute with "pip_" just in case to prevent
         # possible conflicts with the base class.
-        self.pip_trusted_origins: List[Tuple[str, Optional[int]]] = []
+        self.pip_trusted_origins: list[tuple[str, int | None]] = []
         self.pip_proxy = None
 
         # Attach our User Agent to the request
@@ -401,7 +406,7 @@ class PipSession(requests.Session):
         for host in trusted_hosts:
             self.add_trusted_host(host, suppress_logging=True)
 
-    def update_index_urls(self, new_index_urls: List[str]) -> None:
+    def update_index_urls(self, new_index_urls: list[str]) -> None:
         """
         :param new_index_urls: New index urls to update the authentication
             handler with.
@@ -409,7 +414,7 @@ class PipSession(requests.Session):
         self.auth.index_urls = new_index_urls
 
     def add_trusted_host(
-        self, host: str, source: Optional[str] = None, suppress_logging: bool = False
+        self, host: str, source: str | None = None, suppress_logging: bool = False
     ) -> None:
         """
         :param host: It is okay to provide a host that has previously been