pex 2.54.2__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

pex/pip/tool.py

@@ -21,21 +21,21 @@ from pex.common import safe_mkdir, safe_mkdtemp
 from pex.compatibility import get_stderr_bytes_buffer, shlex_quote, urlparse
 from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import Requirement
+from pex.fetcher import URLFetcher
 from pex.interpreter import PythonInterpreter
 from pex.jobs import Job
 from pex.network_configuration import NetworkConfiguration
 from pex.pep_427 import install_wheel_interpreter
-from pex.pip import dependencies, foreign_platform
+from pex.pep_508 import MarkerEnvironment
+from pex.pip import dependencies, foreign_platform, package_repositories
 from pex.pip.download_observer import DownloadObserver, PatchSet
 from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage, LogAnalyzer, LogScrapeJob
 from pex.pip.tailer import Tailer
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.platforms import PlatformSpec
-from pex.resolve.resolver_configuration import (
-    BuildConfiguration,
-    ReposConfiguration,
-    ResolverVersion,
-)
+from pex.resolve.package_repository import ReposConfiguration
+from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion
+from pex.resolve.target_system import UniversalTarget
 from pex.targets import Target
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
@@ -54,7 +54,9 @@ if TYPE_CHECKING:
         Match,
         Optional,
         Sequence,
+        Text,
         Tuple,
+        Union,
     )
 
     import attr  # vendor:skip
@@ -64,8 +66,8 @@ else:
 
 @attr.s(frozen=True)
 class PipArgs(object):
-    indexes = attr.ib(default=None)  # type: Optional[Sequence[str]]
-    find_links = attr.ib(default=None)  # type: Optional[Iterable[str]]
+    indexes = attr.ib(default=None)  # type: Optional[Sequence[Text]]
+    find_links = attr.ib(default=None)  # type: Optional[Iterable[Text]]
     network_configuration = attr.ib(default=None)  # type: Optional[NetworkConfiguration]
 
     def iter(self, version):
@@ -152,10 +154,8 @@ class PackageIndexConfiguration(object):
         cls,
         pip_version=None,  # type: Optional[PipVersionValue]
         resolver_version=None,  # type: Optional[ResolverVersion.Value]
-        indexes=None,  # type: Optional[Sequence[str]]
-        find_links=None,  # type: Optional[Iterable[str]]
+        repos_configuration=ReposConfiguration(),  # type: ReposConfiguration
         network_configuration=None,  # type: Optional[NetworkConfiguration]
-        password_entries=(),  # type: Iterable[PasswordEntry]
         use_pip_config=False,  # type: bool
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
         keyring_provider=None,  # type: Optional[str]
@@ -174,8 +174,8 @@ class PackageIndexConfiguration(object):
             resolver_version=resolver_version,
             network_configuration=network_configuration,
             args=PipArgs(
-                indexes=indexes,
-                find_links=find_links,
+                indexes=repos_configuration.indexes,
+                find_links=repos_configuration.find_links,
                 network_configuration=network_configuration,
             ),
             env=cls._calculate_env(
@@ -183,7 +183,7 @@ class PackageIndexConfiguration(object):
             ),
             use_pip_config=use_pip_config,
             extra_pip_requirements=extra_pip_requirements,
-            password_entries=password_entries,
+            repos_configuration=repos_configuration,
             keyring_provider=keyring_provider,
         )
 
@@ -194,7 +194,7 @@ class PackageIndexConfiguration(object):
         args,  # type: PipArgs
         env,  # type: Iterable[Tuple[str, str]]
         use_pip_config,  # type: bool
-        password_entries=(),  # type: Iterable[PasswordEntry]
+        repos_configuration=ReposConfiguration(),  # type: ReposConfiguration
         pip_version=None,  # type: Optional[PipVersionValue]
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
         keyring_provider=None,  # type: Optional[str]
@@ -205,11 +205,32 @@ class PackageIndexConfiguration(object):
         self.args = args  # type: PipArgs
         self.env = dict(env)  # type: Mapping[str, str]
         self.use_pip_config = use_pip_config  # type: bool
-        self.password_entries = password_entries  # type: Iterable[PasswordEntry]
+        self.repos_configuration = repos_configuration
         self.pip_version = pip_version  # type: Optional[PipVersionValue]
         self.extra_pip_requirements = extra_pip_requirements  # type: Tuple[Requirement, ...]
         self.keyring_provider = keyring_provider  # type: Optional[str]
 
+    @property
+    def password_entries(self):
+        # type: () -> Iterable[PasswordEntry]
+        return self.repos_configuration.password_entries
+
+    def patch(
+        self,
+        pip_version,  # type: PipVersionValue
+        target,  # type:  Union[UniversalTarget, MarkerEnvironment]
+        requirement_files=None,  # type: Optional[Iterable[str]]
+    ):
+        # type: (...) -> Optional[DownloadObserver]
+        return package_repositories.patch(
+            repos_configuration=self.repos_configuration.with_contained_repos(
+                requirement_files,
+                fetcher=URLFetcher(network_configuration=self.network_configuration),
+            ),
+            pip_version=pip_version,
+            target=target,
+        )
+
 
 if TYPE_CHECKING:
     from pex.pip.log_analyzer import ErrorAnalysis
@@ -455,7 +476,7 @@ class Pip(object):
 
         command = pip_args + list(args)
 
-        # N.B.: Package index options in Pep always have the same option names, but they are
+        # N.B.: Package index options in Pip always have the same option names, but they are
         # registered as subcommand-specific, so we must append them here _after_ the pip subcommand
         # specified in `args`.
         if package_index_configuration:
@@ -470,9 +491,7 @@ class Pip(object):
         # since Pip relies upon `shutil.move` which is only atomic when `os.rename` can be used.
         # See https://github.com/pex-tool/pex/issues/1776 for an example of the issues non-atomic
         # moves lead to in the `pip wheel` case.
-        pip_tmpdir = os.path.join(self.cache_dir, ".tmp")
-        safe_mkdir(pip_tmpdir)
-        extra_env.update(TMPDIR=pip_tmpdir)
+        extra_env.update(TMPDIR=safe_mkdtemp(dir=safe_mkdir(self.cache_dir), prefix=".tmp."))
 
         with ENV.strip().patch(
             PEX_ROOT=ENV.PEX_ROOT,
@@ -533,8 +552,7 @@ class Pip(object):
         )
         return Job(command=command, process=process, finalizer=finalizer, context="pip")
 
-    @staticmethod
-    def _iter_build_configuration_options(build_configuration):
+    def _iter_build_configuration_options(self, build_configuration):
         # type: (BuildConfiguration) -> Iterator[str]
 
         # N.B.: BuildConfiguration maintains invariants that ensure --only-binary, --no-binary,
@@ -557,12 +575,54 @@ class Pip(object):
         if build_configuration.prefer_older_binary:
             yield "--prefer-binary"
 
-        if build_configuration.use_pep517 is not None:
+        # N.B.: In 25.3 `--use-pep517` became the default and only option.
+        if build_configuration.use_pep517 is not None and self.version < PipVersion.v25_3:
             yield "--use-pep517" if build_configuration.use_pep517 else "--no-use-pep517"
 
         if not build_configuration.build_isolation:
             yield "--no-build-isolation"
 
+    def spawn_report(
+        self,
+        report_path,  # type: str
+        requirements=None,  # type: Optional[Iterable[str]]
+        requirement_files=None,  # type: Optional[Iterable[str]]
+        constraint_files=None,  # type: Optional[Iterable[str]]
+        allow_prereleases=False,  # type: bool
+        transitive=True,  # type: bool
+        target=None,  # type: Optional[Target]
+        package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
+        build_configuration=BuildConfiguration(),  # type: BuildConfiguration
+        observer=None,  # type: Optional[DownloadObserver]
+        dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+        universal_target=None,  # type: Optional[UniversalTarget]
+        log=None,  # type: Optional[str]
+    ):
+        # type: (...) -> Job
+        report_cmd = [
+            "install",
+            "--no-clean",
+            "--dry-run",
+            "--ignore-installed",
+            "--report",
+            report_path,
+        ]
+        return self._spawn_install_compatible_command(
+            cmd=report_cmd,
+            requirements=requirements,
+            requirement_files=requirement_files,
+            constraint_files=constraint_files,
+            allow_prereleases=allow_prereleases,
+            transitive=transitive,
+            target=target,
+            package_index_configuration=package_index_configuration,
+            build_configuration=build_configuration,
+            observer=observer,
+            dependency_configuration=dependency_configuration,
+            universal_target=universal_target,
+            log=log,
+        )
+
     def spawn_download_distributions(
         self,
         download_dir,  # type: str
@@ -576,35 +636,70 @@ class Pip(object):
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         observer=None,  # type: Optional[DownloadObserver]
         dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+        universal_target=None,  # type: Optional[UniversalTarget]
         log=None,  # type: Optional[str]
     ):
         # type: (...) -> Job
-        target = target or targets.current()
 
         download_cmd = ["download", "--dest", download_dir]
+        return self._spawn_install_compatible_command(
+            cmd=download_cmd,
+            requirements=requirements,
+            requirement_files=requirement_files,
+            constraint_files=constraint_files,
+            allow_prereleases=allow_prereleases,
+            transitive=transitive,
+            target=target,
+            package_index_configuration=package_index_configuration,
+            build_configuration=build_configuration,
+            observer=observer,
+            dependency_configuration=dependency_configuration,
+            universal_target=universal_target,
+            log=log,
+        )
+
+    def _spawn_install_compatible_command(
+        self,
+        cmd,  # type: List[str]
+        requirements=None,  # type: Optional[Iterable[str]]
+        requirement_files=None,  # type: Optional[Iterable[str]]
+        constraint_files=None,  # type: Optional[Iterable[str]]
+        allow_prereleases=False,  # type: bool
+        transitive=True,  # type: bool
+        target=None,  # type: Optional[Target]
+        package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
+        build_configuration=BuildConfiguration(),  # type: BuildConfiguration
+        observer=None,  # type: Optional[DownloadObserver]
+        dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+        universal_target=None,  # type: Optional[UniversalTarget]
+        log=None,  # type: Optional[str]
+    ):
+        # type: (...) -> Job
+        target = target or targets.current()
+
         extra_env = {}  # type: Dict[str, str]
         pex_extra_sys_path = []  # type: List[str]
 
-        download_cmd.extend(self._iter_build_configuration_options(build_configuration))
+        cmd.extend(self._iter_build_configuration_options(build_configuration))
         if not build_configuration.build_isolation:
             pex_extra_sys_path.extend(sys.path)
 
         if allow_prereleases:
-            download_cmd.append("--pre")
+            cmd.append("--pre")
 
         if not transitive:
-            download_cmd.append("--no-deps")
+            cmd.append("--no-deps")
 
         if requirement_files:
             for requirement_file in requirement_files:
-                download_cmd.extend(["--requirement", requirement_file])
+                cmd.extend(["--requirement", requirement_file])
 
         if constraint_files:
             for constraint_file in constraint_files:
-                download_cmd.extend(["--constraint", constraint_file])
+                cmd.extend(["--constraint", constraint_file])
 
         if requirements:
-            download_cmd.extend(requirements)
+            cmd.extend(requirements)
 
         foreign_platform_observer = foreign_platform.patch(target)
         if (
@@ -623,7 +718,18 @@ class Pip(object):
         for obs in (
             foreign_platform_observer,
             observer,
-            dependencies.patch(dependency_configuration),
+            dependencies.patch(
+                dependency_configuration, target=universal_target or target.marker_environment
+            ),
+            (
+                package_index_configuration.patch(
+                    pip_version=self.version,
+                    target=universal_target or target.marker_environment,
+                    requirement_files=requirement_files,
+                )
+                if package_index_configuration
+                else None
+            ),
         ):
             if obs:
                 if obs.analyzer:
@@ -689,7 +795,7 @@ class Pip(object):
                 tailer.stop()
 
         command, process = self._spawn_pip_isolated(
-            download_cmd,
+            cmd,
             package_index_configuration=package_index_configuration,
             interpreter=target.get_interpreter(),
             log=log,
@@ -717,7 +823,7 @@ class Pip(object):
                     build_configuration=BuildConfiguration.create(allow_builds=False),
                 ).wait()
                 for wheel in glob.glob(os.path.join(atomic_dir.work_dir, "*.whl")):
-                    install_wheel_interpreter(wheel_path=wheel, interpreter=pip_interpreter)
+                    install_wheel_interpreter(wheel=wheel, interpreter=pip_interpreter)
 
     def spawn_build_wheels(
         self,

pex/pip/vcs.py

@@ -8,20 +8,37 @@ import re
 
 from pex import hashing
 from pex.artifact_url import VCS, Fingerprint
-from pex.common import is_pyc_dir, is_pyc_file, open_zip, temporary_dir
+from pex.common import is_pyc_dir, is_pyc_file
+from pex.exceptions import reportable_unexpected_error_msg
 from pex.hashing import Sha256
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.result import Error, try_
-from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Optional, Tuple, Union
+    # N.B.: The `re.Pattern` type is not available in all Python versions Pex supports.
+    from re import Pattern  # type: ignore[attr-defined]
+    from typing import Callable, Optional, Text, Tuple, Union
 
     from pex.hashing import HintedDigest
 
 
+def _project_name_re(project_name):
+    # type: (ProjectName) -> str
+    return project_name.normalized.replace("-", "[-_.]+")
+
+
+def _built_source_dist_pattern(project_name):
+    # type: (ProjectName) -> Pattern
+    return re.compile(
+        r"(?P<project_name>{project_name_re})-(?P<version>.+)\.zip".format(
+            project_name_re=_project_name_re(project_name)
+        ),
+        re.IGNORECASE,
+    )
+
+
 def _find_built_source_dist(
     build_dir,  # type: str
     project_name,  # type: ProjectName
@@ -33,12 +50,7 @@ def _find_built_source_dist(
     # encoded in: `pip._internal.req.req_install.InstallRequirement.archive`.
 
     listing = os.listdir(build_dir)
-    pattern = re.compile(
-        r"{project_name}-(?P<version>.+)\.zip".format(
-            project_name=project_name.normalized.replace("-", "[-_.]+")
-        ),
-        re.IGNORECASE,
-    )
+    pattern = _built_source_dist_pattern(project_name)
     for name in listing:
         match = pattern.match(name)
         if match and Version(match.group("version")) == version:
@@ -57,57 +69,111 @@ def _find_built_source_dist(
 
 def fingerprint_downloaded_vcs_archive(
     download_dir,  # type: str
-    project_name,  # type: str
-    version,  # type: str
+    project_name,  # type: ProjectName
+    version,  # type: Version
     vcs,  # type: VCS.Value
-    subdirectory=None,  # type: Optional[str]
 ):
     # type: (...) -> Tuple[Fingerprint, str]
 
     archive_path = try_(
-        _find_built_source_dist(
-            build_dir=download_dir, project_name=ProjectName(project_name), version=Version(version)
-        )
+        _find_built_source_dist(build_dir=download_dir, project_name=project_name, version=version)
     )
     digest = Sha256()
-    digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest, subdirectory=subdirectory)
+    digest_vcs_archive(project_name=project_name, archive_path=archive_path, vcs=vcs, digest=digest)
     return Fingerprint.from_digest(digest), archive_path
 
 
+def _vcs_dir_filter(
+    vcs,  # type: VCS.Value
+    project_name,  # type: ProjectName
+):
+    # type: (...) -> Callable[[Text], bool]
+
+    # Ignore VCS control directories for the purposes of fingerprinting the version controlled
+    # source tree. VCS control directories can contain non-reproducible content (Git at least
+    # has files that contain timestamps).
+    #
+    # We cannot prune these directories from the source archive directly unfortunately since
+    # some build processes use VCS version information to derive their version numbers (C.F.:
+    # https://pypi.org/project/setuptools-scm/). As such, we'll get a stable fingerprint, but be
+    # forced to re-build a wheel each time the VCS requirement is re-locked later, even when it
+    # hashes the same.
+    vcs_control_dir = ".{vcs}".format(vcs=vcs)
+
+    # N.B.: If the VCS project uses setuptools as its build backend, depending on the version of
+    # Pip used, the VCS checkout can have a `<project name>.egg-info/` directory littering its root
+    # left over from Pip generating project metadata to determine version and dependencies. No other
+    # well known build-backend has this problem at this time (checked hatchling, poetry-core,
+    # pdm-backend and uv_build).
+    # C.F.: https://github.com/pypa/pip/pull/13602
+    egg_info_dir_re = re.compile(
+        r"^{project_name_re}\.egg-info$".format(project_name_re=_project_name_re(project_name)),
+        re.IGNORECASE,
+    )
+
+    def vcs_dir_filter(dir_path):
+        # type: (Text) -> bool
+        if is_pyc_dir(dir_path):
+            return False
+
+        base_dir_name = dir_path.split(os.sep)[0]
+        return base_dir_name != vcs_control_dir and not egg_info_dir_re.match(base_dir_name)
+
+    return vcs_dir_filter
+
+
+def _vcs_file_filter(vcs):
+    # type: (VCS.Value) -> Callable[[Text], bool]
+    return lambda f: not is_pyc_file(f)
+
+
 def digest_vcs_archive(
+    project_name,  # type: ProjectName
     archive_path,  # type: str
     vcs,  # type: VCS.Value
     digest,  # type: HintedDigest
-    subdirectory=None,  # type: Optional[str]
 ):
     # type: (...) -> None
 
     # All VCS requirements are prepared as zip archives as encoded in:
-    # `pip._internal.req.req_install.InstallRequirement.archive`.
-    with TRACER.timed(
-        "Digesting {archive} {vcs} archive".format(archive=os.path.basename(archive_path), vcs=vcs)
-    ), temporary_dir() as chroot, open_zip(archive_path) as archive:
-        archive.extractall(chroot)
-
-        # Ignore VCS control directories for the purposes of fingerprinting the version controlled
-        # source tree. VCS control directories can contain non-reproducible content (Git at least
-        # has files that contain timestamps).
-        #
-        # We cannot prune these directories from the source archive directly unfortunately since
-        # some build processes use VCS version information to derive their version numbers (C.F.:
-        # https://pypi.org/project/setuptools-scm/). As such, we'll get a stable fingerprint, but be
-        # forced to re-build a wheel each time the VCS requirement is re-locked later, even when it
-        # hashes the same.
-        vcs_control_dir = ".{vcs}".format(vcs=vcs)
-
-        # TODO(John Sirois): Consider implementing zip_hash to avoid the extractall.
-        hashing.dir_hash(
-            directory=os.path.join(chroot, subdirectory) if subdirectory else chroot,
-            digest=digest,
-            dir_filter=(
-                lambda dir_path: (
-                    not is_pyc_dir(dir_path) and os.path.basename(dir_path) != vcs_control_dir
+    # `pip._internal.req.req_install.InstallRequirement.archive` and the archive is already offset
+    # by a subdirectory (if any).
+
+    # The zip archives created by Pip have a single project name top-level directory housing
+    # the full clone. We look for that to get a consistent clone hash with a bare clone.
+    match = _built_source_dist_pattern(project_name).match(os.path.basename(archive_path))
+    if match is None:
+        raise AssertionError(
+            reportable_unexpected_error_msg(
+                "Failed to determine the project name prefix for the VCS zip {zip} with expected "
+                "canonical project name {project_name}".format(
+                    zip=archive_path, project_name=project_name
                 )
-            ),
-            file_filter=lambda f: not is_pyc_file(f),
+            )
         )
+    top_dir = match.group("project_name")
+
+    hashing.zip_hash(
+        zip_path=archive_path,
+        digest=digest,
+        relpath=top_dir,
+        dir_filter=_vcs_dir_filter(vcs, project_name),
+        file_filter=_vcs_file_filter(vcs),
+    )
+
+
+def digest_vcs_repo(
+    project_name,  # type: ProjectName
+    repo_path,  # type: str
+    vcs,  # type: VCS.Value
+    digest,  # type: HintedDigest
+    subdirectory=None,  # type: Optional[str]
+):
+    # type: (...) -> None
+
+    hashing.dir_hash(
+        directory=os.path.join(repo_path, subdirectory) if subdirectory else repo_path,
+        digest=digest,
+        dir_filter=_vcs_dir_filter(vcs, project_name),
+        file_filter=_vcs_file_filter(vcs),
+    )

pex/pip/version.py

@@ -366,7 +366,14 @@ class PipVersion(Enum["PipVersionValue"]):
         version="25.2",
         setuptools_version="80.9.0",
         wheel_version="0.45.1",
-        requires_python=">=3.9,<3.15",
+        requires_python=">=3.9,<3.16",
+    )
+
+    v25_3 = PipVersionValue(
+        version="25.3",
+        setuptools_version="80.9.0",
+        wheel_version="0.45.1",
+        requires_python=">=3.9,<3.16",
     )
 
     VENDORED = v20_3_4_patched