pex 2.63.0__py2.py3-none-any.whl → 2.64.0__py2.py3-none-any.whl

pex/pip/tool.py

@@ -491,9 +491,7 @@ class Pip(object):
         # since Pip relies upon `shutil.move` which is only atomic when `os.rename` can be used.
         # See https://github.com/pex-tool/pex/issues/1776 for an example of the issues non-atomic
         # moves lead to in the `pip wheel` case.
-        pip_tmpdir = os.path.join(self.cache_dir, ".tmp")
-        safe_mkdir(pip_tmpdir)
-        extra_env.update(TMPDIR=pip_tmpdir)
+        extra_env.update(TMPDIR=safe_mkdtemp(dir=safe_mkdir(self.cache_dir), prefix=".tmp."))
 
         with ENV.strip().patch(
             PEX_ROOT=ENV.PEX_ROOT,
@@ -584,6 +582,47 @@ class Pip(object):
         if not build_configuration.build_isolation:
             yield "--no-build-isolation"
 
+    def spawn_report(
+        self,
+        report_path,  # type: str
+        requirements=None,  # type: Optional[Iterable[str]]
+        requirement_files=None,  # type: Optional[Iterable[str]]
+        constraint_files=None,  # type: Optional[Iterable[str]]
+        allow_prereleases=False,  # type: bool
+        transitive=True,  # type: bool
+        target=None,  # type: Optional[Target]
+        package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
+        build_configuration=BuildConfiguration(),  # type: BuildConfiguration
+        observer=None,  # type: Optional[DownloadObserver]
+        dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+        universal_target=None,  # type: Optional[UniversalTarget]
+        log=None,  # type: Optional[str]
+    ):
+        # type: (...) -> Job
+        report_cmd = [
+            "install",
+            "--no-clean",
+            "--dry-run",
+            "--ignore-installed",
+            "--report",
+            report_path,
+        ]
+        return self._spawn_install_compatible_command(
+            cmd=report_cmd,
+            requirements=requirements,
+            requirement_files=requirement_files,
+            constraint_files=constraint_files,
+            allow_prereleases=allow_prereleases,
+            transitive=transitive,
+            target=target,
+            package_index_configuration=package_index_configuration,
+            build_configuration=build_configuration,
+            observer=observer,
+            dependency_configuration=dependency_configuration,
+            universal_target=universal_target,
+            log=log,
+        )
+
     def spawn_download_distributions(
         self,
         download_dir,  # type: str
@@ -601,32 +640,66 @@ class Pip(object):
         log=None,  # type: Optional[str]
     ):
         # type: (...) -> Job
-        target = target or targets.current()
 
         download_cmd = ["download", "--dest", download_dir]
+        return self._spawn_install_compatible_command(
+            cmd=download_cmd,
+            requirements=requirements,
+            requirement_files=requirement_files,
+            constraint_files=constraint_files,
+            allow_prereleases=allow_prereleases,
+            transitive=transitive,
+            target=target,
+            package_index_configuration=package_index_configuration,
+            build_configuration=build_configuration,
+            observer=observer,
+            dependency_configuration=dependency_configuration,
+            universal_target=universal_target,
+            log=log,
+        )
+
+    def _spawn_install_compatible_command(
+        self,
+        cmd,  # type: List[str]
+        requirements=None,  # type: Optional[Iterable[str]]
+        requirement_files=None,  # type: Optional[Iterable[str]]
+        constraint_files=None,  # type: Optional[Iterable[str]]
+        allow_prereleases=False,  # type: bool
+        transitive=True,  # type: bool
+        target=None,  # type: Optional[Target]
+        package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
+        build_configuration=BuildConfiguration(),  # type: BuildConfiguration
+        observer=None,  # type: Optional[DownloadObserver]
+        dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+        universal_target=None,  # type: Optional[UniversalTarget]
+        log=None,  # type: Optional[str]
+    ):
+        # type: (...) -> Job
+        target = target or targets.current()
+
         extra_env = {}  # type: Dict[str, str]
         pex_extra_sys_path = []  # type: List[str]
 
-        download_cmd.extend(self._iter_build_configuration_options(build_configuration))
+        cmd.extend(self._iter_build_configuration_options(build_configuration))
         if not build_configuration.build_isolation:
             pex_extra_sys_path.extend(sys.path)
 
         if allow_prereleases:
-            download_cmd.append("--pre")
+            cmd.append("--pre")
 
         if not transitive:
-            download_cmd.append("--no-deps")
+            cmd.append("--no-deps")
 
         if requirement_files:
             for requirement_file in requirement_files:
-                download_cmd.extend(["--requirement", requirement_file])
+                cmd.extend(["--requirement", requirement_file])
 
         if constraint_files:
             for constraint_file in constraint_files:
-                download_cmd.extend(["--constraint", constraint_file])
+                cmd.extend(["--constraint", constraint_file])
 
         if requirements:
-            download_cmd.extend(requirements)
+            cmd.extend(requirements)
 
         foreign_platform_observer = foreign_platform.patch(target)
         if (
@@ -722,7 +795,7 @@ class Pip(object):
                 tailer.stop()
 
         command, process = self._spawn_pip_isolated(
-            download_cmd,
+            cmd,
             package_index_configuration=package_index_configuration,
             interpreter=target.get_interpreter(),
             log=log,

pex/pip/vcs.py

@@ -3,6 +3,7 @@
 
 from __future__ import absolute_import
 
+import glob
 import os
 import re
 
@@ -60,7 +61,6 @@ def fingerprint_downloaded_vcs_archive(
     project_name,  # type: str
     version,  # type: str
     vcs,  # type: VCS.Value
-    subdirectory=None,  # type: Optional[str]
 ):
     # type: (...) -> Tuple[Fingerprint, str]
 
@@ -70,7 +70,7 @@ def fingerprint_downloaded_vcs_archive(
         )
     )
     digest = Sha256()
-    digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest, subdirectory=subdirectory)
+    digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest)
     return Fingerprint.from_digest(digest), archive_path
 
 
@@ -78,36 +78,53 @@ def digest_vcs_archive(
     archive_path,  # type: str
     vcs,  # type: VCS.Value
     digest,  # type: HintedDigest
-    subdirectory=None,  # type: Optional[str]
 ):
     # type: (...) -> None
 
     # All VCS requirements are prepared as zip archives as encoded in:
-    # `pip._internal.req.req_install.InstallRequirement.archive`.
+    # `pip._internal.req.req_install.InstallRequirement.archive` and the archive is already offset
+    # by a subdirectory (if any).
     with TRACER.timed(
         "Digesting {archive} {vcs} archive".format(archive=os.path.basename(archive_path), vcs=vcs)
     ), temporary_dir() as chroot, open_zip(archive_path) as archive:
+        # TODO(John Sirois): Consider implementing zip_hash to avoid the extractall.
         archive.extractall(chroot)
 
-        # Ignore VCS control directories for the purposes of fingerprinting the version controlled
-        # source tree. VCS control directories can contain non-reproducible content (Git at least
-        # has files that contain timestamps).
-        #
-        # We cannot prune these directories from the source archive directly unfortunately since
-        # some build processes use VCS version information to derive their version numbers (C.F.:
-        # https://pypi.org/project/setuptools-scm/). As such, we'll get a stable fingerprint, but be
-        # forced to re-build a wheel each time the VCS requirement is re-locked later, even when it
-        # hashes the same.
-        vcs_control_dir = ".{vcs}".format(vcs=vcs)
+        # The zip archives created by Pip have a single project name top-level directory housing
+        # the full clone. We look for that to get a consistent clone hash with a bare clone.
+        listing = glob.glob(os.path.join(chroot, "*"))
+        if len(listing) == 1 and os.path.isdir(listing[0]):
+            chroot = listing[0]
 
-        # TODO(John Sirois): Consider implementing zip_hash to avoid the extractall.
-        hashing.dir_hash(
-            directory=os.path.join(chroot, subdirectory) if subdirectory else chroot,
-            digest=digest,
-            dir_filter=(
-                lambda dir_path: (
-                    not is_pyc_dir(dir_path) and os.path.basename(dir_path) != vcs_control_dir
-                )
-            ),
-            file_filter=lambda f: not is_pyc_file(f),
-        )
+        digest_vcs_repo(repo_path=chroot, vcs=vcs, digest=digest)
+
+
+def digest_vcs_repo(
+    repo_path,  # type: str
+    vcs,  # type: VCS.Value
+    digest,  # type: HintedDigest
+    subdirectory=None,  # type: Optional[str]
+):
+    # type: (...) -> None
+
+    # Ignore VCS control directories for the purposes of fingerprinting the version controlled
+    # source tree. VCS control directories can contain non-reproducible content (Git at least
+    # has files that contain timestamps).
+    #
+    # We cannot prune these directories from the source archive directly unfortunately since
+    # some build processes use VCS version information to derive their version numbers (C.F.:
+    # https://pypi.org/project/setuptools-scm/). As such, we'll get a stable fingerprint, but be
+    # forced to re-build a wheel each time the VCS requirement is re-locked later, even when it
+    # hashes the same.
+    vcs_control_dir = ".{vcs}".format(vcs=vcs)
+
+    hashing.dir_hash(
+        directory=os.path.join(repo_path, subdirectory) if subdirectory else repo_path,
+        digest=digest,
+        dir_filter=(
+            lambda dir_path: (
+                not is_pyc_dir(dir_path) and os.path.basename(dir_path) != vcs_control_dir
+            )
+        ),
+        file_filter=lambda f: not is_pyc_file(f),
+    )

pex/resolve/lock_downloader.py

@@ -157,7 +157,6 @@ class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):
             archive_path=local_distribution.path,
             vcs=artifact.vcs,
             digest=digest,
-            subdirectory=artifact.subdirectory,
         )
         shutil.move(local_distribution.path, os.path.join(dest_dir, filename))
         return filename

pex/resolve/locked_resolve.py

@@ -12,7 +12,7 @@ from functools import total_ordering
 from pex.artifact_url import VCS, ArtifactURL, Fingerprint, VCSScheme
 from pex.common import pluralize
 from pex.dependency_configuration import DependencyConfiguration
-from pex.dist_metadata import DistMetadata, Requirement, is_sdist, is_wheel
+from pex.dist_metadata import ProjectMetadata, Requirement, is_sdist, is_wheel
 from pex.enum import Enum
 from pex.exceptions import production_assert
 from pex.interpreter_constraints import InterpreterConstraint
@@ -660,7 +660,7 @@ class LockedResolve(object):
     def create(
         cls,
         resolved_requirements,  # type: Iterable[ResolvedRequirement]
-        dist_metadatas,  # type: Iterable[DistMetadata]
+        project_metadatas,  # type: Iterable[ProjectMetadata]
         fingerprinter,  # type: Fingerprinter
         platform_tag=None,  # type: Optional[tags.Tag]
         marker=None,  # type: Optional[Marker]
@@ -703,21 +703,21 @@ class LockedResolve(object):
                 editable=partial_artifact.editable,
             )
 
-        dist_metadata_by_pin = {
+        project_metadata_by_pin = {
             Pin(dist_info.project_name, dist_info.version): dist_info
-            for dist_info in dist_metadatas
+            for dist_info in project_metadatas
         }
         locked_requirements = []
         for resolved_requirement in resolved_requirements:
-            distribution_metadata = dist_metadata_by_pin.get(resolved_requirement.pin)
-            if distribution_metadata is None:
+            project_metadata = project_metadata_by_pin.get(resolved_requirement.pin)
+            if project_metadata is None:
                 raise ValueError(
-                    "No distribution metadata found for {project}.\n"
-                    "Given distribution metadata for:\n"
+                    "No project metadata found for {project}.\n"
+                    "Given project metadata for:\n"
                     "{projects}".format(
                         project=resolved_requirement.pin.as_requirement(),
                         projects="\n".join(
-                            sorted(str(pin.as_requirement()) for pin in dist_metadata_by_pin)
+                            sorted(str(pin.as_requirement()) for pin in project_metadata_by_pin)
                         ),
                     )
                 )
@@ -725,8 +725,8 @@ class LockedResolve(object):
                 LockedRequirement.create(
                     pin=resolved_requirement.pin,
                     artifact=resolve_fingerprint(resolved_requirement.artifact),
-                    requires_dists=distribution_metadata.requires_dists,
-                    requires_python=distribution_metadata.requires_python,
+                    requires_dists=project_metadata.requires_dists,
+                    requires_python=project_metadata.requires_python,
                     additional_artifacts=(
                         resolve_fingerprint(artifact)
                         for artifact in resolved_requirement.additional_artifacts

pex/resolve/locker.py

@@ -16,11 +16,12 @@ from pex.dist_metadata import ProjectNameAndVersion, Requirement
 from pex.hashing import Sha256
 from pex.orderedset import OrderedSet
 from pex.pep_440 import Version
+from pex.pep_503 import ProjectName
 from pex.pip import foreign_platform
 from pex.pip.download_observer import Patch, PatchSet
 from pex.pip.local_project import digest_local_project
 from pex.pip.log_analyzer import LogAnalyzer
-from pex.pip.vcs import fingerprint_downloaded_vcs_archive
+from pex.pip.vcs import digest_vcs_repo, fingerprint_downloaded_vcs_archive
 from pex.pip.version import PipVersionValue
 from pex.requirements import LocalProjectRequirement, VCSRequirement
 from pex.resolve.locked_resolve import LockStyle
@@ -194,6 +195,7 @@ class AnalyzeError(Exception):
 
 @attr.s(frozen=True)
 class ArtifactBuildResult(object):
+    path = attr.ib()  # type: str
     url = attr.ib()  # type: ArtifactURL
     pin = attr.ib()  # type: Pin
 
@@ -211,17 +213,18 @@ class ArtifactBuildObserver(object):
         # type: (str) -> Optional[ArtifactBuildResult]
 
         match = re.search(
-            r"Source in .+ has version (?P<version>\S+), which satisfies requirement "
+            r"Source in (?P<path>.+) has version (?P<version>\S+), which satisfies requirement "
             r"(?P<requirement>.+) .*from {url}".format(url=re.escape(self._artifact_url.raw_url)),
             line,
         )
         if not match:
             return None
 
+        path = match.group("path")
         version = Version(match.group("version"))
         requirement = Requirement.parse(match.group("requirement"))
         pin = Pin(project_name=requirement.project_name, version=version)
-        return ArtifactBuildResult(url=self._artifact_url, pin=pin)
+        return ArtifactBuildResult(path=path, url=self._artifact_url, pin=pin)
 
 
 class Locker(LogAnalyzer):
@@ -234,12 +237,14 @@ class Locker(LogAnalyzer):
         download_dir,  # type: str
         fingerprint_service=None,  # type: Optional[FingerprintService]
         pip_version=None,  # type: Optional[PipVersionValue]
+        lock_is_via_pip_download=False,  # type: bool
     ):
         # type: (...) -> None
 
         self._target = target
         self._vcs_url_manager = VCSURLManager.create(root_requirements)
         self._pip_version = pip_version
+        self._lock_is_via_pip_download = lock_is_via_pip_download
         self._resolver = resolver
         self._lock_style = lock_style
         self._download_dir = download_dir
@@ -357,19 +362,18 @@ class Locker(LogAnalyzer):
                 return self.Continue()
 
             build_result = self._artifact_build_observer.build_result(line)
-            if build_result:
+            source_fingerprint = None  # type: Optional[Fingerprint]
+            verified = False
+            commit_id = None  # type: Optional[str]
+            editable = False
+            if build_result and self._lock_is_via_pip_download:
                 artifact_url = build_result.url
-                source_fingerprint = None  # type: Optional[Fingerprint]
-                verified = False
-                commit_id = None  # type: Optional[str]
-                editable = False
                 if isinstance(artifact_url.scheme, VCSScheme):
                     source_fingerprint, archive_path = fingerprint_downloaded_vcs_archive(
                         download_dir=self._download_dir,
                         project_name=str(build_result.pin.project_name),
                         version=str(build_result.pin.version),
                         vcs=artifact_url.scheme.vcs,
-                        subdirectory=artifact_url.subdirectory,
                     )
                     verified = True
                     selected_path = os.path.basename(archive_path)
@@ -378,8 +382,8 @@ class Locker(LogAnalyzer):
                     )
                     self._selected_path_to_pin[selected_path] = build_result.pin
 
-                    vcs, _, vcs_url = build_result.url.raw_url.partition("+")
-                    if "@" in build_result.url.path:
+                    vcs, _, vcs_url = artifact_url.raw_url.partition("+")
+                    if "@" in artifact_url.path:
                         vcs_url, _, _ = vcs_url.rpartition("@")
                     commit_id = self._commit_ids.pop(vcs_url, None)
                 elif isinstance(artifact_url.scheme, ArchiveScheme.Value):
@@ -425,6 +429,74 @@ class Locker(LogAnalyzer):
                 additional_artifacts = self._links[build_result.pin]
                 additional_artifacts.pop(artifact_url, None)
 
+                self._resolved_requirements[build_result.pin] = ResolvedRequirement(
+                    pin=build_result.pin,
+                    artifact=PartialArtifact(
+                        url=artifact_url,
+                        fingerprint=source_fingerprint,
+                        verified=verified,
+                        commit_id=commit_id,
+                        editable=editable,
+                    ),
+                    additional_artifacts=tuple(additional_artifacts.values()),
+                )
+            elif build_result:
+                artifact_url = build_result.url
+                if isinstance(artifact_url.scheme, VCSScheme):
+                    digest = Sha256()
+                    digest_vcs_repo(
+                        repo_path=build_result.path,
+                        vcs=artifact_url.scheme.vcs,
+                        digest=digest,
+                        subdirectory=artifact_url.subdirectory,
+                    )
+                    source_fingerprint = Fingerprint.from_digest(digest)
+                    verified = True  # noqa
+                    vcs, _, vcs_url = artifact_url.raw_url.partition("+")
+                    if "@" in artifact_url.path:
+                        vcs_url, _, _ = vcs_url.rpartition("@")
+                    commit_id = self._commit_ids.pop(vcs_url, None)
+                elif isinstance(artifact_url.scheme, ArchiveScheme.Value):
+                    source_archive_path = build_result.path
+                    # If Pip resolves the artifact from its own cache, we will not find it in the
+                    # download dir for this run; so guard against that. In this case the existing
+                    # machinery that finalizes a locks missing fingerprints will download the
+                    # artifact and hash it.
+                    if os.path.isfile(source_archive_path):
+                        digest = Sha256()
+                        hashing.file_hash(source_archive_path, digest)
+                        source_fingerprint = Fingerprint.from_digest(digest)
+                        verified = True
+                elif "file" == artifact_url.scheme:
+                    digest = Sha256()
+                    if os.path.isfile(artifact_url.path):
+                        hashing.file_hash(artifact_url.path, digest)
+                        self._selected_path_to_pin[
+                            os.path.basename(artifact_url.path)
+                        ] = build_result.pin
+                    else:
+                        digest_local_project(
+                            directory=artifact_url.path,
+                            digest=digest,
+                            pip_version=self._pip_version,
+                            target=self._target,
+                            resolver=self._resolver,
+                        )
+                        self._local_projects.add(artifact_url.path)
+                        self._saved.add(build_result.pin)
+                        editable = artifact_url.path in self._editable_projects
+                    source_fingerprint = Fingerprint.from_digest(digest)
+                    verified = True
+                else:
+                    raise AnalyzeError(
+                        "Unexpected scheme {scheme!r} for artifact at {url}".format(
+                            scheme=artifact_url.scheme, url=artifact_url
+                        )
+                    )
+
+                additional_artifacts = self._links[build_result.pin]
+                additional_artifacts.pop(artifact_url, None)
+
                 self._resolved_requirements[build_result.pin] = ResolvedRequirement(
                     pin=build_result.pin,
                     artifact=PartialArtifact(
@@ -506,13 +578,21 @@ class Locker(LogAnalyzer):
             )
             return self.Continue()
 
-        match = re.search(r"Saved (?P<file_path>.+)$", line)
-        if match:
-            saved_path = match.group("file_path")
-            build_result_pin = self._selected_path_to_pin.get(os.path.basename(saved_path))
-            if build_result_pin:
-                self._saved.add(build_result_pin)
-            return self.Continue()
+        if self._lock_is_via_pip_download:
+            match = re.search(r"Saved (?P<file_path>.+)$", line)
+            if match:
+                saved_path = match.group("file_path")
+                build_result_pin = self._selected_path_to_pin.get(os.path.basename(saved_path))
+                if build_result_pin:
+                    self._saved.add(build_result_pin)
+                return self.Continue()
+        else:
+            match = re.search(r"Would install (?P<pnavs>.+)$", line)
+            if match:
+                for pnav in match.group("pnavs").split():
+                    project_name, _, version = pnav.rpartition("-")
+                    self._saved.add(Pin(ProjectName(project_name), Version(version)))
+                return self.Continue()
 
         if self._lock_style in (LockStyle.SOURCES, LockStyle.UNIVERSAL):
             match = re.search(r"Found link (?P<url>\S+)(?: \(from .*\))?, version: ", line)