pex 2.62.1__py2.py3-none-any.whl → 2.64.0__py2.py3-none-any.whl

pex/resolve/lockfile/create.py

@@ -23,7 +23,7 @@ from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pip.download_observer import DownloadObserver
 from pex.pip.tool import PackageIndexConfiguration
-from pex.pip.version import PipVersionValue
+from pex.pip.version import PipVersion, PipVersionValue
 from pex.resolve import lock_resolver, locker, resolvers
 from pex.resolve.configured_resolver import ConfiguredResolver
 from pex.resolve.downloads import ArtifactDownloader
@@ -41,15 +41,23 @@ from pex.resolve.locked_resolve import (
 from pex.resolve.locker import Locker
 from pex.resolve.lockfile.download_manager import DownloadManager
 from pex.resolve.lockfile.model import Lockfile
-from pex.resolve.lockfile.targets import calculate_download_input
+from pex.resolve.lockfile.targets import DownloadInput, calculate_download_input
 from pex.resolve.package_repository import ReposConfiguration
 from pex.resolve.pep_691.fingerprint_service import FingerprintService
 from pex.resolve.requirement_configuration import RequirementConfiguration
 from pex.resolve.resolved_requirement import Pin, ResolvedRequirement
 from pex.resolve.resolver_configuration import BuildConfiguration, PipConfiguration, ResolverVersion
 from pex.resolve.resolvers import Resolver
-from pex.resolver import BuildRequest, Downloaded, DownloadTarget, ResolveObserver, WheelBuilder
+from pex.resolver import (
+    BuildRequest,
+    Downloaded,
+    DownloadTarget,
+    Reports,
+    ResolveObserver,
+    WheelBuilder,
+)
 from pex.resolver import download_requests as pip_download_requests
+from pex.resolver import reports as pip_reports
 from pex.result import Error, try_
 from pex.targets import Target, Targets
 from pex.tracer import TRACER
@@ -277,6 +285,7 @@ class LockObserver(ResolveObserver):
     wheel_builder = attr.ib()  # type: WheelBuilder
     package_index_configuration = attr.ib()  # type: PackageIndexConfiguration
     max_parallel_jobs = attr.ib(default=None)  # type: Optional[int]
+    lock_is_via_pip_download = attr.ib(default=False)  # type: bool
     _analysis = attr.ib(factory=OrderedSet, eq=False)  # type: OrderedSet[_LockAnalysis]
 
     def observe_download(
@@ -299,6 +308,7 @@ class LockObserver(ResolveObserver):
                 ),
                 max_parallel_jobs=self.max_parallel_jobs,
             ),
+            lock_is_via_pip_download=self.lock_is_via_pip_download,
         )
         patch_set = locker.patch(universal_target=download_target.universal_target)
         observer = DownloadObserver(analyzer=analyzer, patch_set=patch_set)
@@ -454,7 +464,7 @@ class LockObserver(ResolveObserver):
         return tuple(
             LockedResolve.create(
                 resolved_requirements=resolved_requirements,
-                dist_metadatas=dist_metadatas_by_download_target[download_target],
+                project_metadatas=dist_metadatas_by_download_target[download_target],
                 fingerprinter=ArtifactDownloader(
                     resolver=self.resolver,
                     universal_target=download_target.universal_target,
@@ -476,6 +486,126 @@ class LockObserver(ResolveObserver):
             for download_target, resolved_requirements in resolved_requirements_by_download_target.items()
         )
 
+    def lock_reports(self, reports):
+        # type: (Reports) -> Tuple[LockedResolve, ...]
+
+        project_metadatas_by_download_target = {
+            report.download_target: tuple(report.metadata.values()) for report in reports
+        }
+        resolved_requirements_by_download_target = (
+            OrderedDict()
+        )  # type: OrderedDict[DownloadTarget, Tuple[ResolvedRequirement, ...]]
+        for analysis in self._analysis:
+            lock_result = analysis.analyzer.lock_result
+            resolved_requirements_by_download_target[
+                analysis.download_target
+            ] = lock_result.resolved_requirements
+
+        universal_targets = tuple(
+            download_target.universal_target
+            for download_target in resolved_requirements_by_download_target
+            if download_target.universal_target
+        )
+        return tuple(
+            LockedResolve.create(
+                resolved_requirements=resolved_requirements,
+                project_metadatas=project_metadatas_by_download_target[download_target],
+                fingerprinter=ArtifactDownloader(
+                    resolver=self.resolver,
+                    universal_target=download_target.universal_target,
+                    target=download_target.target,
+                    package_index_configuration=self.package_index_configuration,
+                    max_parallel_jobs=self.max_parallel_jobs,
+                ),
+                platform_tag=(
+                    None
+                    if self.lock_style == LockStyle.UNIVERSAL
+                    else download_target.target.platform.tag
+                ),
+                marker=(
+                    download_target.universal_target.marker()
+                    if download_target.universal_target and len(universal_targets) > 1
+                    else None
+                ),
+            )
+            for download_target, resolved_requirements in resolved_requirements_by_download_target.items()
+        )
+
+
+def _create_lock_pip_download(
+    download_input,  # type: DownloadInput
+    pip_configuration,  # type: PipConfiguration
+    network_configuration,  # type: NetworkConfiguration
+    download_dir,  # type: str
+    lock_observer,  # type: LockObserver
+    configured_resolver,  # type: ConfiguredResolver
+    dependency_configuration,  # type: DependencyConfiguration
+):
+    # type: (...) -> Union[Tuple[LockedResolve, ...], Error]
+    try:
+        downloaded = pip_download_requests(
+            requests=download_input.download_requests,
+            direct_requirements=download_input.direct_requirements,
+            allow_prereleases=pip_configuration.allow_prereleases,
+            transitive=pip_configuration.transitive,
+            repos_configuration=pip_configuration.repos_configuration,
+            resolver_version=pip_configuration.resolver_version,
+            network_configuration=network_configuration,
+            build_configuration=pip_configuration.build_configuration,
+            max_parallel_jobs=pip_configuration.max_jobs,
+            observer=lock_observer,
+            dest=download_dir,
+            pip_log=pip_configuration.log,
+            pip_version=pip_configuration.version,
+            resolver=configured_resolver,
+            use_pip_config=pip_configuration.use_pip_config,
+            extra_pip_requirements=pip_configuration.extra_requirements,
+            keyring_provider=pip_configuration.keyring_provider,
+            dependency_configuration=dependency_configuration,
+        )
+    except resolvers.ResolveError as e:
+        return Error(str(e))
+
+    with TRACER.timed("Creating lock from resolve"):
+        return lock_observer.lock(downloaded)
+
+
+def _create_lock_pip_reports(
+    download_input,  # type: DownloadInput
+    pip_configuration,  # type: PipConfiguration
+    network_configuration,  # type: NetworkConfiguration
+    lock_observer,  # type: LockObserver
+    configured_resolver,  # type: ConfiguredResolver
+    dependency_configuration,  # type: DependencyConfiguration
+):
+    # type: (...) -> Union[Tuple[LockedResolve, ...], Error]
+
+    try:
+        reports = pip_reports(
+            requests=download_input.download_requests,
+            direct_requirements=download_input.direct_requirements,
+            allow_prereleases=pip_configuration.allow_prereleases,
+            transitive=pip_configuration.transitive,
+            repos_configuration=pip_configuration.repos_configuration,
+            resolver_version=pip_configuration.resolver_version,
+            network_configuration=network_configuration,
+            build_configuration=pip_configuration.build_configuration,
+            max_parallel_jobs=pip_configuration.max_jobs,
+            observer=lock_observer,
+            pip_log=pip_configuration.log,
+            pip_version=pip_configuration.version,
+            resolver=configured_resolver,
+            use_pip_config=pip_configuration.use_pip_config,
+            extra_pip_requirements=pip_configuration.extra_requirements,
+            keyring_provider=pip_configuration.keyring_provider,
+            dependency_configuration=dependency_configuration,
+        )
+    except resolvers.ResolveError as e:
+        return Error(str(e))
+
+    with TRACER.timed("Creating lock from resolve"):
+        return lock_observer.lock_reports(reports)
+
 
 def create(
     lock_configuration,  # type: LockConfiguration
@@ -483,6 +613,7 @@ def create(
     targets,  # type: Targets
     pip_configuration,  # type: PipConfiguration
     dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+    avoid_downloads=False,  # type: bool
 ):
     # type: (...) -> Union[Lockfile, Error]
     """Create a lock file for the given resolve configurations."""
@@ -510,6 +641,9 @@ def create(
     )
 
     configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration)
+    lock_is_via_pip_download = (
+        pip_configuration.version is PipVersion.VENDORED or not avoid_downloads
+    )
     lock_observer = LockObserver(
         root_requirements=parsed_requirements,
         lock_style=lock_configuration.style,
@@ -522,6 +656,7 @@ def create(
         ),
         package_index_configuration=package_index_configuration,
         max_parallel_jobs=pip_configuration.max_jobs,
+        lock_is_via_pip_download=lock_is_via_pip_download,
     )
 
     download_dir = safe_mkdtemp()
@@ -533,32 +668,27 @@ def create(
             repos_configuration=pip_configuration.repos_configuration,
             universal_target=lock_configuration.universal_target,
         )
-    try:
-        downloaded = pip_download_requests(
-            requests=download_input.download_requests,
-            direct_requirements=download_input.direct_requirements,
-            allow_prereleases=pip_configuration.allow_prereleases,
-            transitive=pip_configuration.transitive,
-            repos_configuration=pip_configuration.repos_configuration,
-            resolver_version=pip_configuration.resolver_version,
+
+    locked_resolves = try_(
+        _create_lock_pip_download(
+            download_input=download_input,
+            pip_configuration=pip_configuration,
             network_configuration=network_configuration,
-            build_configuration=pip_configuration.build_configuration,
-            max_parallel_jobs=pip_configuration.max_jobs,
-            observer=lock_observer,
-            dest=download_dir,
-            pip_log=pip_configuration.log,
-            pip_version=pip_configuration.version,
-            resolver=configured_resolver,
-            use_pip_config=pip_configuration.use_pip_config,
-            extra_pip_requirements=pip_configuration.extra_requirements,
-            keyring_provider=pip_configuration.keyring_provider,
+            download_dir=download_dir,
+            lock_observer=lock_observer,
+            configured_resolver=configured_resolver,
             dependency_configuration=dependency_configuration,
         )
-    except resolvers.ResolveError as e:
-        return Error(str(e))
-
-    with TRACER.timed("Creating lock from resolve"):
-        locked_resolves = lock_observer.lock(downloaded)
+        if lock_is_via_pip_download
+        else _create_lock_pip_reports(
+            download_input=download_input,
+            pip_configuration=pip_configuration,
+            network_configuration=network_configuration,
+            lock_observer=lock_observer,
+            configured_resolver=configured_resolver,
+            dependency_configuration=dependency_configuration,
+        )
+    )
 
     with TRACER.timed("Indexing downloads"):
         create_lock_download_manager = CreateLockDownloadManager.create(

pex/resolve/lockfile/updater.py

@@ -252,6 +252,7 @@ class ResolveUpdater(object):
         lock_configuration,  # type: LockConfiguration
         pip_configuration,  # type: PipConfiguration
         dependency_configuration,  # type: DependencyConfiguration
+        avoid_downloads,  # type: bool
     ):
         # type: (...) -> Union[ResolveUpdater, Error]
 
@@ -310,6 +311,7 @@ class ResolveUpdater(object):
             lock_configuration=lock_configuration,
             pip_configuration=pip_configuration,
             dependency_configuration=dependency_configuration,
+            avoid_downloads=avoid_downloads,
         )
 
     @classmethod
@@ -322,6 +324,7 @@ class ResolveUpdater(object):
         lock_configuration,  # type: LockConfiguration
         pip_configuration,  # type: PipConfiguration
         dependency_configuration,  # type: DependencyConfiguration
+        avoid_downloads,  # type: bool
     ):
         # type: (...) -> ResolveUpdater
 
@@ -362,6 +365,7 @@ class ResolveUpdater(object):
             lock_configuration=lock_configuration,
             pip_configuration=pip_configuration,
             dependency_configuration=dependency_configuration,
+            avoid_downloads=avoid_downloads,
         )
 
     requirement_configuration = attr.ib()  # type: RequirementConfiguration
@@ -377,6 +381,7 @@ class ResolveUpdater(object):
     update_requirements_by_project_name = attr.ib(
         factory=dict
     )  # type: Mapping[ProjectName, Requirement]
+    avoid_downloads = attr.ib(default=False)
 
     def iter_updated_requirements(self):
         # type: () -> Iterator[Requirement]
@@ -502,6 +507,7 @@ class ResolveUpdater(object):
                             targets=Targets.from_target(target),
                             pip_configuration=self.pip_configuration,
                             dependency_configuration=self.dependency_configuration,
+                            avoid_downloads=self.avoid_downloads,
                         )
                     )
                     assert 1 == len(updated_lock_file.locked_resolves)
@@ -691,6 +697,7 @@ class LockUpdater(object):
         use_pip_config,  # type: bool
         dependency_configuration,  # type: DependencyConfiguration
         pip_log,  # type: Optional[PipLog]
+        avoid_downloads,  # type: bool
     ):
         # type: (...) -> LockUpdater
 
@@ -711,12 +718,14 @@ class LockUpdater(object):
             lock_configuration=lock_file.configuration,
             pip_configuration=pip_configuration,
             dependency_configuration=dependency_configuration,
+            avoid_downloads=avoid_downloads,
         )
 
     lock_file = attr.ib()  # type: Lockfile
     lock_configuration = attr.ib()  # type: LockConfiguration
     pip_configuration = attr.ib()  # type: PipConfiguration
     dependency_configuration = attr.ib()  # type: DependencyConfiguration
+    avoid_downloads = attr.ib()  # type: bool
 
     def sync(
         self,
@@ -738,6 +747,7 @@ class LockUpdater(object):
                 lock_configuration=self.lock_configuration,
                 pip_configuration=self.pip_configuration,
                 dependency_configuration=self.dependency_configuration,
+                avoid_downloads=self.avoid_downloads,
             )
         )
         return self._perform_update(
@@ -773,6 +783,7 @@ class LockUpdater(object):
             lock_configuration=self.lock_configuration,
             pip_configuration=self.pip_configuration,
             dependency_configuration=self.dependency_configuration,
+            avoid_downloads=self.avoid_downloads,
         )
         return self._perform_update(
             update_requests=update_requests,

pex/resolver.py

@@ -8,6 +8,7 @@ import functools
 import glob
 import hashlib
 import itertools
+import json
 import os
 import tarfile
 import zipfile
@@ -30,6 +31,7 @@ from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import (
     DistMetadata,
     Distribution,
+    ProjectMetadata,
     Requirement,
     is_tar_sdist,
     is_wheel,
@@ -43,6 +45,7 @@ from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
 from pex.pep_425 import CompatibilityTags
 from pex.pep_427 import InstallableType, WheelError, install_wheel_chroot
+from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pip.download_observer import DownloadObserver
 from pex.pip.installation import get_pip
@@ -63,6 +66,7 @@ from pex.resolve.resolvers import (
 )
 from pex.resolve.target_system import TargetSystem, UniversalTarget
 from pex.targets import AbbreviatedPlatform, CompletePlatform, LocalInterpreter, Target, Targets
+from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.packaging.tags import Tag
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
@@ -239,7 +243,52 @@ class PipLogManager(object):
 
 
 @attr.s(frozen=True)
-class _DownloadSession(object):
+class Report(object):
+    @classmethod
+    def parse(
+        cls,
+        download_target,  # type: DownloadTarget
+        report,  # type: str
+    ):
+        # type: (...) -> Report
+
+        with open(report) as fp:
+            data = json.load(fp)
+
+        project_metadata = []  # type: List[ProjectMetadata]
+        for distribution in data["install"]:
+            metadata = distribution["metadata"]
+            project_metadata.append(
+                ProjectMetadata(
+                    project_name=ProjectName(metadata["name"]),
+                    version=Version(metadata["version"]),
+                    requires_python=SpecifierSet(metadata.get("requires_python", "")),
+                    requires_dists=tuple(
+                        Requirement.parse(requirement)
+                        for requirement in metadata.get("requires_dist", ())
+                    ),
+                )
+            )
+        return cls(
+            download_target=download_target,
+            metadata={metadata.project_name: metadata for metadata in project_metadata},
+        )
+
+    download_target = attr.ib()  # type: DownloadTarget
+    metadata = attr.ib()  # type: Mapping[ProjectName, ProjectMetadata]
+
+
+@attr.s(frozen=True)
+class Reports(object):
+    reports = attr.ib(default=())  # type: Tuple[Report, ...]
+
+    def __iter__(self):
+        # type: () -> Iterator[Report]
+        return iter(self.reports)
+
+
+@attr.s(frozen=True)
+class _PipSession(object):
     requests = attr.ib(converter=_uniqued_download_requests)  # type: Tuple[DownloadRequest, ...]
     direct_requirements = attr.ib()  # type: Iterable[ParsedRequirement]
     allow_prereleases = attr.ib(default=False)  # type: bool
@@ -266,8 +315,108 @@ class _DownloadSession(object):
                         pip_version=self.pip_version,
                     )
 
+    def generate_reports(self, max_parallel_jobs=None):
+        # type: (Optional[int]) -> Reports
+
+        if not self.requests or not any(request.has_requirements for request in self.requests):
+            # Nothing to report.
+            return Reports()
+
+        dest = safe_mkdtemp(
+            prefix="resolver_report.", dir=safe_mkdir(CacheDir.DOWNLOADS.path(".tmp"))
+        )
+
+        log_manager = PipLogManager.create(
+            self.pip_log,
+            download_targets=tuple(request.download_target for request in self.requests),
+        )
+        if self.pip_log and not self.pip_log.user_specified:
+            TRACER.log(
+                "Preserving `pip install --dry-run` log at {log_path}".format(
+                    log_path=self.pip_log.path
+                ),
+                V=ENV.PEX_VERBOSE,
+            )
+
+        spawn_report = functools.partial(
+            self._spawn_report, resolved_target_dir=dest, log_manager=log_manager
+        )
+        with TRACER.timed(
+            "Resolving for:\n  {}".format(
+                "\n  ".join(request.render_description() for request in self.requests)
+            )
+        ):
+            try:
+                return Reports(
+                    reports=tuple(
+                        Report.parse(download_request.download_target, report)
+                        for download_request, report in zip(
+                            self.requests,
+                            execute_parallel(
+                                inputs=self.requests,
+                                spawn_func=spawn_report,
+                                error_handler=Raise[DownloadRequest, str](Unsatisfiable),
+                                max_jobs=max_parallel_jobs,
+                            ),
+                        )
+                    )
+                )
+            finally:
+                log_manager.finalize_log()
+
+    def _spawn_report(
+        self,
+        request,  # type: DownloadRequest
+        resolved_target_dir,  # type: str
+        log_manager,  # type: PipLogManager
+    ):
+        # type: (...) -> SpawnedJob[str]
+
+        report_dir = safe_mkdir(
+            os.path.join(resolved_target_dir, request.download_target.id(complete=True))
+        )
+        report = os.path.join(report_dir, "pip-report.json")
+        download_target = request.download_target
+        observer = (
+            self.observer.observe_download(
+                download_target=download_target, download_dir=resolved_target_dir
+            )
+            if self.observer
+            else None
+        )
+
+        target = download_target.target
+
+        download_job = get_pip(
+            interpreter=target.get_interpreter(),
+            version=self.pip_version,
+            resolver=self.resolver,
+            extra_requirements=(
+                self.package_index_configuration.extra_pip_requirements
+                if self.package_index_configuration
+                else ()
+            ),
+        ).spawn_report(
+            report_path=report,
+            requirements=request.requirements,
+            requirement_files=request.requirement_files,
+            constraint_files=request.constraint_files,
+            allow_prereleases=self.allow_prereleases,
+            transitive=self.transitive,
+            target=target,
+            package_index_configuration=self.package_index_configuration,
+            build_configuration=self.build_configuration,
+            observer=observer,
+            dependency_configuration=self.dependency_configuration,
+            universal_target=download_target.universal_target,
+            log=log_manager.get_log(download_target),
+        )
+
+        return SpawnedJob.wait(job=download_job, result=report)
+
     def download_distributions(self, dest=None, max_parallel_jobs=None):
         # type: (...) -> List[DownloadResult]
+
         if not self.requests or not any(request.has_requirements for request in self.requests):
             # Nothing to resolve.
             return []
@@ -1515,7 +1664,7 @@ def _download_internal(
 ):
     # type: (...) -> Tuple[List[BuildRequest], List[DownloadResult]]
 
-    download_session = _DownloadSession(
+    pip_session = _PipSession(
         requests=requests,
         direct_requirements=direct_requirements,
         allow_prereleases=allow_prereleases,
@@ -1529,8 +1678,8 @@ def _download_internal(
         dependency_configuration=dependency_configuration,
     )
 
-    local_projects = list(download_session.iter_local_projects())
-    download_results = download_session.download_distributions(
+    local_projects = list(pip_session.iter_local_projects())
+    download_results = pip_session.download_distributions(
         dest=dest, max_parallel_jobs=max_parallel_jobs
     )
     return local_projects, download_results
@@ -1780,3 +1929,51 @@ def download_requests(
             )
 
     return Downloaded(local_distributions=tuple(local_distributions))
+
+
+def reports(
+    requests,  # type: Tuple[DownloadRequest, ...]
+    direct_requirements,  # type: Tuple[ParsedRequirement, ...]
+    allow_prereleases=False,  # type: bool
+    transitive=True,  # type: bool
+    repos_configuration=ReposConfiguration(),  # type: ReposConfiguration
+    resolver_version=None,  # type: Optional[ResolverVersion.Value]
+    network_configuration=None,  # type: Optional[NetworkConfiguration]
+    build_configuration=BuildConfiguration(),  # type: BuildConfiguration
+    max_parallel_jobs=None,  # type: Optional[int]
+    observer=None,  # type: Optional[ResolveObserver]
+    pip_log=None,  # type: Optional[PipLog]
+    pip_version=None,  # type: Optional[PipVersionValue]
+    resolver=None,  # type: Optional[Resolver]
+    use_pip_config=False,  # type: bool
+    extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+    keyring_provider=None,  # type: Optional[str]
+    dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+):
+    # type: (...) -> Reports
+
+    package_index_configuration = PackageIndexConfiguration.create(
+        pip_version=pip_version,
+        resolver_version=resolver_version,
+        repos_configuration=repos_configuration,
+        network_configuration=network_configuration,
+        use_pip_config=use_pip_config,
+        extra_pip_requirements=extra_pip_requirements,
+        keyring_provider=keyring_provider,
+    )
+
+    pip_session = _PipSession(
+        requests=requests,
+        direct_requirements=direct_requirements,
+        allow_prereleases=allow_prereleases,
+        transitive=transitive,
+        package_index_configuration=package_index_configuration,
+        build_configuration=build_configuration,
+        observer=observer,
+        pip_log=pip_log,
+        pip_version=pip_version,
+        resolver=resolver,
+        dependency_configuration=dependency_configuration,
+    )
+
+    return pip_session.generate_reports(max_parallel_jobs=max_parallel_jobs)