pex 2.66.0__py2.py3-none-any.whl → 2.67.0__py2.py3-none-any.whl

pex/resolve/resolver_options.py

@@ -56,9 +56,10 @@ class _HandleTransitiveAction(Action):
 class _ResolveVenvAction(Action):
     def __init__(self, *args, **kwargs):
         kwargs["nargs"] = "?"
-        super(_ResolveVenvAction, self).__init__(*args, **kwargs)
+        super(_ResolveVenvAction, self).__init__(*args, default=[], **kwargs)
 
     def __call__(self, parser, namespace, value, option_str=None):
+        venvs = getattr(namespace, self.dest)
         if value:
             if not os.path.exists(value):
                 raise ArgumentError(
@@ -86,7 +87,7 @@ class _ResolveVenvAction(Action):
                         "path.".format(option=option_str, value=value)
                     ),
                 )
-            setattr(namespace, self.dest, venv)
+            venvs.append(venv)
         else:
             current_venv = Virtualenv.enclosing(python=sys.executable)
             if not current_venv:
@@ -105,7 +106,7 @@ class _ResolveVenvAction(Action):
                             )
                         ),
                     )
-            setattr(namespace, self.dest, current_venv)
+            venvs.append(current_venv)
 
 
 def register(
@@ -320,14 +321,18 @@ def register(
     if include_venv_repository:
         repository_choice.add_argument(
             "--venv-repository",
-            dest="venv_repository",
+            dest="venv_repositories",
             action=_ResolveVenvAction,
-            type=str,
             help=(
                 "Resolve requirements from the given virtual environment instead of from "
                 "--index servers, --find-links repos or a --lock file. The virtual environment to "
                 "resolve from can be specified as the path to the venv or the path of its"
-                "interpreter. If no value is specified, the current active venv is used."
+                "interpreter. If no value is specified, the current active venv is used. Multiple "
+                "virtual environments may be specified via multiple --venv-repository options and "
+                "the resolve will be the combined results. Each virtual environment will be "
+                "resolved from individually and must contain the full transitive closure of "
+                "requirements. This allows for creating a multi-platform PEX by specifying "
+                "multiple virtual environments; say one for Python 3.12 and one for Python 3.13."
             ),
         )
 
@@ -803,9 +808,9 @@ def configure(
             sdists=tuple(sdists), wheels=tuple(wheels), pip_configuration=pip_configuration
         )
 
-    venv = getattr(options, "venv_repository", None)
-    if venv:
-        return VenvRepositoryConfiguration(venv=venv, pip_configuration=pip_configuration)
+    venvs = getattr(options, "venv_repositories", None)
+    if venvs:
+        return VenvRepositoryConfiguration(venvs=tuple(venvs), pip_configuration=pip_configuration)
 
     if pylock:
         return PylockRepositoryConfiguration(

pex/resolve/venv_resolver.py

@@ -5,14 +5,13 @@ from __future__ import absolute_import
 
 import functools
 import hashlib
-import itertools
 import os
 from collections import defaultdict, deque
 
 from pex import pex_warnings
 from pex.atomic_directory import atomic_directory
 from pex.cache.dirs import CacheDir, InstalledWheelDir
-from pex.common import safe_relative_symlink
+from pex.common import pluralize, safe_relative_symlink
 from pex.compatibility import commonpath
 from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import (
@@ -26,7 +25,6 @@ from pex.dist_metadata import (
 from pex.exceptions import production_assert, reportable_unexpected_error_msg
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.installed_wheel import InstalledWheel
-from pex.interpreter import PythonInterpreter
 from pex.jobs import DEFAULT_MAX_JOBS, iter_map_parallel
 from pex.orderedset import OrderedSet
 from pex.pep_376 import Record
@@ -47,7 +45,18 @@ from pex.wheel import WHEEL, Wheel
 from pex.whl import repacked_whl
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Deque, FrozenSet, Iterable, Iterator, List, Mapping, Set, Union
+    from typing import (
+        DefaultDict,
+        Deque,
+        FrozenSet,
+        Iterable,
+        Iterator,
+        List,
+        Mapping,
+        Set,
+        Tuple,
+        Union,
+    )
 
     import attr  # vendor:skip
 else:
@@ -96,12 +105,14 @@ def _normalize_record(
 
 
 def _install_distribution(
-    interpreter,  # type: PythonInterpreter
+    venv_distribution,  # type: VenvDistribution
     result_type,  # type: InstallableType.Value
     use_system_time,  # type: bool
-    distribution,  # type: Distribution
 ):
-    # type: (...) -> FingerprintedDistribution
+    # type: (...) -> ResolvedDistribution
+
+    interpreter = venv_distribution.target.interpreter
+    distribution = venv_distribution.distribution
 
     production_assert(distribution.type is DistributionType.INSTALLED)
     production_assert(distribution.metadata.files.metadata.type is MetadataType.DIST_INFO)
@@ -160,31 +171,86 @@ def _install_distribution(
         raise AssertionError(reportable_unexpected_error_msg())
 
     if result_type is InstallableType.INSTALLED_WHEEL_CHROOT:
-        return FingerprintedDistribution(
-            distribution=Distribution.load(installed_wheel.prefix_dir),
-            fingerprint=installed_wheel.fingerprint,
+        return ResolvedDistribution(
+            target=venv_distribution.target,
+            fingerprinted_distribution=FingerprintedDistribution(
+                distribution=Distribution.load(installed_wheel.prefix_dir),
+                fingerprint=installed_wheel.fingerprint,
+            ),
+            direct_requirements=venv_distribution.direct_requirements,
         )
-    return repacked_whl(
-        installed_wheel, fingerprint=installed_wheel.fingerprint, use_system_time=use_system_time
+
+    return ResolvedDistribution(
+        target=venv_distribution.target,
+        fingerprinted_distribution=repacked_whl(
+            installed_wheel,
+            fingerprint=installed_wheel.fingerprint,
+            use_system_time=use_system_time,
+        ),
+        direct_requirements=venv_distribution.direct_requirements,
     )
 
 
+@attr.s(frozen=True)
+class VenvDistribution(object):
+    target = attr.ib()  # type: LocalInterpreter
+    distribution = attr.ib()  # type: Distribution
+    direct_requirements = attr.ib()  # type: Iterable[Requirement]
+
+
 def _install_venv_distributions(
-    venv,  # type: Virtualenv
-    distributions,  # type: Iterable[Distribution]
+    venv_resolve_results,  # type: Iterable[VenvResolveResult]
     max_install_jobs=DEFAULT_MAX_JOBS,  # type: int
     result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
     use_system_time=False,  # type: bool
 ):
-    # type: (...) -> Iterator[FingerprintedDistribution]
+    # type: (...) -> Iterator[ResolvedDistribution]
+
+    seen = set()  # type: Set[str]
+
+    venv_distributions = []  # type: List[VenvDistribution]
+    for venv_resolve_result in venv_resolve_results:
+        target = venv_resolve_result.target
+        direct_requirements = venv_resolve_result.direct_requirements_by_project_name
+        for re_resolved_distribution in venv_resolve_result.re_resolved_distributions:
+            wheel_file_name = Wheel.from_distribution(
+                re_resolved_distribution.distribution
+            ).wheel_file_name
+            if wheel_file_name in seen:
+                continue
+
+            seen.add(wheel_file_name)
+            yield ResolvedDistribution(
+                target=target,
+                fingerprinted_distribution=re_resolved_distribution,
+                direct_requirements=direct_requirements.get(
+                    re_resolved_distribution.project_name, ()
+                ),
+            )
+        for venv_distribution in venv_resolve_result.venv_distributions:
+            wheel_file_name = Wheel.from_distribution(venv_distribution).wheel_file_name
+            if wheel_file_name in seen:
+                continue
+
+            seen.add(wheel_file_name)
+            venv_distributions.append(
+                VenvDistribution(
+                    target=target,
+                    distribution=venv_distribution,
+                    direct_requirements=direct_requirements.get(
+                        venv_distribution.metadata.project_name, ()
+                    ),
+                )
+            )
 
-    return iter_map_parallel(
-        inputs=distributions,
+    for resolved_distribution in iter_map_parallel(
+        inputs=venv_distributions,
         function=functools.partial(
-            _install_distribution, venv.interpreter, result_type, use_system_time
+            _install_distribution, result_type=result_type, use_system_time=use_system_time
         ),
         max_jobs=max_install_jobs,
-    )
+    ):
+        yield resolved_distribution
 
 
 @attr.s(frozen=True)
@@ -359,27 +425,32 @@ def _resolve_distributions(
             )
 
 
-def resolve_from_venv(
-    targets,  # type: Targets
+@attr.s(frozen=True)
+class VenvResolveResult(object):
+    venv = attr.ib()  # type: Virtualenv
+    venv_distributions = attr.ib()  # type: Tuple[Distribution, ...]
+    re_resolved_distributions = attr.ib()  # type: Tuple[FingerprintedDistribution, ...]
+    direct_requirements_by_project_name = attr.ib(
+        eq=False
+    )  # type: Mapping[ProjectName, Iterable[Requirement]]
+
+    @property
+    def target(self):
+        # type: () -> LocalInterpreter
+        return LocalInterpreter.create(self.venv.interpreter)
+
+
+def _resolve_from_venv(
     venv,  # type: Virtualenv
-    requirement_configuration=RequirementConfiguration(),  # type: RequirementConfiguration
-    pip_configuration=PipConfiguration(),  # type: PipConfiguration
-    compile=False,  # type: bool
-    ignore_errors=False,  # type: bool
-    result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
-    dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+    requirement_configuration,  # type: RequirementConfiguration
+    pip_configuration,  # type: PipConfiguration
+    compile,  # type: bool
+    ignore_errors,  # type: bool
+    result_type,  # type: InstallableType.Value
+    dependency_configuration,  # type: DependencyConfiguration
 ):
-    # type: (...) -> Union[ResolveResult, Error]
-
+    # type: (...) -> Union[VenvResolveResult, Error]
     target = LocalInterpreter.create(venv.interpreter)
-    if not targets.is_empty:
-        return Error(
-            "You configured custom targets via --python, --interpreter-constraint, --platform or "
-            "--complete-platform but custom targets are not allowed when resolving from a virtual "
-            "environment.\n"
-            "For such resolves, the supported target is implicitly the one matching the venv "
-            "interpreter; in this case: {target}.".format(target=target.render_description())
-        )
 
     if pip_configuration.version:
         compatible_pip_version = (
@@ -487,27 +558,90 @@ def resolve_from_venv(
             dist.fingerprinted_distribution for dist in result.distributions
         )
 
+    return VenvResolveResult(
+        venv=venv,
+        venv_distributions=tuple(venv_distributions),
+        re_resolved_distributions=tuple(fingerprinted_distributions),
+        direct_requirements_by_project_name=direct_requirements_by_project_name,
+    )
+
+
+def resolve_from_venvs(
+    targets,  # type: Targets
+    venvs,  # type: Tuple[Virtualenv, ...]
+    requirement_configuration=RequirementConfiguration(),  # type: RequirementConfiguration
+    pip_configuration=PipConfiguration(),  # type: PipConfiguration
+    compile=False,  # type: bool
+    ignore_errors=False,  # type: bool
+    result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
+    dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+):
+    # type: (...) -> Union[ResolveResult, Error]
+
+    if not targets.is_empty:
+        return Error(
+            "You configured custom targets via --python, --interpreter-constraint, --platform or "
+            "--complete-platform but custom targets are not allowed when resolving from {venvs}.\n"
+            "For such resolves, the supported target is implicitly the one matching the venv "
+            "{interpreters}; in this case:{targets}.".format(
+                venvs="a virtual environment" if len(venvs) == 1 else "virtual environments",
+                interpreters=pluralize(venvs, "interpreter"),
+                targets=(
+                    " {target}".format(
+                        target=LocalInterpreter.create(venvs[0].interpreter).render_description()
+                    )
+                    if len(venvs) == 1
+                    else "\n  {targets}".format(
+                        targets="\n  ".join(
+                            LocalInterpreter.create(venv.interpreter).render_description()
+                            for venv in venvs
+                        )
+                    )
+                ),
+            )
+        )
+
+    errors = []  # type: List[Error]
+    venv_resolve_results = []  # type: List[VenvResolveResult]
+    for result in iter_map_parallel(
+        venvs,
+        functools.partial(
+            _resolve_from_venv,
+            requirement_configuration=requirement_configuration,
+            pip_configuration=pip_configuration,
+            compile=compile,
+            ignore_errors=ignore_errors,
+            result_type=result_type,
+            dependency_configuration=dependency_configuration,
+        ),
+    ):
+        if isinstance(result, Error):
+            errors.append(result)
+        else:
+            venv_resolve_results.append(result)
+
+    if len(errors) == 1:
+        return errors[0]
+    elif errors:
+        return Error(
+            "Failed to resolve from {count} of {total} virtual environments:\n{failures}".format(
+                count=len(errors),
+                total=len(venvs),
+                failures="\n".join(
+                    "{index}. {error}".format(index=index, error=error)
+                    for index, error in enumerate(errors, start=1)
+                ),
+            )
+        )
+
     return ResolveResult(
         dependency_configuration=dependency_configuration,
         distributions=tuple(
-            ResolvedDistribution(
-                target=target,
-                fingerprinted_distribution=fingerprinted_distribution,
-                direct_requirements=direct_requirements_by_project_name[
-                    fingerprinted_distribution.project_name
-                ],
-            )
-            for fingerprinted_distribution in itertools.chain(
-                list(
-                    _install_venv_distributions(
-                        venv=venv,
-                        distributions=venv_distributions,
-                        max_install_jobs=pip_configuration.max_jobs,
-                        result_type=result_type,
-                        use_system_time=True,
-                    )
-                ),
-                fingerprinted_distributions,
+            _install_venv_distributions(
+                venv_resolve_results,
+                max_install_jobs=pip_configuration.max_jobs,
+                result_type=result_type,
+                use_system_time=True,
             )
         ),
         type=result_type,

pex/resolver.py

@@ -65,6 +65,7 @@ from pex.resolve.resolvers import (
     check_resolve,
 )
 from pex.resolve.target_system import TargetSystem, UniversalTarget
+from pex.result import Error
 from pex.targets import AbbreviatedPlatform, CompletePlatform, LocalInterpreter, Target, Targets
 from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.packaging.tags import Tag
@@ -309,10 +310,7 @@ class _PipSession(object):
             if isinstance(requirement, LocalProjectRequirement):
                 for request in self.requests:
                     yield BuildRequest.for_directory(
-                        target=request.target,
-                        source_path=requirement.path,
-                        resolver=self.resolver,
-                        pip_version=self.pip_version,
+                        target=request.target, source_path=requirement.path
                     )
 
     def generate_reports(self, max_parallel_jobs=None):
@@ -594,12 +592,17 @@ def _fingerprint_directory(path):
     return CacheHelper.dir_hash(path, digest=_hasher())
 
 
+class BuildError(Exception):
+    pass
+
+
 def _fingerprint_local_project(
     path,  # type: str
     target,  # type: Target
     resolver=None,  # type: Optional[Resolver]
     pip_version=None,  # type: Optional[PipVersionValue]
 ):
+    # type: (...) -> str
     if resolver:
         build_system_resolver = resolver
     else:
@@ -607,17 +610,20 @@ def _fingerprint_local_project(
 
         build_system_resolver = ConfiguredResolver.default()
 
-    return digest_local_project(
+    hasher = _hasher()
+    result = digest_local_project(
         directory=path,
-        digest=_hasher(),
+        digest=hasher,
         target=target,
         resolver=build_system_resolver,
         pip_version=pip_version,
     )
-
-
-class BuildError(Exception):
-    pass
+    if isinstance(result, Error):
+        raise BuildError(
+            "Failed to create an sdist for hashing from the local project at {path}: "
+            "{err}".format(path=path, err=result)
+        )
+    return hasher.hexdigest()
 
 
 def _as_download_target(target):
@@ -649,27 +655,19 @@ class BuildRequest(object):
         target,  # type: Union[DownloadTarget, Target]
         source_path,  # type: str
         subdirectory=None,  # type: Optional[str]
-        resolver=None,  # type: Optional[Resolver]
-        pip_version=None,  # type: Optional[PipVersionValue]
     ):
         # type: (...) -> BuildRequest
         download_target = _as_download_target(target)
-        fingerprint = _fingerprint_local_project(
-            path=source_path,
-            target=download_target.target,
-            resolver=resolver,
-            pip_version=pip_version,
-        )
         return cls(
             download_target=download_target,
             source_path=source_path,
-            fingerprint=fingerprint,
+            fingerprint=None,
             subdirectory=subdirectory,
         )
 
     download_target = attr.ib(converter=_as_download_target)  # type: DownloadTarget
     source_path = attr.ib()  # type: str
-    fingerprint = attr.ib()  # type: str
+    fingerprint = attr.ib()  # type: Optional[str]
     subdirectory = attr.ib()  # type: Optional[str]
 
     @property
@@ -724,12 +722,16 @@ class BuildResult(object):
         source_path=None,  # type: Optional[str]
     ):
         # type: (...) -> BuildResult
-        built_wheel = BuiltWheelDir.create(
-            sdist=source_path or build_request.source_path,
-            fingerprint=build_request.fingerprint,
-            target=build_request.target,
-        )
-        return cls(request=build_request, atomic_dir=AtomicDirectory(built_wheel.dist_dir))
+        if build_request.fingerprint:
+            built_wheel = BuiltWheelDir.create(
+                sdist=source_path or build_request.source_path,
+                fingerprint=build_request.fingerprint,
+                target=build_request.target,
+            )
+            target_dir = built_wheel.dist_dir
+        else:
+            target_dir = os.path.join(safe_mkdtemp(), "build")
+        return cls(request=build_request, atomic_dir=AtomicDirectory(target_dir))
 
     request = attr.ib()  # type: BuildRequest
     _atomic_dir = attr.ib()  # type: AtomicDirectory
@@ -1290,14 +1292,7 @@ class BuildAndInstallRequest(object):
                 if is_wheel(dist_path):
                     to_install.add(InstallRequest.create(install_request.target, dist_path))
                 elif os.path.isdir(dist_path):
-                    to_build.add(
-                        BuildRequest.for_directory(
-                            install_request.target,
-                            dist_path,
-                            resolver=self._resolver,
-                            pip_version=self._pip_version,
-                        )
-                    )
+                    to_build.add(BuildRequest.for_directory(install_request.target, dist_path))
                 else:
                     to_build.add(BuildRequest.for_file(install_request.target, dist_path))
             already_analyzed.add(metadata.project_name)
@@ -1907,11 +1902,21 @@ def download_requests(
     def add_build_requests(requests):
         # type: (Iterable[BuildRequest]) -> None
         for request in requests:
+            if request.fingerprint:
+                fingerprint = request.fingerprint
+            else:
+                production_assert(os.path.isdir(request.source_path))
+                fingerprint = _fingerprint_local_project(
+                    path=request.source_path,
+                    target=request.target,
+                    resolver=resolver,
+                    pip_version=pip_version,
+                )
             local_distributions.append(
                 LocalDistribution(
                     download_target=request.download_target,
                     path=request.source_path,
-                    fingerprint=request.fingerprint,
+                    fingerprint=fingerprint,
                     subdirectory=request.subdirectory,
                 )
             )

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.66.0"
+__version__ = "2.67.0"

{pex-2.66.0.dist-info → pex-2.67.0.dist-info}/METADATA

@@ -1,15 +1,15 @@
 Metadata-Version: 2.4
 Name: pex
-Version: 2.66.0
+Version: 2.67.0
 Summary: The PEX packaging toolchain.
 Home-page: https://github.com/pex-tool/pex
-Download-URL: https://github.com/pex-tool/pex/releases/download/v2.66.0/pex
+Download-URL: https://github.com/pex-tool/pex/releases/download/v2.67.0/pex
 Author: The PEX developers
 Author-email: developers@pex-tool.org
 License-Expression: Apache-2.0
-Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.66.0/CHANGES.md
+Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.67.0/CHANGES.md
 Project-URL: Documentation, https://docs.pex-tool.org/
-Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.66.0
+Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.67.0
 Keywords: package,executable,virtualenv,lock,freeze
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers