pex 2.66.0__tar.gz → 2.67.0__tar.gz

{pex-2.66.0 → pex-2.67.0}/CHANGES.md

@@ -1,5 +1,21 @@
 # Release Notes
 
+## 2.67.0
+
+This release adds support for specifying multiple `--venv-repository`s when building a PEX. This
+allows creating multi-platform PEXes from multiple venvs that all satisfy a resolve, but for
+different interpreters.
+
+* Multi-platform PEXes via multiple `--venv-repository`s. (#2977)
+
+## 2.66.1
+
+This release improves upon the local project directory hashing fix in [2.61.1](#2611) by
+avoiding the hashing altogether unless creating a lock, where the resulting fingerprint is
+needed.
+
+* Avoid fingerprinting local projects. (#2975)
+
 ## 2.66.0
 
 This release adds support for `--pip-version 25.3`.

{pex-2.66.0 → pex-2.67.0}/PKG-INFO

@@ -1,15 +1,15 @@
 Metadata-Version: 2.4
 Name: pex
-Version: 2.66.0
+Version: 2.67.0
 Summary: The PEX packaging toolchain.
 Home-page: https://github.com/pex-tool/pex
-Download-URL: https://github.com/pex-tool/pex/releases/download/v2.66.0/pex
+Download-URL: https://github.com/pex-tool/pex/releases/download/v2.67.0/pex
 Author: The PEX developers
 Author-email: developers@pex-tool.org
 License-Expression: Apache-2.0
-Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.66.0/CHANGES.md
+Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.67.0/CHANGES.md
 Project-URL: Documentation, https://docs.pex-tool.org/
-Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.66.0
+Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.67.0
 Keywords: package,executable,virtualenv,lock,freeze
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers

{pex-2.66.0 → pex-2.67.0}/pex/jobs.py

@@ -724,6 +724,19 @@ def iter_map_parallel(
     if not input_items:
         return
 
+    # We want each of the job slots to process MULTIPROCESSING_DEFAULT_MIN_AVERAGE_LOAD on
+    # average in order to overcome multiprocessing overheads. If we don't need at least 2 slots, we
+    # are unlikely to get any boost from multiprocessing.
+    needed_slots = len(input_items) // min_average_load
+    if needed_slots < 2:
+        for item in input_items:
+            yield function(item)
+        return
+
+    # Of course, if there are fewer available cores than that or the user has pinned max jobs lower,
+    # we clamp to that.
+    pool_size = min(needed_slots, _sanitize_max_jobs(max_jobs))
+
     if costing_function is not None:
         # We ensure no job slot is so unlucky as to get all the biggest jobs and thus become an
         # un-necessarily long pole by sorting based on cost. Some examples to illustrate the effect
@@ -740,12 +753,6 @@ def iter_map_parallel(
         #
         input_items.sort(key=costing_function, reverse=True)
 
-    # We want each of the job slots above to process MULTIPROCESSING_DEFAULT_MIN_AVERAGE_LOAD on
-    # average in order to overcome multiprocessing overheads. Of course, if there are fewer
-    # available cores than that or the user has pinned max jobs lower, we clamp to that. Finally, we
-    # always want at least two slots to ensure we process input items in parallel.
-    pool_size = max(2, min(len(input_items) // min_average_load, _sanitize_max_jobs(max_jobs)))
-
     apply_function = functools.partial(_apply_function, function)
 
     slots = defaultdict(list)  # type: DefaultDict[int, List[float]]

{pex-2.66.0 → pex-2.67.0}/pex/resolve/configured_resolve.py

@@ -20,7 +20,7 @@ from pex.resolve.resolver_configuration import (
     VenvRepositoryConfiguration,
 )
 from pex.resolve.resolvers import ResolveResult
-from pex.resolve.venv_resolver import resolve_from_venv
+from pex.resolve.venv_resolver import resolve_from_venvs
 from pex.resolver import resolve as resolve_via_pip
 from pex.result import try_
 from pex.targets import Targets
@@ -143,14 +143,22 @@ def resolve(
             )
     elif isinstance(resolver_configuration, VenvRepositoryConfiguration):
         with TRACER.timed(
-            "Resolving requirements from venv at {venv}.".format(
-                venv=resolver_configuration.venv.venv_dir
+            "Resolving requirements from {count} {venvs} at:{paths}.".format(
+                count=len(resolver_configuration.venvs),
+                venvs=pluralize(resolver_configuration.venvs, "venv"),
+                paths=(
+                    " {venv}".format(venv=resolver_configuration.venvs[0].venv_dir)
+                    if len(resolver_configuration.venvs) == 1
+                    else "\n  {venvs}".format(
+                        venvs="\n  ".join(venv.venv_dir for venv in resolver_configuration.venvs)
+                    )
+                ),
             )
         ):
             return try_(
-                resolve_from_venv(
+                resolve_from_venvs(
                     targets=targets,
-                    venv=resolver_configuration.venv,
+                    venvs=resolver_configuration.venvs,
                     requirement_configuration=requirement_configuration,
                     pip_configuration=resolver_configuration.pip_configuration,
                     compile=compile_pyc,

{pex-2.66.0 → pex-2.67.0}/pex/resolve/locker.py

@@ -30,6 +30,7 @@ from pex.resolve.pep_691.model import Endpoint
 from pex.resolve.resolved_requirement import PartialArtifact, Pin, ResolvedRequirement
 from pex.resolve.resolvers import Resolver
 from pex.resolve.target_system import UniversalTarget
+from pex.result import try_
 from pex.targets import Target
 from pex.typing import TYPE_CHECKING
 
@@ -417,12 +418,14 @@ class Locker(LogAnalyzer):
                             os.path.basename(artifact_url.path)
                         ] = build_result.pin
                     else:
-                        digest_local_project(
-                            directory=artifact_url.path,
-                            digest=digest,
-                            pip_version=self._pip_version,
-                            target=self._target,
-                            resolver=self._resolver,
+                        try_(
+                            digest_local_project(
+                                directory=artifact_url.path,
+                                digest=digest,
+                                pip_version=self._pip_version,
+                                target=self._target,
+                                resolver=self._resolver,
+                            )
                         )
                         self._local_projects.add(artifact_url.path)
                         self._saved.add(build_result.pin)
@@ -490,12 +493,14 @@ class Locker(LogAnalyzer):
                             os.path.basename(artifact_url.path)
                         ] = build_result.pin
                     else:
-                        digest_local_project(
-                            directory=artifact_url.path,
-                            digest=digest,
-                            pip_version=self._pip_version,
-                            target=self._target,
-                            resolver=self._resolver,
+                        try_(
+                            digest_local_project(
+                                directory=artifact_url.path,
+                                digest=digest,
+                                pip_version=self._pip_version,
+                                target=self._target,
+                                resolver=self._resolver,
+                            )
                         )
                         self._local_projects.add(artifact_url.path)
                         self._saved.add(build_result.pin)

{pex-2.66.0 → pex-2.67.0}/pex/resolve/lockfile/create.py

@@ -357,8 +357,6 @@ class LockObserver(ResolveObserver):
                         target=local_distribution.download_target,
                         source_path=local_distribution.path,
                         subdirectory=local_distribution.subdirectory,
-                        resolver=self.resolver,
-                        pip_version=self.package_index_configuration.pip_version,
                     )
                 )
 
@@ -369,10 +367,7 @@ class LockObserver(ResolveObserver):
             lock_result = analysis.analyzer.lock_result
             build_requests.update(
                 BuildRequest.for_directory(
-                    target=analysis.download_target,
-                    source_path=local_project,
-                    resolver=self.resolver,
-                    pip_version=self.package_index_configuration.pip_version,
+                    target=analysis.download_target, source_path=local_project
                 )
                 for local_project in lock_result.local_projects
             )

{pex-2.66.0 → pex-2.67.0}/pex/resolve/pre_resolved_resolver.py

@@ -101,19 +101,13 @@ def resolve_from_dists(
                 extra_pip_requirements=pip_configuration.extra_requirements,
                 keyring_provider=pip_configuration.keyring_provider,
             )
-            resolver = ConfiguredResolver(pip_configuration=pip_configuration)
             build_requests = [
                 BuildRequest.for_file(target=target, source_path=sdist)
                 for sdist in sdists
                 for target in unique_targets
             ]
             build_requests.extend(
-                BuildRequest.for_directory(
-                    target=target,
-                    source_path=local_project.path,
-                    resolver=resolver,
-                    pip_version=pip_configuration.version,
-                )
+                BuildRequest.for_directory(target=target, source_path=local_project.path)
                 for local_project in local_projects
                 for target in unique_targets
             )

{pex-2.66.0 → pex-2.67.0}/pex/resolve/resolver_configuration.py

@@ -251,7 +251,7 @@ class PreResolvedConfiguration(object):
 
 @attr.s(frozen=True)
 class VenvRepositoryConfiguration(object):
-    venv = attr.ib()  # type: Virtualenv
+    venvs = attr.ib()  # type: Tuple[Virtualenv, ...]
     pip_configuration = attr.ib()  # type: PipConfiguration
 
     @property

{pex-2.66.0 → pex-2.67.0}/pex/resolve/resolver_options.py

@@ -56,9 +56,10 @@ class _HandleTransitiveAction(Action):
 class _ResolveVenvAction(Action):
     def __init__(self, *args, **kwargs):
         kwargs["nargs"] = "?"
-        super(_ResolveVenvAction, self).__init__(*args, **kwargs)
+        super(_ResolveVenvAction, self).__init__(*args, default=[], **kwargs)
 
     def __call__(self, parser, namespace, value, option_str=None):
+        venvs = getattr(namespace, self.dest)
         if value:
             if not os.path.exists(value):
                 raise ArgumentError(
@@ -86,7 +87,7 @@ class _ResolveVenvAction(Action):
                         "path.".format(option=option_str, value=value)
                     ),
                 )
-            setattr(namespace, self.dest, venv)
+            venvs.append(venv)
         else:
             current_venv = Virtualenv.enclosing(python=sys.executable)
             if not current_venv:
@@ -105,7 +106,7 @@ class _ResolveVenvAction(Action):
                             )
                         ),
                     )
-            setattr(namespace, self.dest, current_venv)
+            venvs.append(current_venv)
 
 
 def register(
@@ -320,14 +321,18 @@ def register(
     if include_venv_repository:
         repository_choice.add_argument(
             "--venv-repository",
-            dest="venv_repository",
+            dest="venv_repositories",
             action=_ResolveVenvAction,
-            type=str,
             help=(
                 "Resolve requirements from the given virtual environment instead of from "
                 "--index servers, --find-links repos or a --lock file. The virtual environment to "
                 "resolve from can be specified as the path to the venv or the path of its"
-                "interpreter. If no value is specified, the current active venv is used."
+                "interpreter. If no value is specified, the current active venv is used. Multiple "
+                "virtual environments may be specified via multiple --venv-repository options and "
+                "the resolve will be the combined results. Each virtual environment will be "
+                "resolved from individually and must contain the full transitive closure of "
+                "requirements. This allows for creating a multi-platform PEX by specifying "
+                "multiple virtual environments; say one for Python 3.12 and one for Python 3.13."
             ),
         )
 
@@ -803,9 +808,9 @@ def configure(
             sdists=tuple(sdists), wheels=tuple(wheels), pip_configuration=pip_configuration
         )
 
-    venv = getattr(options, "venv_repository", None)
-    if venv:
-        return VenvRepositoryConfiguration(venv=venv, pip_configuration=pip_configuration)
+    venvs = getattr(options, "venv_repositories", None)
+    if venvs:
+        return VenvRepositoryConfiguration(venvs=tuple(venvs), pip_configuration=pip_configuration)
 
     if pylock:
         return PylockRepositoryConfiguration(

{pex-2.66.0 → pex-2.67.0}/pex/resolve/venv_resolver.py

@@ -5,14 +5,13 @@ from __future__ import absolute_import
 
 import functools
 import hashlib
-import itertools
 import os
 from collections import defaultdict, deque
 
 from pex import pex_warnings
 from pex.atomic_directory import atomic_directory
 from pex.cache.dirs import CacheDir, InstalledWheelDir
-from pex.common import safe_relative_symlink
+from pex.common import pluralize, safe_relative_symlink
 from pex.compatibility import commonpath
 from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import (
@@ -26,7 +25,6 @@ from pex.dist_metadata import (
 from pex.exceptions import production_assert, reportable_unexpected_error_msg
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.installed_wheel import InstalledWheel
-from pex.interpreter import PythonInterpreter
 from pex.jobs import DEFAULT_MAX_JOBS, iter_map_parallel
 from pex.orderedset import OrderedSet
 from pex.pep_376 import Record
@@ -47,7 +45,18 @@ from pex.wheel import WHEEL, Wheel
 from pex.whl import repacked_whl
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Deque, FrozenSet, Iterable, Iterator, List, Mapping, Set, Union
+    from typing import (
+        DefaultDict,
+        Deque,
+        FrozenSet,
+        Iterable,
+        Iterator,
+        List,
+        Mapping,
+        Set,
+        Tuple,
+        Union,
+    )
 
     import attr  # vendor:skip
 else:
@@ -96,12 +105,14 @@ def _normalize_record(
 
 
 def _install_distribution(
-    interpreter,  # type: PythonInterpreter
+    venv_distribution,  # type: VenvDistribution
     result_type,  # type: InstallableType.Value
     use_system_time,  # type: bool
-    distribution,  # type: Distribution
 ):
-    # type: (...) -> FingerprintedDistribution
+    # type: (...) -> ResolvedDistribution
+
+    interpreter = venv_distribution.target.interpreter
+    distribution = venv_distribution.distribution
 
     production_assert(distribution.type is DistributionType.INSTALLED)
     production_assert(distribution.metadata.files.metadata.type is MetadataType.DIST_INFO)
@@ -160,31 +171,86 @@ def _install_distribution(
         raise AssertionError(reportable_unexpected_error_msg())
 
     if result_type is InstallableType.INSTALLED_WHEEL_CHROOT:
-        return FingerprintedDistribution(
-            distribution=Distribution.load(installed_wheel.prefix_dir),
-            fingerprint=installed_wheel.fingerprint,
+        return ResolvedDistribution(
+            target=venv_distribution.target,
+            fingerprinted_distribution=FingerprintedDistribution(
+                distribution=Distribution.load(installed_wheel.prefix_dir),
+                fingerprint=installed_wheel.fingerprint,
+            ),
+            direct_requirements=venv_distribution.direct_requirements,
         )
-    return repacked_whl(
-        installed_wheel, fingerprint=installed_wheel.fingerprint, use_system_time=use_system_time
+
+    return ResolvedDistribution(
+        target=venv_distribution.target,
+        fingerprinted_distribution=repacked_whl(
+            installed_wheel,
+            fingerprint=installed_wheel.fingerprint,
+            use_system_time=use_system_time,
+        ),
+        direct_requirements=venv_distribution.direct_requirements,
     )
 
 
+@attr.s(frozen=True)
+class VenvDistribution(object):
+    target = attr.ib()  # type: LocalInterpreter
+    distribution = attr.ib()  # type: Distribution
+    direct_requirements = attr.ib()  # type: Iterable[Requirement]
+
+
 def _install_venv_distributions(
-    venv,  # type: Virtualenv
-    distributions,  # type: Iterable[Distribution]
+    venv_resolve_results,  # type: Iterable[VenvResolveResult]
     max_install_jobs=DEFAULT_MAX_JOBS,  # type: int
     result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
     use_system_time=False,  # type: bool
 ):
-    # type: (...) -> Iterator[FingerprintedDistribution]
+    # type: (...) -> Iterator[ResolvedDistribution]
+
+    seen = set()  # type: Set[str]
+
+    venv_distributions = []  # type: List[VenvDistribution]
+    for venv_resolve_result in venv_resolve_results:
+        target = venv_resolve_result.target
+        direct_requirements = venv_resolve_result.direct_requirements_by_project_name
+        for re_resolved_distribution in venv_resolve_result.re_resolved_distributions:
+            wheel_file_name = Wheel.from_distribution(
+                re_resolved_distribution.distribution
+            ).wheel_file_name
+            if wheel_file_name in seen:
+                continue
+
+            seen.add(wheel_file_name)
+            yield ResolvedDistribution(
+                target=target,
+                fingerprinted_distribution=re_resolved_distribution,
+                direct_requirements=direct_requirements.get(
+                    re_resolved_distribution.project_name, ()
+                ),
+            )
+        for venv_distribution in venv_resolve_result.venv_distributions:
+            wheel_file_name = Wheel.from_distribution(venv_distribution).wheel_file_name
+            if wheel_file_name in seen:
+                continue
+
+            seen.add(wheel_file_name)
+            venv_distributions.append(
+                VenvDistribution(
+                    target=target,
+                    distribution=venv_distribution,
+                    direct_requirements=direct_requirements.get(
+                        venv_distribution.metadata.project_name, ()
+                    ),
+                )
+            )
 
-    return iter_map_parallel(
-        inputs=distributions,
+    for resolved_distribution in iter_map_parallel(
+        inputs=venv_distributions,
         function=functools.partial(
-            _install_distribution, venv.interpreter, result_type, use_system_time
+            _install_distribution, result_type=result_type, use_system_time=use_system_time
         ),
         max_jobs=max_install_jobs,
-    )
+    ):
+        yield resolved_distribution
 
 
 @attr.s(frozen=True)
@@ -359,27 +425,32 @@ def _resolve_distributions(
             )
 
 
-def resolve_from_venv(
-    targets,  # type: Targets
+@attr.s(frozen=True)
+class VenvResolveResult(object):
+    venv = attr.ib()  # type: Virtualenv
+    venv_distributions = attr.ib()  # type: Tuple[Distribution, ...]
+    re_resolved_distributions = attr.ib()  # type: Tuple[FingerprintedDistribution, ...]
+    direct_requirements_by_project_name = attr.ib(
+        eq=False
+    )  # type: Mapping[ProjectName, Iterable[Requirement]]
+
+    @property
+    def target(self):
+        # type: () -> LocalInterpreter
+        return LocalInterpreter.create(self.venv.interpreter)
+
+
+def _resolve_from_venv(
     venv,  # type: Virtualenv
-    requirement_configuration=RequirementConfiguration(),  # type: RequirementConfiguration
-    pip_configuration=PipConfiguration(),  # type: PipConfiguration
-    compile=False,  # type: bool
-    ignore_errors=False,  # type: bool
-    result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
-    dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+    requirement_configuration,  # type: RequirementConfiguration
+    pip_configuration,  # type: PipConfiguration
+    compile,  # type: bool
+    ignore_errors,  # type: bool
+    result_type,  # type: InstallableType.Value
+    dependency_configuration,  # type: DependencyConfiguration
 ):
-    # type: (...) -> Union[ResolveResult, Error]
-
+    # type: (...) -> Union[VenvResolveResult, Error]
     target = LocalInterpreter.create(venv.interpreter)
-    if not targets.is_empty:
-        return Error(
-            "You configured custom targets via --python, --interpreter-constraint, --platform or "
-            "--complete-platform but custom targets are not allowed when resolving from a virtual "
-            "environment.\n"
-            "For such resolves, the supported target is implicitly the one matching the venv "
-            "interpreter; in this case: {target}.".format(target=target.render_description())
-        )
 
     if pip_configuration.version:
         compatible_pip_version = (
@@ -487,27 +558,90 @@ def resolve_from_venv(
             dist.fingerprinted_distribution for dist in result.distributions
         )
 
+    return VenvResolveResult(
+        venv=venv,
+        venv_distributions=tuple(venv_distributions),
+        re_resolved_distributions=tuple(fingerprinted_distributions),
+        direct_requirements_by_project_name=direct_requirements_by_project_name,
+    )
+
+
+def resolve_from_venvs(
+    targets,  # type: Targets
+    venvs,  # type: Tuple[Virtualenv, ...]
+    requirement_configuration=RequirementConfiguration(),  # type: RequirementConfiguration
+    pip_configuration=PipConfiguration(),  # type: PipConfiguration
+    compile=False,  # type: bool
+    ignore_errors=False,  # type: bool
+    result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
+    dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
+):
+    # type: (...) -> Union[ResolveResult, Error]
+
+    if not targets.is_empty:
+        return Error(
+            "You configured custom targets via --python, --interpreter-constraint, --platform or "
+            "--complete-platform but custom targets are not allowed when resolving from {venvs}.\n"
+            "For such resolves, the supported target is implicitly the one matching the venv "
+            "{interpreters}; in this case:{targets}.".format(
+                venvs="a virtual environment" if len(venvs) == 1 else "virtual environments",
+                interpreters=pluralize(venvs, "interpreter"),
+                targets=(
+                    " {target}".format(
+                        target=LocalInterpreter.create(venvs[0].interpreter).render_description()
+                    )
+                    if len(venvs) == 1
+                    else "\n  {targets}".format(
+                        targets="\n  ".join(
+                            LocalInterpreter.create(venv.interpreter).render_description()
+                            for venv in venvs
+                        )
+                    )
+                ),
+            )
+        )
+
+    errors = []  # type: List[Error]
+    venv_resolve_results = []  # type: List[VenvResolveResult]
+    for result in iter_map_parallel(
+        venvs,
+        functools.partial(
+            _resolve_from_venv,
+            requirement_configuration=requirement_configuration,
+            pip_configuration=pip_configuration,
+            compile=compile,
+            ignore_errors=ignore_errors,
+            result_type=result_type,
+            dependency_configuration=dependency_configuration,
+        ),
+    ):
+        if isinstance(result, Error):
+            errors.append(result)
+        else:
+            venv_resolve_results.append(result)
+
+    if len(errors) == 1:
+        return errors[0]
+    elif errors:
+        return Error(
+            "Failed to resolve from {count} of {total} virtual environments:\n{failures}".format(
+                count=len(errors),
+                total=len(venvs),
+                failures="\n".join(
+                    "{index}. {error}".format(index=index, error=error)
+                    for index, error in enumerate(errors, start=1)
+                ),
+            )
+        )
+
     return ResolveResult(
         dependency_configuration=dependency_configuration,
         distributions=tuple(
-            ResolvedDistribution(
-                target=target,
-                fingerprinted_distribution=fingerprinted_distribution,
-                direct_requirements=direct_requirements_by_project_name[
-                    fingerprinted_distribution.project_name
-                ],
-            )
-            for fingerprinted_distribution in itertools.chain(
-                list(
-                    _install_venv_distributions(
-                        venv=venv,
-                        distributions=venv_distributions,
-                        max_install_jobs=pip_configuration.max_jobs,
-                        result_type=result_type,
-                        use_system_time=True,
-                    )
-                ),
-                fingerprinted_distributions,
+            _install_venv_distributions(
+                venv_resolve_results,
+                max_install_jobs=pip_configuration.max_jobs,
+                result_type=result_type,
+                use_system_time=True,
             )
         ),
         type=result_type,