pex 2.54.2__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

pex/resolve/resolver_options.py

@@ -5,22 +5,26 @@ from __future__ import absolute_import
 
 import glob
 import os
+import sys
 import tempfile
 from argparse import Action, ArgumentError, ArgumentTypeError, Namespace, _ActionsContainer
+from collections import OrderedDict, defaultdict
 
 from pex import pex_warnings
 from pex.argparse import HandleBoolAction
+from pex.common import pluralize
 from pex.dist_metadata import Requirement, is_sdist, is_wheel
 from pex.fetcher import initialize_ssl_context
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
+from pex.os import is_exe
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.resolve.lockfile import json_codec
 from pex.resolve.lockfile.model import Lockfile
+from pex.resolve.package_repository import PYPI, Repo, ReposConfiguration, Scope
 from pex.resolve.path_mappings import PathMapping, PathMappings
 from pex.resolve.resolver_configuration import (
-    PYPI,
     BuildConfiguration,
     LockRepositoryConfiguration,
     PexRepositoryConfiguration,
@@ -28,41 +32,81 @@ from pex.resolve.resolver_configuration import (
     PipLog,
     PreResolvedConfiguration,
     PylockRepositoryConfiguration,
-    ReposConfiguration,
     ResolverVersion,
+    VenvRepositoryConfiguration,
 )
 from pex.result import Error
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
+from pex.venv.virtualenv import InvalidVirtualenvError, Virtualenv
 
 if TYPE_CHECKING:
-    from typing import List, Optional, Union
+    from typing import DefaultDict, Iterable, List, Mapping, Optional, Tuple, Union
 
 
-class _ManylinuxAction(Action):
+class _HandleTransitiveAction(Action):
     def __init__(self, *args, **kwargs):
-        kwargs["nargs"] = "?"
-        super(_ManylinuxAction, self).__init__(*args, **kwargs)
+        kwargs["nargs"] = 0
+        super(_HandleTransitiveAction, self).__init__(*args, **kwargs)
 
     def __call__(self, parser, namespace, value, option_str=None):
-        if option_str.startswith("--no"):
-            setattr(namespace, self.dest, None)
-        elif value.startswith("manylinux"):
-            setattr(namespace, self.dest, value)
-        else:
-            raise ArgumentTypeError(
-                "Please specify a manylinux standard; ie: --manylinux=manylinux1. "
-                "Given {}".format(value)
-            )
+        setattr(namespace, self.dest, option_str == "--transitive")
 
 
-class _HandleTransitiveAction(Action):
+class _ResolveVenvAction(Action):
     def __init__(self, *args, **kwargs):
-        kwargs["nargs"] = 0
-        super(_HandleTransitiveAction, self).__init__(*args, **kwargs)
+        kwargs["nargs"] = "?"
+        super(_ResolveVenvAction, self).__init__(*args, default=[], **kwargs)
 
     def __call__(self, parser, namespace, value, option_str=None):
-        setattr(namespace, self.dest, option_str == "--transitive")
+        venvs = getattr(namespace, self.dest)
+        if value:
+            if not os.path.exists(value):
+                raise ArgumentError(
+                    argument=self,
+                    message=(
+                        "The {option} {value} does not point to an existing path.".format(
+                            option=option_str, value=value
+                        )
+                    ),
+                )
+
+            venv = None  # type: Optional[Virtualenv]
+            if is_exe(value):
+                venv = Virtualenv.enclosing(value)
+            else:
+                try:
+                    venv = Virtualenv(value)
+                except InvalidVirtualenvError:
+                    pass
+            if not venv:
+                raise ArgumentError(
+                    argument=self,
+                    message=(
+                        "The {option} {value} is not a valid virtual environment interpreter "
+                        "path.".format(option=option_str, value=value)
+                    ),
+                )
+            venvs.append(venv)
+        else:
+            current_venv = Virtualenv.enclosing(python=sys.executable)
+            if not current_venv:
+                try:
+                    current_venv = Virtualenv(venv_dir=".")
+                except InvalidVirtualenvError:
+                    raise ArgumentError(
+                        argument=self,
+                        message=(
+                            "The {option} option was requested but Pex is not currently running "
+                            "from a venv to use as the implicit {option} nor is the current "
+                            "directory a venv.\n"
+                            "You'll need to specify the path to a virtual environment directory or "
+                            "virtual environment interpreter as an argument.".format(
+                                option=option_str
+                            )
+                        ),
+                    )
+            venvs.append(current_venv)
 
 
 def register(
@@ -71,6 +115,7 @@ def register(
     include_pex_lock=False,  # type: bool
     include_pylock=False,  # type: bool
     include_pre_resolved=False,  # type: bool
+    include_venv_repository=False,  # type: bool
 ):
     # type: (...) -> None
     """Register resolver configuration options with the given parser.
@@ -81,6 +126,7 @@ def register(
     :param include_pylock: Whether to include the `--pylock` / `--pep-751-lock` option.
     :param include_pre_resolved: Whether to include the `--pre-resolved-dist` and
                                  `--pre-resolved-dists` options.
+    :param include_venv_repository: Whether to include the `--venv-repository` option.
     """
 
     default_resolver_configuration = PipConfiguration()
@@ -191,6 +237,8 @@ def register(
         repository_types += 1
     if include_pre_resolved:
         repository_types += 1
+    if include_venv_repository:
+        repository_types += 1
 
     repository_choice = parser.add_mutually_exclusive_group() if repository_types > 1 else parser
     if include_pex_repository:
@@ -270,6 +318,23 @@ def register(
                 "default, Pex will ensure the dependencies added form a closure."
             ),
         )
+    if include_venv_repository:
+        repository_choice.add_argument(
+            "--venv-repository",
+            dest="venv_repositories",
+            action=_ResolveVenvAction,
+            help=(
+                "Resolve requirements from the given virtual environment instead of from "
+                "--index servers, --find-links repos or a --lock file. The virtual environment to "
+                "resolve from can be specified as the path to the venv or the path of its"
+                "interpreter. If no value is specified, the current active venv is used. Multiple "
+                "virtual environments may be specified via multiple --venv-repository options and "
+                "the resolve will be the combined results. Each virtual environment will be "
+                "resolved from individually and must contain the full transitive closure of "
+                "requirements. This allows for creating a multi-platform PEX by specifying "
+                "multiple virtual environments; say one for Python 3.12 and one for Python 3.13."
+            ),
+        )
 
     parser.add_argument(
         "--pre",
@@ -525,21 +590,64 @@ def register_repos_options(parser):
         "-f",
         "--find-links",
         "--repo",
-        metavar="PATH/URL",
+        metavar="(NAME=)PATH/URL",
         action="append",
         dest="find_links",
         type=str,
-        help="Additional repository path (directory or URL) to look for requirements.",
+        default=[],
+        help=(
+            "Additional repository path (directory or URL) to look for requirements. If the "
+            "repository should only be used as a source for a subset of distributions, you can "
+            "associate a name to refer to it by in `--source` mappings by using "
+            "`<name>=<path or url>` syntax, where the name cannot itself contain `=`. For example; "
+            "`local=/mnt/share/py/find-links` or `internal=https://dev.corp.internal/py-repo`."
+        ),
     )
     parser.add_argument(
         "-i",
         "--index",
         "--index-url",
-        metavar="URL",
+        metavar="(NAME=)URL",
         action="append",
         dest="indexes",
         type=str,
-        help="Additional cheeseshop indices to use to satisfy requirements.",
+        default=[],
+        help=(
+            "Additional cheeseshop indices to use to satisfy requirements. If the index should "
+            "only be used as a source for a subset of distributions, you can associate a name to "
+            "refer to it by in `--source` mappings by using `<name>=<url>` syntax, where the name "
+            "itself cannot contain `=`. For example; internal=https://dev.corp.internal/py-simple`."
+        ),
+    )
+    parser.add_argument(
+        "--source",
+        metavar="NAME=SCOPE",
+        action="append",
+        dest="repo_scopes",
+        type=str,
+        default=[],
+        help=(
+            "Defines a limited scope to use a named find links repo or index in. A source takes "
+            "the form `<name>=<scope>` where name is as defined for `--find-links` and `--index`; "
+            "namely a string not containing `=`. The scope can be a project name; e.g.: "
+            "`internal=torch` to resolve the `torch` project from the `internal` repo, a project "
+            "name and marker; e.g.: `internal=torch; sys_platform != 'darwin'` to resolve the "
+            "`torch` project from the `internal` repo unless targeting macOS, or just a marker; "
+            "e.g.: `piwheels=platform_machine == 'armv7l' to resolve from the `piwheels` repo "
+            "whenever 32bit arm machines are being targeted. N.B. wherever you use a project name "
+            "in a scope, you can use a regex instead."
+        ),
+    )
+    parser.add_argument(
+        "--derive-sources-from-requirements-files",
+        dest="derive_scopes_from_requirements_files",
+        action=HandleBoolAction,
+        default=False,
+        help=(
+            "If any requirements files are specified that contain `-f` / `--find-links`, `-i` / "
+            "`--index-url`, or `--extra-index-url` options, automatically map these repos as the "
+            "`--source` for the requirements (if any) declared in that same requirements file."
+        ),
     )
 
 
@@ -632,6 +740,7 @@ if TYPE_CHECKING:
         PipConfiguration,
         PreResolvedConfiguration,
         PylockRepositoryConfiguration,
+        VenvRepositoryConfiguration,
     ]
 
 
@@ -699,6 +808,18 @@ def configure(
             sdists=tuple(sdists), wheels=tuple(wheels), pip_configuration=pip_configuration
         )
 
+    venvs = getattr(options, "venv_repositories", None)
+    if venvs:
+        return VenvRepositoryConfiguration(venvs=tuple(venvs), pip_configuration=pip_configuration)
+
+    if pylock:
+        return PylockRepositoryConfiguration(
+            lock_file_path=pylock,
+            extras=frozenset(options.pylock_extras),
+            dependency_groups=frozenset(options.pylock_dependency_groups),
+            pip_configuration=pip_configuration,
+        )
+
     return pip_configuration
 
 
@@ -766,17 +887,102 @@ def create_pip_configuration(
     )
 
 
+def _parse_repo_scope(repo_scope):
+    # type: (str) -> Tuple[str, Scope]
+
+    def create_invalid_error(footer=None):
+        # type: (Optional[str]) -> Exception
+        error_msg_lines = ["Invalid --source value: {source}".format(source=repo_scope)]
+        if footer:
+            error_msg_lines.append(footer)
+        return InvalidConfigurationError(os.linesep.join(error_msg_lines))
+
+    name, _, scope = repo_scope.partition("=")
+    if not scope:
+        raise create_invalid_error("A source must be of the form `<name>=<scope>`.")
+    try:
+        return name, Scope.parse(scope)
+    except ValueError as e:
+        raise create_invalid_error(str(e))
+
+
+def _parse_package_repositories(
+    package_repository_type,  # type: str
+    scopes_by_name,  # type: Mapping[str, Iterable[Scope]]
+    repositories,  # type: Iterable[str]
+):
+    # type: (...) -> OrderedDict[str, Repo]
+
+    parsed = OrderedDict()  # type: OrderedDict[str, List[str]]
+    for repository in repositories:
+        name, _, location = repository.partition("=")
+        if name in scopes_by_name:
+            parsed.setdefault(name, []).append(location)
+        else:
+            parsed.setdefault("", []).append(repository)
+
+    duplicate_names = [name for name, repos in parsed.items() if len(repos) > 1]
+    if duplicate_names:
+        raise InvalidConfigurationError(
+            "The following names are being re-used across multiple {package_repositories}: "
+            "{duplicate_names}\n"
+            "{package_repository} names must be unique.".format(
+                package_repositories=pluralize(2, package_repository_type),
+                duplicate_names=" ".join(sorted(duplicate_names)),
+                package_repository=package_repository_type.capitalize(),
+            )
+        )
+
+    package_repositories = OrderedDict()  # type: OrderedDict[str, Repo]
+    for name, locations in parsed.items():
+        if name:
+            package_repositories[name] = Repo(locations[0], scopes=tuple(scopes_by_name[name]))
+        else:
+            package_repositories[""] = Repo(locations[0])
+    return package_repositories
+
+
 def create_repos_configuration(options):
     # type: (Namespace) -> ReposConfiguration
     """Creates a repos configuration from options registered by `register_repos_options`.
 
     :param options: The Pip resolver configuration options.
     """
-    indexes = OrderedSet(
-        ([PYPI] if options.pypi else []) + (options.indexes or [])
-    )  # type: OrderedSet[str]
-    find_links = OrderedSet(options.find_links or ())  # type: OrderedSet[str]
-    return ReposConfiguration.create(indexes=tuple(indexes), find_links=tuple(find_links))
+    indexes = OrderedSet(([Repo(PYPI)] if options.pypi else []))  # type: OrderedSet[Repo]
+    scopes_by_name = defaultdict(OrderedSet)  # type: DefaultDict[str, OrderedSet[Scope]]
+    for repo_scope in options.repo_scopes:
+        name, scope = _parse_repo_scope(repo_scope)
+        scopes_by_name[name].add(scope)
+
+    if not scopes_by_name:
+        indexes.update(Repo(index) for index in options.indexes)
+        return ReposConfiguration.create(
+            indexes=tuple(indexes),
+            find_links=tuple(Repo(find_links_repo) for find_links_repo in options.find_links),
+            derive_scopes_from_requirements_files=options.derive_scopes_from_requirements_files,
+        )
+
+    parsed_indexes = _parse_package_repositories("index", scopes_by_name, options.indexes)
+    parsed_find_links = _parse_package_repositories(
+        "find links repository", scopes_by_name, options.find_links
+    )
+
+    duplicate_names = set(parsed_indexes).intersection(parsed_find_links)
+    if duplicate_names:
+        raise InvalidConfigurationError(
+            "The following names are being re-used across indexes and find links repos: "
+            "{duplicate_names}\n"
+            "Package repository names must be unique.".format(
+                duplicate_names=" ".join(sorted(duplicate_names))
+            )
+        )
+
+    indexes.update(parsed_indexes.values())
+    return ReposConfiguration.create(
+        indexes=tuple(indexes),
+        find_links=tuple(parsed_find_links.values()),
+        derive_scopes_from_requirements_files=options.derive_scopes_from_requirements_files,
+    )
 
 
 def create_network_configuration(options):

pex/resolve/resolvers.py

@@ -16,7 +16,6 @@ from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.pep_427 import InstallableType
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersionValue
-from pex.resolve.lockfile.model import Lockfile
 from pex.sorted_tuple import SortedTuple
 from pex.targets import AbbreviatedPlatform, Target, Targets
 from pex.typing import TYPE_CHECKING
@@ -247,17 +246,6 @@ class Resolver(object):
         # type: () -> bool
         raise NotImplementedError()
 
-    @abstractmethod
-    def resolve_lock(
-        self,
-        lock,  # type: Lockfile
-        targets=Targets(),  # type: Targets
-        pip_version=None,  # type: Optional[PipVersionValue]
-        result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
-    ):
-        # type: (...) -> ResolveResult
-        raise NotImplementedError()
-
     @abstractmethod
     def resolve_requirements(
         self,
@@ -267,6 +255,9 @@ class Resolver(object):
         transitive=None,  # type: Optional[bool]
         extra_resolver_requirements=None,  # type: Optional[Tuple[Requirement, ...]]
         result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
+        constraint_files=None,  # type: Optional[Iterable[str]]
+        compile=False,  # type: bool
+        ignore_errors=False,  # type: bool
     ):
         # type: (...) -> ResolveResult
         raise NotImplementedError()

pex/resolve/target_options.py

@@ -6,7 +6,7 @@ from __future__ import absolute_import
 import json
 import os.path
 import sys
-from argparse import ArgumentTypeError, Namespace, _ActionsContainer
+from argparse import Action, ArgumentTypeError, Namespace, _ActionsContainer
 
 from pex.argparse import HandleBoolAction
 from pex.interpreter_constraints import InterpreterConstraints
@@ -16,7 +16,6 @@ from pex.pep_508 import MarkerEnvironment
 from pex.platforms import Platform, PlatformSpec
 from pex.resolve import abbreviated_platforms
 from pex.resolve.resolver_configuration import PipConfiguration
-from pex.resolve.resolver_options import _ManylinuxAction
 from pex.resolve.target_configuration import InterpreterConfiguration, TargetConfiguration
 from pex.targets import CompletePlatform
 from pex.typing import TYPE_CHECKING
@@ -25,6 +24,23 @@ if TYPE_CHECKING:
     from typing import Optional
 
 
+class _ManylinuxAction(Action):
+    def __init__(self, *args, **kwargs):
+        kwargs["nargs"] = "?"
+        super(_ManylinuxAction, self).__init__(*args, **kwargs)
+
+    def __call__(self, parser, namespace, value, option_str=None):
+        if option_str.startswith("--no"):
+            setattr(namespace, self.dest, None)
+        elif value.startswith("manylinux"):
+            setattr(namespace, self.dest, value)
+        else:
+            raise ArgumentTypeError(
+                "Please specify a manylinux standard; ie: --manylinux=manylinux1. "
+                "Given {}".format(value)
+            )
+
+
 def register(
     parser,  # type: _ActionsContainer
     include_platforms=True,  # type: bool