pex 2.54.2__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

pex/requirements.py

@@ -5,6 +5,9 @@ from __future__ import absolute_import
 
 import os
 import re
+import shlex
+import sys
+from argparse import ArgumentParser
 from contextlib import contextmanager
 
 from pex import attrs, dist_metadata, pex_warnings
@@ -17,6 +20,7 @@ from pex.dist_metadata import (
     RequirementParseError,
 )
 from pex.fetcher import URLFetcher
+from pex.orderedset import OrderedSet
 from pex.pep_503 import ProjectName
 from pex.third_party.packaging.markers import Marker
 from pex.third_party.packaging.specifiers import SpecifierSet
@@ -139,7 +143,7 @@ class Source(object):
 
 
 @attr.s(frozen=True)
-class _ParsedRequirement(object):
+class _ParsedItem(object):
     line = attr.ib()  # type: LogicalLine
 
     def __str__(self):
@@ -148,38 +152,67 @@ class _ParsedRequirement(object):
 
 
 @attr.s(frozen=True)
-class PyPIRequirement(_ParsedRequirement):
+class _Repo(_ParsedItem):
+    location = attr.ib()  # Text
+
+
+@attr.s(frozen=True)
+class FindLinks(_Repo):
+    pass
+
+
+@attr.s(frozen=True)
+class Index(_Repo):
+    pass
+
+
+@attr.s(frozen=True)
+class PyPIRequirement(_ParsedItem):
     """A requirement realized through a package index or find links repository."""
 
     requirement = attr.ib()  # type: Requirement
 
+    @property
+    def project_name(self):
+        return self.requirement.project_name
+
     @property
     def extras(self):
         # type: () -> FrozenSet[str]
         return self.requirement.extras
 
+    @property
+    def marker(self):
+        # type: () -> Optional[Marker]
+        return self.requirement.marker
+
 
 @attr.s(frozen=True)
-class URLRequirement(_ParsedRequirement):
+class URLRequirement(_ParsedItem):
     """A requirement realized through a distribution archive at a fixed URL."""
 
     url = attr.ib()  # type: ArtifactURL
     requirement = attr.ib()  # type: Requirement
 
+    @property
+    def project_name(self):
+        return self.requirement.project_name
+
     @property
     def extras(self):
         # type: () -> FrozenSet[str]
         return self.requirement.extras
 
+    @property
+    def marker(self):
+        # type: () -> Optional[Marker]
+        return self.requirement.marker
+
     @property
     def filename(self):
         # type: () -> str
         return os.path.basename(self.url.path)
 
-    @property
-    def project_name(self):
-        return self.requirement.project_name
-
     @property
     def subdirectory(self):
         # type: () -> Optional[str]
@@ -188,18 +221,27 @@ class URLRequirement(_ParsedRequirement):
 
 
 @attr.s(frozen=True)
-class VCSRequirement(_ParsedRequirement):
+class VCSRequirement(_ParsedItem):
     """A requirement realized by building a distribution from sources retrieved from a VCS."""
 
     vcs = attr.ib()  # type: VCS.Value
     url = attr.ib()  # type: Text
     requirement = attr.ib()  # type: Requirement
 
+    @property
+    def project_name(self):
+        return self.requirement.project_name
+
     @property
     def extras(self):
         # type: () -> FrozenSet[str]
         return self.requirement.extras
 
+    @property
+    def marker(self):
+        # type: () -> Optional[Marker]
+        return self.requirement.marker
+
 
 def parse_requirement_from_project_name_and_specifier(
     project_name,  # type: Text
@@ -246,7 +288,7 @@ def parse_requirement_from_dist(
 
 
 @attr.s(frozen=True)
-class LocalProjectRequirement(_ParsedRequirement):
+class LocalProjectRequirement(_ParsedItem):
     """A requirement realized by building a distribution from local sources."""
 
     path = attr.ib()  # type: str
@@ -283,9 +325,18 @@ if TYPE_CHECKING:
 
 
 @attr.s(frozen=True)
-class Constraint(_ParsedRequirement):
+class Constraint(_ParsedItem):
     requirement = attr.ib()  # type: Requirement
 
+    @property
+    def project_name(self):
+        return self.requirement.project_name
+
+    @property
+    def marker(self):
+        # type: () -> Optional[Marker]
+        return self.requirement.marker
+
 
 class ParseError(Exception):
     def __init__(
@@ -598,11 +649,59 @@ def _get_parameter(line):
     return split_line[1]
 
 
+_REPOS_PARSER = None  # type: Optional[ArgumentParser]
+
+
+def _parse_repos(line):
+    # type: (LogicalLine) -> Iterator[Union[FindLinks, Index]]
+
+    try:
+        # The arg-type type ignore is due to Python 2.7 shlex.split not being able to parse unicode
+        # strings; but we handle that with a useful error message.
+        args = shlex.split(line.processed_text)  # type: ignore[arg-type]
+    except UnicodeEncodeError as e:
+        raise ParseError(
+            line,
+            "Options line has unicode characters which are not supported under Python {version}: "
+            "{err}".format(version=".".join(map(str, sys.version[:2])), err=e),
+        )
+
+    global _REPOS_PARSER
+    if _REPOS_PARSER is not None:
+        parser = _REPOS_PARSER
+    else:
+        parser = ArgumentParser()
+        # See:
+        # + https://pip.pypa.io/en/stable/reference/requirements-file-format/#global-options
+        # + https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-f
+        # + https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-i
+        # + https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-extra-index-url
+        parser.add_argument("-f", "--find-links", dest="find_links", action="append", default=[])
+        parser.add_argument("-i", "--index-url", dest="index_url")
+        parser.add_argument(
+            "--extra-index-url", dest="extra_index_urls", action="append", default=[]
+        )
+        _REPOS_PARSER = parser
+
+    options, _ = parser.parse_known_args(args)
+
+    for find_links in OrderedSet(options.find_links):  # type: OrderedSet[str]
+        yield FindLinks(line, location=find_links)
+
+    index_locations = OrderedSet()  # type: OrderedSet[str]
+    if options.index_url:
+        index_locations.add(options.index_url)
+    index_locations.update(options.extra_index_urls)
+
+    for index in index_locations:
+        yield Index(line, location=index)
+
+
 def parse_requirements(
     source,  # type: Source
     fetcher=None,  # type: Optional[URLFetcher]
 ):
-    # type: (...) -> Iterator[Union[ParsedRequirement, Constraint]]
+    # type: (...) -> Iterator[Union[ParsedRequirement, Constraint, FindLinks, Index]]
 
     # For the format specification, see:
     #   https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
@@ -654,11 +753,17 @@ def parse_requirements(
                         yield requirement
                 continue
 
-            # Skip empty lines, comment lines and all Pip global options.
-            if not processed_text or (
-                processed_text.startswith("-")
-                and not re.match(r"^(?:-e|--editable)\s.*", processed_text)
+            # Skip empty lines and comment lines.
+            if not processed_text:
+                continue
+
+            if processed_text.startswith("-") and not re.match(
+                # N.B.: We deal with editable when parsing requirement lines below.
+                r"^(?:-e|--editable)\s.*",
+                processed_text,
             ):
+                for repo in _parse_repos(logical_line):
+                    yield repo
                 continue
 
             # Only requirement lines remain.
@@ -689,7 +794,7 @@ def parse_requirement_file(
     is_constraints=False,  # type: bool
     fetcher=None,  # type: Optional[URLFetcher]
 ):
-    # type: (...) -> Iterator[Union[ParsedRequirement, Constraint]]
+    # type: (...) -> Iterator[Union[ParsedRequirement, Constraint, FindLinks, Index]]
     def open_source():
         url = urlparse.urlparse(location)
         if url.scheme and url.netloc:

pex/resolve/config.py

@@ -87,7 +87,11 @@ def finalize(
                 resolver_configuration.pip_configuration,
                 targets,
                 context,
-                pip_version=pip_version or lock_file.pip_version,
+                pip_version=(
+                    pip_version
+                    or resolver_configuration.pip_configuration.version
+                    or lock_file.pip_version
+                ),
             )
         )
         return cast(

pex/resolve/configured_resolve.py

@@ -17,8 +17,10 @@ from pex.resolve.resolver_configuration import (
     PexRepositoryConfiguration,
     PreResolvedConfiguration,
     PylockRepositoryConfiguration,
+    VenvRepositoryConfiguration,
 )
 from pex.resolve.resolvers import ResolveResult
+from pex.resolve.venv_resolver import resolve_from_venvs
 from pex.resolver import resolve as resolve_via_pip
 from pex.result import try_
 from pex.targets import Targets
@@ -55,15 +57,13 @@ def resolve(
                     requirement_files=requirement_configuration.requirement_files,
                     constraint_files=requirement_configuration.constraint_files,
                     transitive=pip_configuration.transitive,
-                    indexes=pip_configuration.repos_configuration.indexes,
-                    find_links=pip_configuration.repos_configuration.find_links,
+                    repos_configuration=pip_configuration.repos_configuration,
                     resolver_version=pip_configuration.resolver_version,
                     network_configuration=pip_configuration.network_configuration,
-                    password_entries=pip_configuration.repos_configuration.password_entries,
                     build_configuration=pip_configuration.build_configuration,
                     compile=compile_pyc,
                     max_parallel_jobs=pip_configuration.max_jobs,
-                    pip_version=lock.pip_version,
+                    pip_version=pip_configuration.version or lock.pip_version,
                     use_pip_config=pip_configuration.use_pip_config,
                     extra_pip_requirements=pip_configuration.extra_requirements,
                     keyring_provider=pip_configuration.keyring_provider,
@@ -88,11 +88,9 @@ def resolve(
                     dependency_groups=resolver_configuration.dependency_groups,
                     constraint_files=requirement_configuration.constraint_files,
                     transitive=pip_configuration.transitive,
-                    indexes=pip_configuration.repos_configuration.indexes,
-                    find_links=pip_configuration.repos_configuration.find_links,
+                    repos_configuration=pip_configuration.repos_configuration,
                     resolver_version=pip_configuration.resolver_version,
                     network_configuration=pip_configuration.network_configuration,
-                    password_entries=pip_configuration.repos_configuration.password_entries,
                     build_configuration=pip_configuration.build_configuration,
                     compile=compile_pyc,
                     max_parallel_jobs=pip_configuration.max_jobs,
@@ -143,6 +141,32 @@ def resolve(
                 result_type=result_type,
                 dependency_configuration=dependency_configuration,
             )
+    elif isinstance(resolver_configuration, VenvRepositoryConfiguration):
+        with TRACER.timed(
+            "Resolving requirements from {count} {venvs} at:{paths}.".format(
+                count=len(resolver_configuration.venvs),
+                venvs=pluralize(resolver_configuration.venvs, "venv"),
+                paths=(
+                    " {venv}".format(venv=resolver_configuration.venvs[0].venv_dir)
+                    if len(resolver_configuration.venvs) == 1
+                    else "\n  {venvs}".format(
+                        venvs="\n  ".join(venv.venv_dir for venv in resolver_configuration.venvs)
+                    )
+                ),
+            )
+        ):
+            return try_(
+                resolve_from_venvs(
+                    targets=targets,
+                    venvs=resolver_configuration.venvs,
+                    requirement_configuration=requirement_configuration,
+                    pip_configuration=resolver_configuration.pip_configuration,
+                    compile=compile_pyc,
+                    ignore_errors=ignore_errors,
+                    result_type=result_type,
+                    dependency_configuration=dependency_configuration,
+                )
+            )
     else:
         with TRACER.timed("Resolving requirements."):
             return resolve_via_pip(
@@ -152,11 +176,9 @@ def resolve(
                 constraint_files=requirement_configuration.constraint_files,
                 allow_prereleases=resolver_configuration.allow_prereleases,
                 transitive=resolver_configuration.transitive,
-                indexes=resolver_configuration.repos_configuration.indexes,
-                find_links=resolver_configuration.repos_configuration.find_links,
+                repos_configuration=resolver_configuration.repos_configuration,
                 resolver_version=resolver_configuration.resolver_version,
                 network_configuration=resolver_configuration.network_configuration,
-                password_entries=resolver_configuration.repos_configuration.password_entries,
                 build_configuration=resolver_configuration.build_configuration,
                 compile=compile_pyc,
                 max_parallel_jobs=resolver_configuration.max_jobs,

pex/resolve/configured_resolver.py

@@ -7,11 +7,9 @@ from pex import resolver
 from pex.dist_metadata import Requirement
 from pex.pep_427 import InstallableType
 from pex.pip.version import PipVersion, PipVersionValue
-from pex.resolve import lock_resolver
-from pex.resolve.lockfile.model import Lockfile
-from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration, ResolverVersion
+from pex.resolve.package_repository import ReposConfiguration
+from pex.resolve.resolver_configuration import PipConfiguration, ResolverVersion
 from pex.resolve.resolvers import Resolver, ResolveResult
-from pex.result import try_
 from pex.targets import Targets
 from pex.typing import TYPE_CHECKING
 
@@ -52,36 +50,6 @@ class ConfiguredResolver(Resolver):
         # type: () -> bool
         return self.pip_configuration.build_configuration.use_system_time
 
-    def resolve_lock(
-        self,
-        lock,  # type: Lockfile
-        targets=Targets(),  # type: Targets
-        pip_version=None,  # type: Optional[PipVersionValue]
-        result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
-    ):
-        # type: (...) -> ResolveResult
-        return try_(
-            lock_resolver.resolve_from_pex_lock(
-                targets=targets,
-                lock=lock,
-                resolver=self,
-                indexes=self.pip_configuration.repos_configuration.indexes,
-                find_links=self.pip_configuration.repos_configuration.find_links,
-                resolver_version=self.pip_configuration.resolver_version,
-                network_configuration=self.pip_configuration.network_configuration,
-                build_configuration=self.pip_configuration.build_configuration,
-                compile=False,
-                transitive=self.pip_configuration.transitive,
-                verify_wheels=True,
-                max_parallel_jobs=self.pip_configuration.max_jobs,
-                pip_version=pip_version or self.pip_configuration.version,
-                use_pip_config=self.pip_configuration.use_pip_config,
-                extra_pip_requirements=self.pip_configuration.extra_requirements,
-                keyring_provider=self.pip_configuration.keyring_provider,
-                result_type=result_type,
-            )
-        )
-
     def resolve_requirements(
         self,
         requirements,  # type: Iterable[str]
@@ -90,21 +58,24 @@ class ConfiguredResolver(Resolver):
         transitive=None,  # type: Optional[bool]
         extra_resolver_requirements=None,  # type: Optional[Tuple[Requirement, ...]]
         result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
+        constraint_files=None,  # type: Optional[Iterable[str]]
+        compile=False,  # type: bool
+        ignore_errors=False,  # type: bool
     ):
         # type: (...) -> ResolveResult
         return resolver.resolve(
             targets=targets,
             requirements=requirements,
-            allow_prereleases=False,
+            constraint_files=constraint_files,
+            allow_prereleases=self.pip_configuration.allow_prereleases,
             transitive=transitive if transitive is not None else self.pip_configuration.transitive,
-            indexes=self.pip_configuration.repos_configuration.indexes,
-            find_links=self.pip_configuration.repos_configuration.find_links,
+            repos_configuration=self.pip_configuration.repos_configuration,
             resolver_version=self.pip_configuration.resolver_version,
             network_configuration=self.pip_configuration.network_configuration,
             build_configuration=self.pip_configuration.build_configuration,
-            compile=False,
+            compile=compile,
             max_parallel_jobs=self.pip_configuration.max_jobs,
-            ignore_errors=False,
+            ignore_errors=ignore_errors,
             verify_wheels=True,
             pip_version=pip_version or self.pip_configuration.version,
             resolver=self,

pex/resolve/downloads.py

@@ -18,9 +18,10 @@ from pex.pip.download_observer import DownloadObserver
 from pex.pip.installation import get_pip
 from pex.pip.tool import PackageIndexConfiguration, Pip
 from pex.resolve import locker
-from pex.resolve.locked_resolve import Artifact, FileArtifact, LockConfiguration
+from pex.resolve.locked_resolve import Artifact, FileArtifact
 from pex.resolve.resolved_requirement import PartialArtifact
 from pex.resolve.resolvers import Resolver
+from pex.resolve.target_system import UniversalTarget
 from pex.result import Error
 from pex.targets import LocalInterpreter, Target
 from pex.typing import TYPE_CHECKING
@@ -38,7 +39,7 @@ else:
 @attr.s(frozen=True)
 class ArtifactDownloader(object):
     resolver = attr.ib()  # type: Resolver
-    lock_configuration = attr.ib()  # type: LockConfiguration
+    universal_target = attr.ib(default=None)  # type: Optional[UniversalTarget]
     target = attr.ib(factory=LocalInterpreter.create)  # type: Target
     package_index_configuration = attr.ib(
         factory=PackageIndexConfiguration.create
@@ -103,7 +104,7 @@ class ArtifactDownloader(object):
         # generates no patches if the lock is not universal.
         download_observer = foreign_platform.patch(self.target) or DownloadObserver(
             analyzer=None,
-            patch_set=locker.patch(lock_configuration=self.lock_configuration),
+            patch_set=locker.patch(universal_target=self.universal_target),
         )
         return self.pip.spawn_download_distributions(
             download_dir=download_dir,

pex/resolve/lock_downloader.py

@@ -9,7 +9,7 @@ from collections import OrderedDict
 from multiprocessing.pool import ThreadPool
 
 from pex import resolver
-from pex.auth import PasswordDatabase, PasswordEntry
+from pex.auth import PasswordDatabase
 from pex.common import pluralize
 from pex.compatibility import cpu_count
 from pex.dist_metadata import Requirement
@@ -31,6 +31,7 @@ from pex.resolve.locked_resolve import (
     VCSArtifact,
 )
 from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager
+from pex.resolve.package_repository import ReposConfiguration
 from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion
 from pex.resolve.resolvers import MAX_PARALLEL_DOWNLOADS, Resolver
 from pex.result import Error, catch
@@ -76,11 +77,9 @@ class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):
         self,
         target,  # type: Target
         file_lock_style,  # type: FileLockStyle.Value
-        indexes=None,  # type: Optional[Sequence[str]]
-        find_links=None,  # type: Optional[Sequence[str]]
+        repos_configuration=ReposConfiguration(),  # type: ReposConfiguration
         resolver_version=None,  # type: Optional[ResolverVersion.Value]
         network_configuration=None,  # type: Optional[NetworkConfiguration]
-        password_entries=(),  # type: Iterable[PasswordEntry]
         cache=None,  # type: Optional[str]
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         pex_root=ENV,  # type: Union[str, Variables]
@@ -94,11 +93,9 @@ class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):
             pex_root=pex_root, file_lock_style=file_lock_style
         )
         self._target = target
-        self._indexes = indexes
-        self._find_links = find_links
+        self._repos_configuration = repos_configuration
         self._resolver_version = resolver_version
         self._network_configuration = network_configuration
-        self._password_entries = password_entries
         self._cache = cache
 
         # Since a VCSArtifactDownloadManager is only used for VCS requirements, a build is both
@@ -127,11 +124,9 @@ class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):
             targets=Targets.from_target(self._target),
             requirements=[requirement],
             transitive=False,
-            indexes=self._indexes,
-            find_links=self._find_links,
+            repos_configuration=self._repos_configuration,
             resolver_version=self._resolver_version,
             network_configuration=self._network_configuration,
-            password_entries=self._password_entries,
             build_configuration=self._build_configuration,
             max_parallel_jobs=1,
             pip_version=self._pip_version,
@@ -159,10 +154,10 @@ class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):
         local_distribution = downloaded_vcs.local_distributions[0]
         filename = os.path.basename(local_distribution.path)
         digest_vcs_archive(
+            project_name=project_name,
             archive_path=local_distribution.path,
             vcs=artifact.vcs,
             digest=digest,
-            subdirectory=artifact.subdirectory,
         )
         shutil.move(local_distribution.path, os.path.join(dest_dir, filename))
         return filename
@@ -213,13 +208,11 @@ class LockDownloader(object):
         targets,  # type: Iterable[Target]
         lock_configuration,  # type: LockConfiguration
         resolver,  # type: Resolver
-        indexes=None,  # type: Optional[Sequence[str]]
-        find_links=None,  # type: Optional[Sequence[str]]
+        repos_configuration=ReposConfiguration(),  # type: ReposConfiguration
         max_parallel_jobs=None,  # type: Optional[int]
         pip_version=None,  # type: Optional[PipVersionValue]
         resolver_version=None,  # type: Optional[ResolverVersion.Value]
         network_configuration=None,  # type: Optional[NetworkConfiguration]
-        password_entries=(),  # type: Iterable[PasswordEntry]
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         use_pip_config=False,  # type: bool
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
@@ -234,22 +227,24 @@ class LockDownloader(object):
         # errors under the right interleaving of processes and threads and download artifact targets.
         file_lock_style = FileLockStyle.BSD
 
+        password_database = PasswordDatabase.from_netrc().append(
+            repos_configuration.password_entries
+        )
+        repos_configuration = attr.evolve(
+            repos_configuration, password_entries=password_database.entries
+        )
         file_download_managers_by_target = {
             target: FileArtifactDownloadManager(
                 file_lock_style=file_lock_style,
                 downloader=ArtifactDownloader(
                     resolver=resolver,
-                    lock_configuration=lock_configuration,
+                    universal_target=lock_configuration.universal_target,
                     target=target,
                     package_index_configuration=PackageIndexConfiguration.create(
                         pip_version=pip_version,
                         resolver_version=resolver_version,
-                        indexes=indexes,
-                        find_links=find_links,
+                        repos_configuration=repos_configuration,
                         network_configuration=network_configuration,
-                        password_entries=(
-                            PasswordDatabase.from_netrc().append(password_entries).entries
-                        ),
                         use_pip_config=use_pip_config,
                         extra_pip_requirements=extra_pip_requirements,
                         keyring_provider=keyring_provider,
@@ -264,11 +259,9 @@ class LockDownloader(object):
             target: VCSArtifactDownloadManager(
                 target=target,
                 file_lock_style=file_lock_style,
-                indexes=indexes,
-                find_links=find_links,
+                repos_configuration=repos_configuration,
                 resolver_version=resolver_version,
                 network_configuration=network_configuration,
-                password_entries=password_entries,
                 build_configuration=build_configuration,
                 pip_version=pip_version,
                 resolver=resolver,