pex 2.54.2__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

pex/resolve/lockfile/targets.py

@@ -0,0 +1,445 @@
+from __future__ import absolute_import
+
+import itertools
+import os.path
+import tempfile
+from collections import OrderedDict, defaultdict
+
+from pex import atexit
+from pex.common import pluralize, safe_delete
+from pex.interpreter_constraints import iter_compatible_versions
+from pex.interpreter_implementation import InterpreterImplementation
+from pex.network_configuration import NetworkConfiguration
+from pex.orderedset import OrderedSet
+from pex.pep_503 import ProjectName
+from pex.resolve.package_repository import ReposConfiguration
+from pex.resolve.requirement_configuration import RequirementConfiguration
+from pex.resolve.target_system import (
+    ExtraMarkers,
+    MarkerEnv,
+    TargetSystem,
+    UniversalTarget,
+    has_marker,
+)
+from pex.resolver import DownloadRequest
+from pex.targets import LocalInterpreter, Targets
+from pex.third_party.packaging.markers import Marker
+from pex.third_party.packaging.specifiers import SpecifierSet
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import (
+        DefaultDict,
+        Dict,
+        FrozenSet,
+        Iterable,
+        Iterator,
+        List,
+        Mapping,
+        Optional,
+        Text,
+        Tuple,
+    )
+
+    import attr  # vendor:skip
+else:
+    from pex.third_party import attr
+
+
+def _calculate_split_markers(
+    requirement_configuration,  # type: RequirementConfiguration
+    network_configuration,  # type: NetworkConfiguration
+    repos_configuration,  # type: ReposConfiguration
+):
+    # type: (...) -> Mapping[str, Marker]
+
+    split_markers = {
+        str(scope.marker): scope.marker
+        for repo in itertools.chain(
+            repos_configuration.index_repos, repos_configuration.find_links_repos
+        )
+        for scope in repo.scopes
+        if scope.marker
+    }
+
+    projects_with_markers = defaultdict(dict)  # type: DefaultDict[ProjectName, Dict[str, Marker]]
+    for requirement in requirement_configuration.parse_requirements(network_configuration):
+        if requirement.project_name and requirement.marker:
+            projects_with_markers[requirement.project_name][
+                str(requirement.marker)
+            ] = requirement.marker
+    for constraint in requirement_configuration.parse_constraints(network_configuration):
+        if constraint.marker:
+            projects_with_markers[constraint.project_name][
+                str(constraint.marker)
+            ] = constraint.marker
+    split_markers.update(
+        (marker_str, marker)
+        for markers in projects_with_markers.values()
+        for marker_str, marker in markers.items()
+        # N.B.: We only split the universal resolve for root requirements that have two or
+        # more marker variants. If there is just one, that represents a conditional
+        # dependency which can be included in a single resolve without splitting.
+        if len(markers) > 1
+    )
+
+    return split_markers
+
+
+def _iter_universal_targets(
+    universal_target,  # type: UniversalTarget
+    split_markers,  # type: Mapping[str, Marker]
+):
+    # type: (...) -> Iterator[UniversalTarget]
+
+    target_systems = universal_target.systems or TargetSystem.values()
+
+    interpreter_implementations = (
+        (universal_target.implementation,)
+        if universal_target.implementation
+        else InterpreterImplementation.values()
+    )
+
+    requires_pythons = OrderedSet()  # type: OrderedSet[SpecifierSet]
+    has_python_version = any(
+        has_marker(marker, "python_version") for marker in split_markers.values()
+    )
+    has_python_full_version = any(
+        has_marker(marker, "python_full_version") for marker in split_markers.values()
+    )
+    if has_python_full_version:
+        requires_pythons.update(
+            SpecifierSet(
+                "=={major}.{minor}.{patch}".format(
+                    major=version[0], minor=version[1], patch=version[2]
+                )
+            )
+            for version in iter_compatible_versions(universal_target.requires_python)
+        )
+    elif has_python_version:
+        requires_pythons.update(
+            SpecifierSet("=={major}.{minor}.*".format(major=version[0], minor=version[1]))
+            for version in OrderedSet(
+                version[:2]
+                for version in iter_compatible_versions(universal_target.requires_python)
+            )
+        )
+    else:
+        requires_pythons.update(universal_target.requires_python)
+    if not requires_pythons:
+        requires_pythons.add(SpecifierSet())
+
+    systems_by_markers = defaultdict(
+        list
+    )  # type: DefaultDict[FrozenSet[str], List[Tuple[TargetSystem.Value, InterpreterImplementation.Value, SpecifierSet]]]
+    for system in target_systems:
+        for implementation in interpreter_implementations:
+            for python_specifier in requires_pythons:
+                marker_env = MarkerEnv.create(
+                    extras=(),
+                    universal_target=UniversalTarget(
+                        implementation=implementation,
+                        systems=(system,),
+                        requires_python=(python_specifier,),
+                    ),
+                )
+                system_repo_markers = frozenset(
+                    marker_str
+                    for marker_str, marker in split_markers.items()
+                    if marker_env.evaluate(marker)
+                )
+                systems_by_markers[system_repo_markers].append(
+                    (system, implementation, python_specifier)
+                )
+
+    for markers, value in systems_by_markers.items():
+        systems = OrderedSet()  # type: OrderedSet[TargetSystem.Value]
+        implementations = OrderedSet()  # type: OrderedSet[InterpreterImplementation.Value]
+        requires_python = OrderedSet()  # type: OrderedSet[SpecifierSet]
+        for system, implementation, python_specifier in value:
+            systems.add(system)
+            implementations.add(implementation)
+            requires_python.add(python_specifier)
+        impl = implementations.pop() if len(implementations) == 1 else None
+        yield UniversalTarget(
+            implementation=impl,
+            systems=tuple(systems),
+            requires_python=tuple(requires_python),
+        )
+
+
+if TYPE_CHECKING:
+    from pex.requirements import ParsedRequirement
+
+
+@attr.s(frozen=True)
+class DownloadInput(object):
+    download_requests = attr.ib()  # type: Tuple[DownloadRequest, ...]
+    direct_requirements = attr.ib()  # type: Tuple[ParsedRequirement, ...]
+
+
+def _comment_out_requirements(
+    requirements_file,  # type: Text
+    requirements,  # type: Iterable[ParsedRequirement]
+):
+    # type: (...) -> Text
+
+    lines_to_comment_out = set(
+        itertools.chain.from_iterable(
+            range(requirement.line.start_line, requirement.line.end_line + 1)
+            for requirement in requirements
+        )
+    )
+
+    # N.B.: We create the edited requirements file in the same directory as the original so that any
+    # relative path references in the requirements file are still valid.
+    out_fd, edited_requirements_file = tempfile.mkstemp(
+        dir=os.path.dirname(requirements_file),
+        prefix="pex_lock_split.",
+        suffix=".{file_name}".format(file_name=os.path.basename(requirements_file)),
+    )
+    atexit.register(safe_delete, edited_requirements_file)
+    try:
+        with open(requirements_file, "rb") as in_fp:
+            for line_no, text in enumerate(in_fp, start=1):
+                if line_no in lines_to_comment_out:
+                    os.write(out_fd, b"# ")
+                os.write(out_fd, text)
+    finally:
+        os.close(out_fd)
+    return edited_requirements_file
+
+
+@attr.s
+class Split(object):
+    requirements_by_project_name = attr.ib(
+        factory=OrderedDict
+    )  # type: OrderedDict[ProjectName, ParsedRequirement]
+    provenance = attr.ib(factory=OrderedSet)  # type: OrderedSet[ParsedRequirement]
+
+    def applies(
+        self,
+        universal_target,  # type: UniversalTarget
+        requirement,  # type: ParsedRequirement
+    ):
+        # type: (...) -> bool
+
+        if not requirement.marker:
+            return True
+
+        requirements = [(req.marker, str(req)) for req in self.provenance if req.marker]
+        if not requirements:
+            return True
+
+        marker_env = attr.evolve(
+            universal_target, extra_markers=ExtraMarkers.extract(requirements)
+        ).marker_env()
+        return marker_env.evaluate(requirement.marker)
+
+    def add(
+        self,
+        universal_target,  # type: UniversalTarget
+        project_name,  # type: ProjectName
+        requirement,  # type: ParsedRequirement
+    ):
+        # type: (...) -> Optional[Split]
+
+        if not self.applies(universal_target, requirement):
+            return None
+
+        existing_requirement = self.requirements_by_project_name.setdefault(
+            project_name, requirement
+        )
+        if existing_requirement == requirement:
+            return None
+
+        self.provenance.add(existing_requirement)
+
+        provenance = OrderedSet(req for req in self.provenance if req != existing_requirement)
+        provenance.add(requirement)
+
+        requirements_by_project_name = self.requirements_by_project_name.copy()
+        requirements_by_project_name[project_name] = requirement
+
+        return Split(
+            requirements_by_project_name=requirements_by_project_name, provenance=provenance
+        )
+
+    def requirement_configuration(
+        self,
+        unnamed_requirements,  # type: Iterable[ParsedRequirement]
+        requirement_configuration,  # type: RequirementConfiguration
+        network_configuration=None,  # type: Optional[NetworkConfiguration]
+    ):
+        # type: (...) -> RequirementConfiguration
+
+        if not self.provenance:
+            return requirement_configuration
+
+        requirements = list(str(req) for req in unnamed_requirements)
+
+        requirement_files = OrderedSet(
+            os.path.realpath(requirement_file)
+            for requirement_file in requirement_configuration.requirement_files
+        )  # type: OrderedSet[Text]
+        if requirement_files:
+            provenance_by_project_name = {
+                parsed_requirement.project_name: parsed_requirement
+                for parsed_requirement in self.provenance
+                if parsed_requirement.project_name
+            }
+
+            requirements_to_comment_out_by_source = defaultdict(
+                list
+            )  # type: DefaultDict[Text, List[ParsedRequirement]]
+            for parsed_requirement in requirement_configuration.parse_requirements(
+                network_configuration=network_configuration
+            ):
+                if not parsed_requirement.project_name:
+                    continue
+
+                provenance = provenance_by_project_name.get(parsed_requirement.project_name)
+                if provenance and (parsed_requirement != provenance):
+                    if parsed_requirement.line.source in requirement_files:
+                        # We comment out the requirement
+                        requirements_to_comment_out_by_source[
+                            parsed_requirement.line.source
+                        ].append(parsed_requirement)
+                    else:
+                        # We drop the requirement
+                        pass
+                elif parsed_requirement.line.source not in requirement_files:
+                    requirements.append(str(parsed_requirement))
+            if requirements_to_comment_out_by_source:
+                new_requirement_files = OrderedSet()  # type: OrderedSet[Text]
+                for source, parsed_requirements in requirements_to_comment_out_by_source.items():
+                    if source in requirement_files:
+                        new_requirement_files.add(
+                            _comment_out_requirements(source, parsed_requirements)
+                        )
+                    else:
+                        new_requirement_files.add(source)
+                requirement_files = new_requirement_files
+        else:
+            requirements.extend(str(req) for req in self.requirements_by_project_name.values())
+
+        return RequirementConfiguration(
+            requirements=tuple(requirements),
+            requirement_files=tuple(requirement_files),
+            constraint_files=requirement_configuration.constraint_files,
+        )
+
+
+def calculate_download_input(
+    targets,  # type: Targets
+    requirement_configuration,  # type: RequirementConfiguration
+    network_configuration,  # type: NetworkConfiguration
+    repos_configuration,  # type: ReposConfiguration
+    universal_target=None,  # type: Optional[UniversalTarget]
+):
+    # type: (...) -> DownloadInput
+
+    direct_requirements = requirement_configuration.parse_requirements(network_configuration)
+    if not universal_target:
+        return DownloadInput(
+            download_requests=tuple(
+                DownloadRequest.create(
+                    target=target, requirement_configuration=requirement_configuration
+                )
+                for target in targets.unique_targets()
+            ),
+            direct_requirements=direct_requirements,
+        )
+
+    target = LocalInterpreter.create(targets.interpreter)
+    split_markers = _calculate_split_markers(
+        requirement_configuration, network_configuration, repos_configuration
+    )
+    if not split_markers:
+        return DownloadInput(
+            download_requests=tuple(
+                [
+                    DownloadRequest.create(
+                        target=target,
+                        universal_target=universal_target,
+                        requirement_configuration=requirement_configuration,
+                    )
+                ]
+            ),
+            direct_requirements=direct_requirements,
+        )
+
+    named_requirements = (
+        OrderedDict()
+    )  # type: OrderedDict[ProjectName, OrderedSet[ParsedRequirement]]
+    unnamed_requirements = OrderedSet()  # type: OrderedSet[ParsedRequirement]
+    for direct_requirement in direct_requirements:
+        if direct_requirement.project_name:
+            named_requirements.setdefault(direct_requirement.project_name, OrderedSet()).add(
+                direct_requirement
+            )
+        else:
+            unnamed_requirements.add(direct_requirement)
+
+    requirement_splits_by_universal_target = defaultdict(
+        lambda: [Split()]
+    )  # type: DefaultDict[UniversalTarget, List[Split]]
+    for universal_target in _iter_universal_targets(universal_target, split_markers):
+        marker_env = universal_target.marker_env()
+        requirement_splits = requirement_splits_by_universal_target[universal_target]
+        for project_name, remote_requirements in named_requirements.items():
+            for requirement_split in list(requirement_splits):
+                for remote_requirement in remote_requirements:
+                    if remote_requirement.marker and not marker_env.evaluate(
+                        remote_requirement.marker
+                    ):
+                        continue
+                    new_split = requirement_split.add(
+                        universal_target, project_name, remote_requirement
+                    )
+                    if new_split:
+                        requirement_splits.append(new_split)
+
+    download_requests = []
+    for universal_target, splits in requirement_splits_by_universal_target.items():
+        if len(splits) == 1:
+            download_requests.append(
+                DownloadRequest.create(
+                    target=target,
+                    universal_target=universal_target,
+                    requirement_configuration=splits[0].requirement_configuration(
+                        unnamed_requirements,
+                        requirement_configuration,
+                        network_configuration=network_configuration,
+                    ),
+                )
+            )
+            continue
+
+        for split in splits:
+            download_requests.append(
+                DownloadRequest.create(
+                    target=target,
+                    universal_target=attr.evolve(
+                        universal_target,
+                        extra_markers=ExtraMarkers.extract(
+                            (requirement.marker, str(requirement))
+                            for requirement in split.provenance
+                            if requirement.marker
+                        ),
+                    ),
+                    requirement_configuration=split.requirement_configuration(
+                        unnamed_requirements,
+                        requirement_configuration,
+                        network_configuration=network_configuration,
+                    ),
+                    provenance="split by {requirements} {reqs}".format(
+                        requirements=pluralize(split.provenance, "requirement"),
+                        reqs=", ".join("'{req}'".format(req=req) for req in split.provenance),
+                    ),
+                )
+            )
+
+    return DownloadInput(
+        download_requests=tuple(download_requests), direct_requirements=direct_requirements
+    )

pex/resolve/lockfile/updater.py

@@ -26,8 +26,9 @@ from pex.resolve.locked_resolve import (
 )
 from pex.resolve.lockfile.create import create
 from pex.resolve.lockfile.model import Lockfile
+from pex.resolve.package_repository import ReposConfiguration
 from pex.resolve.requirement_configuration import RequirementConfiguration
-from pex.resolve.resolver_configuration import PipConfiguration, PipLog, ReposConfiguration
+from pex.resolve.resolver_configuration import PipConfiguration, PipLog
 from pex.result import Error, ResultError, catch, try_
 from pex.sorted_tuple import SortedTuple
 from pex.targets import Target, Targets
@@ -251,6 +252,7 @@ class ResolveUpdater(object):
         lock_configuration,  # type: LockConfiguration
         pip_configuration,  # type: PipConfiguration
         dependency_configuration,  # type: DependencyConfiguration
+        avoid_downloads,  # type: bool
     ):
         # type: (...) -> Union[ResolveUpdater, Error]
 
@@ -309,6 +311,7 @@ class ResolveUpdater(object):
             lock_configuration=lock_configuration,
             pip_configuration=pip_configuration,
             dependency_configuration=dependency_configuration,
+            avoid_downloads=avoid_downloads,
         )
 
     @classmethod
@@ -321,6 +324,7 @@ class ResolveUpdater(object):
         lock_configuration,  # type: LockConfiguration
         pip_configuration,  # type: PipConfiguration
         dependency_configuration,  # type: DependencyConfiguration
+        avoid_downloads,  # type: bool
     ):
         # type: (...) -> ResolveUpdater
 
@@ -361,6 +365,7 @@ class ResolveUpdater(object):
             lock_configuration=lock_configuration,
             pip_configuration=pip_configuration,
             dependency_configuration=dependency_configuration,
+            avoid_downloads=avoid_downloads,
         )
 
     requirement_configuration = attr.ib()  # type: RequirementConfiguration
@@ -376,6 +381,7 @@ class ResolveUpdater(object):
     update_requirements_by_project_name = attr.ib(
         factory=dict
     )  # type: Mapping[ProjectName, Requirement]
+    avoid_downloads = attr.ib(default=False)
 
     def iter_updated_requirements(self):
         # type: () -> Iterator[Requirement]
@@ -501,6 +507,7 @@ class ResolveUpdater(object):
                             targets=Targets.from_target(target),
                             pip_configuration=self.pip_configuration,
                             dependency_configuration=self.dependency_configuration,
+                            avoid_downloads=self.avoid_downloads,
                         )
                     )
                     assert 1 == len(updated_lock_file.locked_resolves)
@@ -690,6 +697,7 @@ class LockUpdater(object):
         use_pip_config,  # type: bool
         dependency_configuration,  # type: DependencyConfiguration
         pip_log,  # type: Optional[PipLog]
+        avoid_downloads,  # type: bool
     ):
         # type: (...) -> LockUpdater
 
@@ -707,15 +715,17 @@ class LockUpdater(object):
         )
         return cls(
             lock_file=lock_file,
-            lock_configuration=lock_file.lock_configuration(),
+            lock_configuration=lock_file.configuration,
             pip_configuration=pip_configuration,
             dependency_configuration=dependency_configuration,
+            avoid_downloads=avoid_downloads,
         )
 
     lock_file = attr.ib()  # type: Lockfile
     lock_configuration = attr.ib()  # type: LockConfiguration
     pip_configuration = attr.ib()  # type: PipConfiguration
     dependency_configuration = attr.ib()  # type: DependencyConfiguration
+    avoid_downloads = attr.ib()  # type: bool
 
     def sync(
         self,
@@ -737,6 +747,7 @@ class LockUpdater(object):
                 lock_configuration=self.lock_configuration,
                 pip_configuration=self.pip_configuration,
                 dependency_configuration=self.dependency_configuration,
+                avoid_downloads=self.avoid_downloads,
             )
         )
         return self._perform_update(
@@ -772,6 +783,7 @@ class LockUpdater(object):
             lock_configuration=self.lock_configuration,
             pip_configuration=self.pip_configuration,
             dependency_configuration=self.dependency_configuration,
+            avoid_downloads=self.avoid_downloads,
         )
         return self._perform_update(
             update_requests=update_requests,
@@ -788,11 +800,11 @@ class LockUpdater(object):
         # type: (...) -> Union[LockUpdate, Error]
 
         error_by_target = OrderedDict()  # type: OrderedDict[Target, Error]
-        locked_resolve_by_platform_tag = OrderedDict(
-            (locked_resolve.platform_tag, locked_resolve)
+        locked_resolve_by_target_platform = OrderedDict(
+            (locked_resolve.target_platform, locked_resolve)
             for locked_resolve in self.lock_file.locked_resolves
         )  # type: OrderedDict[Optional[tags.Tag], LockedResolve]
-        resolve_updates_by_platform_tag = (
+        resolve_updates_by_target_platform = (
             {}
         )  # type: Dict[Optional[tags.Tag], Mapping[ProjectName, Optional[Update]]]
 
@@ -808,9 +820,9 @@ class LockUpdater(object):
             if isinstance(result, Error):
                 error_by_target[update_request.target] = result
             else:
-                platform_tag = update_request.locked_resolve.platform_tag
-                locked_resolve_by_platform_tag[platform_tag] = result.updated_resolve
-                resolve_updates_by_platform_tag[platform_tag] = result.updates
+                target_platform = update_request.locked_resolve.target_platform
+                locked_resolve_by_target_platform[target_platform] = result.updated_resolve
+                resolve_updates_by_target_platform[target_platform] = result.updates
 
         if error_by_target:
             return Error(
@@ -833,8 +845,8 @@ class LockUpdater(object):
             resolves=tuple(
                 ResolveUpdate(
                     updated_resolve=updated_resolve,
-                    updates=resolve_updates_by_platform_tag.get(platform_tag, {}),
+                    updates=resolve_updates_by_target_platform.get(target_platform, {}),
                 )
-                for platform_tag, updated_resolve in locked_resolve_by_platform_tag.items()
+                for target_platform, updated_resolve in locked_resolve_by_target_platform.items()
             ),
         )