pex 2.61.1__py2.py3-none-any.whl → 2.62.1__py2.py3-none-any.whl

pex/resolve/lockfile/targets.py

@@ -1,24 +1,45 @@
 from __future__ import absolute_import
 
 import itertools
-from collections import defaultdict
+import os.path
+import tempfile
+from collections import OrderedDict, defaultdict
 
+from pex import atexit
+from pex.common import pluralize, safe_delete
 from pex.interpreter_constraints import iter_compatible_versions
 from pex.interpreter_implementation import InterpreterImplementation
 from pex.network_configuration import NetworkConfiguration
 from pex.orderedset import OrderedSet
 from pex.pep_503 import ProjectName
-from pex.requirements import LocalProjectRequirement
 from pex.resolve.package_repository import ReposConfiguration
 from pex.resolve.requirement_configuration import RequirementConfiguration
-from pex.resolve.target_system import MarkerEnv, TargetSystem, UniversalTarget, has_marker
+from pex.resolve.target_system import (
+    ExtraMarkers,
+    MarkerEnv,
+    TargetSystem,
+    UniversalTarget,
+    has_marker,
+)
+from pex.resolver import DownloadRequest
 from pex.targets import LocalInterpreter, Targets
 from pex.third_party.packaging.markers import Marker
 from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Dict, FrozenSet, Iterator, List, Mapping, Optional, Tuple
+    from typing import (
+        DefaultDict,
+        Dict,
+        FrozenSet,
+        Iterable,
+        Iterator,
+        List,
+        Mapping,
+        Optional,
+        Text,
+        Tuple,
+    )
 
     import attr  # vendor:skip
 else:
@@ -43,7 +64,7 @@ def _calculate_split_markers(
 
     projects_with_markers = defaultdict(dict)  # type: DefaultDict[ProjectName, Dict[str, Marker]]
     for requirement in requirement_configuration.parse_requirements(network_configuration):
-        if not isinstance(requirement, LocalProjectRequirement) and requirement.marker:
+        if requirement.project_name and requirement.marker:
             projects_with_markers[requirement.project_name][
                 str(requirement.marker)
             ] = requirement.marker
@@ -147,33 +168,278 @@ def _iter_universal_targets(
         )
 
 
+if TYPE_CHECKING:
+    from pex.requirements import ParsedRequirement
+
+
 @attr.s(frozen=True)
-class LockTargets(object):
-    @classmethod
-    def calculate(
-        cls,
-        targets,  # type: Targets
+class DownloadInput(object):
+    download_requests = attr.ib()  # type: Tuple[DownloadRequest, ...]
+    direct_requirements = attr.ib()  # type: Tuple[ParsedRequirement, ...]
+
+
+def _comment_out_requirements(
+    requirements_file,  # type: Text
+    requirements,  # type: Iterable[ParsedRequirement]
+):
+    # type: (...) -> Text
+
+    lines_to_comment_out = set(
+        itertools.chain.from_iterable(
+            range(requirement.line.start_line, requirement.line.end_line + 1)
+            for requirement in requirements
+        )
+    )
+
+    # N.B.: We create the edited requirements file in the same directory as the original so that any
+    # relative path references in the requirements file are still valid.
+    out_fd, edited_requirements_file = tempfile.mkstemp(
+        dir=os.path.dirname(requirements_file),
+        prefix="pex_lock_split.",
+        suffix=".{file_name}".format(file_name=os.path.basename(requirements_file)),
+    )
+    atexit.register(safe_delete, edited_requirements_file)
+    try:
+        with open(requirements_file, "rb") as in_fp:
+            for line_no, text in enumerate(in_fp, start=1):
+                if line_no in lines_to_comment_out:
+                    os.write(out_fd, b"# ")
+                os.write(out_fd, text)
+    finally:
+        os.close(out_fd)
+    return edited_requirements_file
+
+
+@attr.s
+class Split(object):
+    requirements_by_project_name = attr.ib(
+        factory=OrderedDict
+    )  # type: OrderedDict[ProjectName, ParsedRequirement]
+    provenance = attr.ib(factory=OrderedSet)  # type: OrderedSet[ParsedRequirement]
+
+    def applies(
+        self,
+        universal_target,  # type: UniversalTarget
+        requirement,  # type: ParsedRequirement
+    ):
+        # type: (...) -> bool
+
+        if not requirement.marker:
+            return True
+
+        requirements = [(req.marker, str(req)) for req in self.provenance if req.marker]
+        if not requirements:
+            return True
+
+        marker_env = attr.evolve(
+            universal_target, extra_markers=ExtraMarkers.extract(requirements)
+        ).marker_env()
+        return marker_env.evaluate(requirement.marker)
+
+    def add(
+        self,
+        universal_target,  # type: UniversalTarget
+        project_name,  # type: ProjectName
+        requirement,  # type: ParsedRequirement
+    ):
+        # type: (...) -> Optional[Split]
+
+        if not self.applies(universal_target, requirement):
+            return None
+
+        existing_requirement = self.requirements_by_project_name.setdefault(
+            project_name, requirement
+        )
+        if existing_requirement == requirement:
+            return None
+
+        self.provenance.add(existing_requirement)
+
+        provenance = OrderedSet(req for req in self.provenance if req != existing_requirement)
+        provenance.add(requirement)
+
+        requirements_by_project_name = self.requirements_by_project_name.copy()
+        requirements_by_project_name[project_name] = requirement
+
+        return Split(
+            requirements_by_project_name=requirements_by_project_name, provenance=provenance
+        )
+
+    def requirement_configuration(
+        self,
+        unnamed_requirements,  # type: Iterable[ParsedRequirement]
         requirement_configuration,  # type: RequirementConfiguration
-        network_configuration,  # type: NetworkConfiguration
-        repos_configuration,  # type: ReposConfiguration
-        universal_target=None,  # type: Optional[UniversalTarget]
+        network_configuration=None,  # type: Optional[NetworkConfiguration]
     ):
-        # type: (...) -> LockTargets
+        # type: (...) -> RequirementConfiguration
+
+        if not self.provenance:
+            return requirement_configuration
+
+        requirements = list(str(req) for req in unnamed_requirements)
+
+        requirement_files = OrderedSet(
+            os.path.realpath(requirement_file)
+            for requirement_file in requirement_configuration.requirement_files
+        )  # type: OrderedSet[Text]
+        if requirement_files:
+            provenance_by_project_name = {
+                parsed_requirement.project_name: parsed_requirement
+                for parsed_requirement in self.provenance
+                if parsed_requirement.project_name
+            }
 
-        if not universal_target:
-            return cls(targets=targets)
+            requirements_to_comment_out_by_source = defaultdict(
+                list
+            )  # type: DefaultDict[Text, List[ParsedRequirement]]
+            for parsed_requirement in requirement_configuration.parse_requirements(
+                network_configuration=network_configuration
+            ):
+                if not parsed_requirement.project_name:
+                    continue
 
-        targets = Targets.from_target(LocalInterpreter.create(targets.interpreter))
-        split_markers = _calculate_split_markers(
-            requirement_configuration, network_configuration, repos_configuration
+                provenance = provenance_by_project_name.get(parsed_requirement.project_name)
+                if provenance and (parsed_requirement != provenance):
+                    if parsed_requirement.line.source in requirement_files:
+                        # We comment out the requirement
+                        requirements_to_comment_out_by_source[
+                            parsed_requirement.line.source
+                        ].append(parsed_requirement)
+                    else:
+                        # We drop the requirement
+                        pass
+                elif parsed_requirement.line.source not in requirement_files:
+                    requirements.append(str(parsed_requirement))
+            if requirements_to_comment_out_by_source:
+                new_requirement_files = OrderedSet()  # type: OrderedSet[Text]
+                for source, parsed_requirements in requirements_to_comment_out_by_source.items():
+                    if source in requirement_files:
+                        new_requirement_files.add(
+                            _comment_out_requirements(source, parsed_requirements)
+                        )
+                    else:
+                        new_requirement_files.add(source)
+                requirement_files = new_requirement_files
+        else:
+            requirements.extend(str(req) for req in self.requirements_by_project_name.values())
+
+        return RequirementConfiguration(
+            requirements=tuple(requirements),
+            requirement_files=tuple(requirement_files),
+            constraint_files=requirement_configuration.constraint_files,
         )
-        if not split_markers:
-            return cls(targets=targets, universal_targets=(universal_target,))
 
-        return cls(
-            targets=targets,
-            universal_targets=tuple(_iter_universal_targets(universal_target, split_markers)),
+
+def calculate_download_input(
+    targets,  # type: Targets
+    requirement_configuration,  # type: RequirementConfiguration
+    network_configuration,  # type: NetworkConfiguration
+    repos_configuration,  # type: ReposConfiguration
+    universal_target=None,  # type: Optional[UniversalTarget]
+):
+    # type: (...) -> DownloadInput
+
+    direct_requirements = requirement_configuration.parse_requirements(network_configuration)
+    if not universal_target:
+        return DownloadInput(
+            download_requests=tuple(
+                DownloadRequest.create(
+                    target=target, requirement_configuration=requirement_configuration
+                )
+                for target in targets.unique_targets()
+            ),
+            direct_requirements=direct_requirements,
+        )
+
+    target = LocalInterpreter.create(targets.interpreter)
+    split_markers = _calculate_split_markers(
+        requirement_configuration, network_configuration, repos_configuration
+    )
+    if not split_markers:
+        return DownloadInput(
+            download_requests=tuple(
+                [
+                    DownloadRequest.create(
+                        target=target,
+                        universal_target=universal_target,
+                        requirement_configuration=requirement_configuration,
+                    )
+                ]
+            ),
+            direct_requirements=direct_requirements,
         )
 
-    targets = attr.ib()  # type: Targets
-    universal_targets = attr.ib(default=())  # type: Tuple[UniversalTarget, ...]
+    named_requirements = (
+        OrderedDict()
+    )  # type: OrderedDict[ProjectName, OrderedSet[ParsedRequirement]]
+    unnamed_requirements = OrderedSet()  # type: OrderedSet[ParsedRequirement]
+    for direct_requirement in direct_requirements:
+        if direct_requirement.project_name:
+            named_requirements.setdefault(direct_requirement.project_name, OrderedSet()).add(
+                direct_requirement
+            )
+        else:
+            unnamed_requirements.add(direct_requirement)
+
+    requirement_splits_by_universal_target = defaultdict(
+        lambda: [Split()]
+    )  # type: DefaultDict[UniversalTarget, List[Split]]
+    for universal_target in _iter_universal_targets(universal_target, split_markers):
+        marker_env = universal_target.marker_env()
+        requirement_splits = requirement_splits_by_universal_target[universal_target]
+        for project_name, remote_requirements in named_requirements.items():
+            for requirement_split in list(requirement_splits):
+                for remote_requirement in remote_requirements:
+                    if remote_requirement.marker and not marker_env.evaluate(
+                        remote_requirement.marker
+                    ):
+                        continue
+                    new_split = requirement_split.add(
+                        universal_target, project_name, remote_requirement
+                    )
+                    if new_split:
+                        requirement_splits.append(new_split)
+
+    download_requests = []
+    for universal_target, splits in requirement_splits_by_universal_target.items():
+        if len(splits) == 1:
+            download_requests.append(
+                DownloadRequest.create(
+                    target=target,
+                    universal_target=universal_target,
+                    requirement_configuration=splits[0].requirement_configuration(
+                        unnamed_requirements,
+                        requirement_configuration,
+                        network_configuration=network_configuration,
+                    ),
+                )
+            )
+            continue
+
+        for split in splits:
+            download_requests.append(
+                DownloadRequest.create(
+                    target=target,
+                    universal_target=attr.evolve(
+                        universal_target,
+                        extra_markers=ExtraMarkers.extract(
+                            (requirement.marker, str(requirement))
+                            for requirement in split.provenance
+                            if requirement.marker
+                        ),
+                    ),
+                    requirement_configuration=split.requirement_configuration(
+                        unnamed_requirements,
+                        requirement_configuration,
+                        network_configuration=network_configuration,
+                    ),
+                    provenance="split by {requirements} {reqs}".format(
+                        requirements=pluralize(split.provenance, "requirement"),
+                        reqs=", ".join("'{req}'".format(req=req) for req in split.provenance),
+                    ),
+                )
+            )
+
+    return DownloadInput(
+        download_requests=tuple(download_requests), direct_requirements=direct_requirements
+    )

pex/resolve/requirement_configuration.py

@@ -17,7 +17,7 @@ from pex.requirements import (
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterable, List, Optional
+    from typing import Iterable, List, Optional, Tuple
 
     import attr  # vendor:skip
 
@@ -26,14 +26,21 @@ else:
     from pex.third_party import attr
 
 
+def _as_str_tuple(items):
+    # type: (Optional[Iterable[str]]) -> Tuple[str, ...]
+    if not items:
+        return ()
+    return items if isinstance(items, tuple) else tuple(items)
+
+
 @attr.s(frozen=True)
 class RequirementConfiguration(object):
-    requirements = attr.ib(default=None)  # type: Optional[Iterable[str]]
-    requirement_files = attr.ib(default=None)  # type: Optional[Iterable[str]]
-    constraint_files = attr.ib(default=None)  # type: Optional[Iterable[str]]
+    requirements = attr.ib(default=(), converter=_as_str_tuple)  # type: Tuple[str, ...]
+    requirement_files = attr.ib(default=(), converter=_as_str_tuple)  # type: Tuple[str, ...]
+    constraint_files = attr.ib(default=(), converter=_as_str_tuple)  # type: Tuple[str, ...]
 
     def parse_requirements(self, network_configuration=None):
-        # type: (Optional[NetworkConfiguration]) -> Iterable[ParsedRequirement]
+        # type: (Optional[NetworkConfiguration]) -> Tuple[ParsedRequirement, ...]
         parsed_requirements = []  # type: List[ParsedRequirement]
         if self.requirements:
             parsed_requirements.extend(parse_requirement_strings(self.requirements))
@@ -50,10 +57,10 @@ class RequirementConfiguration(object):
                         (PyPIRequirement, URLRequirement, VCSRequirement, LocalProjectRequirement),
                     )
                 )
-        return parsed_requirements
+        return tuple(parsed_requirements)
 
     def parse_constraints(self, network_configuration=None):
-        # type: (Optional[NetworkConfiguration]) -> Iterable[Constraint]
+        # type: (Optional[NetworkConfiguration]) -> Tuple[Constraint, ...]
         parsed_constraints = []  # type: List[Constraint]
         if self.constraint_files:
             fetcher = URLFetcher(network_configuration=network_configuration)
@@ -65,7 +72,7 @@ class RequirementConfiguration(object):
                     )
                     if isinstance(requirement_or_constraint, Constraint)
                 )
-        return parsed_constraints
+        return tuple(parsed_constraints)
 
     @property
     def has_requirements(self):