pex 2.59.0__tar.gz → 2.59.2__tar.gz

{pex-2.59.0 → pex-2.59.2}/CHANGES.md

@@ -1,5 +1,21 @@
 # Release Notes
 
+## 2.59.2
+
+This release fixes two bugs handling split universal resolves. Previously, when a universal resolve
+was split by markers other than `python_version` and `python_full_version` and no interpreter
+constraints were specified, locking would fail. Additionally, when a split lock had differing
+transitive dependencies in splits, lock sub-setting would fail. Both issues are now corrected.
+
+* Fix split universal lock corner cases. (#2922)
+
+## 2.59.1
+
+This release fixes a regression in VCS URL handling introduced by Pex 2.38.0 when VCS URLs included
+user info in the authority.
+
+* Fix `pex3 lock create/export` for VCS URLs with userinfo (#2918)
+
 ## 2.59.0
 
 This release adds support for a `--venv-repository` resolution source. This allows creating a PEX

{pex-2.59.0 → pex-2.59.2}/PKG-INFO

@@ -1,15 +1,15 @@
 Metadata-Version: 2.4
 Name: pex
-Version: 2.59.0
+Version: 2.59.2
 Summary: The PEX packaging toolchain.
 Home-page: https://github.com/pex-tool/pex
-Download-URL: https://github.com/pex-tool/pex/releases/download/v2.59.0/pex
+Download-URL: https://github.com/pex-tool/pex/releases/download/v2.59.2/pex
 Author: The PEX developers
 Author-email: developers@pex-tool.org
 License-Expression: Apache-2.0
-Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.59.0/CHANGES.md
+Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.59.2/CHANGES.md
 Project-URL: Documentation, https://docs.pex-tool.org/
-Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.59.0
+Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.59.2
 Keywords: package,executable,virtualenv,lock,freeze
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers

{pex-2.59.0 → pex-2.59.2}/pex/cli/commands/lock.py

@@ -76,7 +76,7 @@ from pex.resolve.resolver_options import parse_lockfile
 from pex.resolve.resolvers import Resolver
 from pex.resolve.script_metadata import ScriptMetadataApplication, apply_script_metadata
 from pex.resolve.target_configuration import InterpreterConstraintsNotSatisfied, TargetConfiguration
-from pex.resolve.target_system import TargetSystem
+from pex.resolve.target_system import TargetSystem, UniversalTarget
 from pex.result import Error, Ok, Result, try_
 from pex.sorted_tuple import SortedTuple
 from pex.targets import LocalInterpreter, Target, Targets
@@ -535,6 +535,110 @@ class RootRequirements(object):
         return iter(self._root_requirements)
 
 
+def subset_locked_resolve(
+    locked_resolve,  # type: LockedResolve
+    root_requirements,  # type: RootRequirements
+    constraint_by_project_name,  # type: Mapping[ProjectName, Constraint]
+    lock_file_description,  # type: str
+    universal_target=None,  # type: Optional[UniversalTarget]
+):
+    # type: (...) -> Union[LockedResolve, Error]
+
+    available = {
+        locked_req.pin.project_name: (
+            ProjectNameAndVersion(locked_req.pin.project_name.raw, locked_req.pin.version.raw),
+            locked_req,
+        )
+        for locked_req in locked_resolve.locked_requirements
+    }
+    retain = set()
+    to_resolve = deque(root_requirements)  # type: Deque[Union[RootRequirement, Requirement]]
+    while to_resolve:
+        req = to_resolve.popleft()
+        if req.project_name in retain:
+            continue
+        retain.add(req.project_name)
+
+        dep = available.get(req.project_name)
+        if not dep:
+            return Error(
+                "There is no lock entry for {project} in {lock_file} to satisfy the "
+                "{transitive}{req} requirement.".format(
+                    project=req.project_name,
+                    lock_file=lock_file_description,
+                    transitive="" if isinstance(req, RootRequirement) else "transitive ",
+                    req=(req if isinstance(req, RootRequirement) else "'{req}'".format(req=req)),
+                )
+            )
+
+        pnav, locked_req = dep
+        if isinstance(req, RootRequirement):
+            reqs = req.select(pnav)
+            if not reqs:
+                return Error(
+                    "The locked version of {project} in {lock_file} is {version} which "
+                    "does not satisfy the {req} requirement.".format(
+                        project=pnav.project_name,
+                        lock_file=lock_file_description,
+                        version=pnav.version,
+                        req=req,
+                    )
+                )
+        elif pnav not in req:
+            production_assert(
+                isinstance(req, RootRequirement),
+                "Transitive requirements in a lock should always match existing lock "
+                "entries. Found {project} {version} in {lock_file}, which does not satisfy "
+                "transitive requirement '{req}' found in the same lock.",
+                project=pnav.project_name,
+                version=pnav.version,
+                lock_file=lock_file_description,
+                req=req,
+            )
+            return Error(
+                "The locked version of {project} in {lock_file} is {version} which does "
+                "not satisfy the '{req}' requirement.".format(
+                    project=pnav.project_name,
+                    lock_file=lock_file_description,
+                    version=pnav.version,
+                    req=req,
+                )
+            )
+        elif (
+            req.project_name in constraint_by_project_name
+            and pnav not in constraint_by_project_name[req.project_name]
+        ):
+            return Error(
+                "The locked version of {project} in {lock_file} is {version} which does "
+                "not satisfy the '{constraint}' constraint.".format(
+                    project=pnav.project_name,
+                    lock_file=lock_file_description,
+                    version=pnav.version,
+                    constraint=constraint_by_project_name[req.project_name],
+                )
+            )
+        else:
+            reqs = (req,)
+
+        for req in reqs:
+            to_resolve.extend(
+                requires_dist.filter_dependencies(
+                    req,
+                    locked_req,
+                    universal_target=universal_target,
+                )
+            )
+
+    return attr.evolve(
+        locked_resolve,
+        locked_requirements=SortedTuple(
+            locked_requirement
+            for locked_requirement in locked_resolve.locked_requirements
+            if locked_requirement.pin.project_name in retain
+        ),
+    )
+
+
 class Lock(OutputMixin, JsonMixin, BuildTimeCommand):
     """Operate on PEX lock files."""
 
@@ -1587,6 +1691,7 @@ class Lock(OutputMixin, JsonMixin, BuildTimeCommand):
         )
 
         resolve_subsets = []  # type: List[LockedResolve]
+        errors = []  # type: List[Error]
         for index, locked_resolve in enumerate(lock_file.locked_resolves, start=0):
             if len(lock_file.locked_resolves) == 1:
                 lock_file_description = lockfile_path
@@ -1595,107 +1700,27 @@ class Lock(OutputMixin, JsonMixin, BuildTimeCommand):
                     index=index, lock_file=lockfile_path
                 )
 
-            available = {
-                locked_req.pin.project_name: (
-                    ProjectNameAndVersion(
-                        locked_req.pin.project_name.raw, locked_req.pin.version.raw
-                    ),
-                    locked_req,
-                )
-                for locked_req in locked_resolve.locked_requirements
-            }
-            retain = set()
-            to_resolve = deque(
-                root_requirements
-            )  # type: Deque[Union[RootRequirement, Requirement]]
-            while to_resolve:
-                req = to_resolve.popleft()
-                if req.project_name in retain:
-                    continue
-                retain.add(req.project_name)
-
-                dep = available.get(req.project_name)
-                if not dep:
-                    return Error(
-                        "There is no lock entry for {project} in {lock_file} to satisfy the "
-                        "{transitive}{req} requirement.".format(
-                            project=req.project_name,
-                            lock_file=lock_file_description,
-                            transitive="" if isinstance(req, RootRequirement) else "transitive ",
-                            req=(
-                                req
-                                if isinstance(req, RootRequirement)
-                                else "'{req}'".format(req=req)
-                            ),
-                        )
-                    )
-
-                pnav, locked_req = dep
-                if isinstance(req, RootRequirement):
-                    reqs = req.select(pnav)
-                    if not reqs:
-                        return Error(
-                            "The locked version of {project} in {lock_file} is {version} which "
-                            "does not satisfy the {req} requirement.".format(
-                                project=pnav.project_name,
-                                lock_file=lock_file_description,
-                                version=pnav.version,
-                                req=req,
-                            )
-                        )
-                elif pnav not in req:
-                    production_assert(
-                        isinstance(req, RootRequirement),
-                        "Transitive requirements in a lock should always match existing lock "
-                        "entries. Found {project} {version} in {lock_file}, which does not satisfy "
-                        "transitive requirement '{req}' found in the same lock.",
-                        project=pnav.project_name,
-                        version=pnav.version,
-                        lock_file=lock_file_description,
-                        req=req,
-                    )
-                    return Error(
-                        "The locked version of {project} in {lock_file} is {version} which does "
-                        "not satisfy the '{req}' requirement.".format(
-                            project=pnav.project_name,
-                            lock_file=lock_file_description,
-                            version=pnav.version,
-                            req=req,
-                        )
-                    )
-                elif (
-                    req.project_name in constraint_by_project_name
-                    and pnav not in constraint_by_project_name[req.project_name]
-                ):
-                    return Error(
-                        "The locked version of {project} in {lock_file} is {version} which does "
-                        "not satisfy the '{constraint}' constraint.".format(
-                            project=pnav.project_name,
-                            lock_file=lock_file_description,
-                            version=pnav.version,
-                            constraint=constraint_by_project_name[req.project_name],
-                        )
-                    )
-                else:
-                    reqs = (req,)
-
-                for req in reqs:
-                    to_resolve.extend(
-                        requires_dist.filter_dependencies(
-                            req,
-                            locked_req,
-                            universal_target=lock_file.configuration.universal_target,
-                        )
-                    )
+            result = subset_locked_resolve(
+                locked_resolve,
+                root_requirements,
+                constraint_by_project_name,
+                lock_file_description=lock_file_description,
+                universal_target=lock_file.configuration.universal_target,
+            )
+            if isinstance(result, LockedResolve):
+                resolve_subsets.append(result)
+            else:
+                errors.append(result)
 
-            resolve_subsets.append(
-                attr.evolve(
-                    locked_resolve,
-                    locked_requirements=SortedTuple(
-                        locked_requirement
-                        for locked_requirement in locked_resolve.locked_requirements
-                        if locked_requirement.pin.project_name in retain
-                    ),
+        if not resolve_subsets:
+            if len(errors) == 1:
+                return errors[0]
+            return Error(
+                "Failed to subset any of the {count} locked resolves contained in {lock_file}:\n"
+                "{errors}".format(
+                    count=len(lock_file.locked_resolves),
+                    lock_file=lockfile_path,
+                    errors="\n".join(str(error) for error in errors),
                 )
             )
 

{pex-2.59.0 → pex-2.59.2}/pex/resolve/locked_resolve.py

@@ -281,7 +281,11 @@ class VCSArtifact(Artifact):
     def split_requested_revision(artifact_url):
         # type: (ArtifactURL) -> Tuple[ArtifactURL, Optional[str]]
 
-        vcs_url, _, requested_revision = artifact_url.normalized_url.partition("@")
+        vcs_url = artifact_url.normalized_url
+        if "@" in artifact_url.path:
+            vcs_url, _, requested_revision = vcs_url.rpartition("@")
+        else:
+            requested_revision = None
         return ArtifactURL.parse(vcs_url), requested_revision or None
 
     @classmethod

{pex-2.59.0 → pex-2.59.2}/pex/resolve/locker.py

@@ -379,8 +379,9 @@ class Locker(LogAnalyzer):
                     )
                     self._selected_path_to_pin[selected_path] = build_result.pin
 
-                    vcs, _, vcs_url = artifact_url.normalized_url.partition("+")
-                    vcs_url, _, _ = vcs_url.partition("@")
+                    vcs, _, vcs_url = build_result.url.raw_url.partition("+")
+                    if "@" in build_result.url.path:
+                        vcs_url, _, _ = vcs_url.rpartition("@")
                     commit_id = self._commit_ids.pop(vcs_url, None)
                 elif isinstance(artifact_url.scheme, ArchiveScheme.Value):
                     selected_path = os.path.basename(artifact_url.path)

{pex-2.59.0 → pex-2.59.2}/pex/resolve/target_system.py

@@ -407,7 +407,7 @@ def _as_platform_system_marker(system):
 
 
 def _as_python_version_marker(specifier):
-    # type: (SpecifierSet) -> str
+    # type: (SpecifierSet) -> Optional[str]
 
     clauses = [
         "python_full_version {operator} '{version}'".format(
@@ -416,7 +416,7 @@ def _as_python_version_marker(specifier):
         for spec in specifier
     ]
     if not clauses:
-        return ""
+        return None
 
     if len(clauses) == 1:
         return clauses[0]
@@ -426,19 +426,10 @@ def _as_python_version_marker(specifier):
 
 @attr.s(frozen=True)
 class UniversalTarget(object):
-    @classmethod
-    def from_json(cls, data):
-        # type: (Any) -> UniversalTarget
-        return cls()
-
     implementation = attr.ib(default=None)  # type: Optional[InterpreterImplementation.Value]
     requires_python = attr.ib(default=())  # type: Tuple[SpecifierSet, ...]
     systems = attr.ib(default=())  # type: Tuple[TargetSystem.Value, ...]
 
-    def as_json(self):
-        # type: () -> Any
-        return {}
-
     def iter_interpreter_constraints(self):
         # type: () -> Iterator[InterpreterConstraint]
         for specifier in self.requires_python:
@@ -509,14 +500,16 @@ class UniversalTarget(object):
                 )
             )
         if len(self.requires_python) == 1:
-            clauses.append(_as_python_version_marker(self.requires_python[0]))
+            python_version_marker = _as_python_version_marker(self.requires_python[0])
+            if python_version_marker:
+                clauses.append(python_version_marker)
         elif self.requires_python:
+            python_version_markers = []  # type: List[str]
+            for requires_python in self.requires_python:
+                python_version_marker = _as_python_version_marker(requires_python)
+                if python_version_marker:
+                    python_version_markers.append(python_version_marker)
             clauses.append(
-                "({requires_pythons})".format(
-                    requires_pythons=" or ".join(
-                        _as_python_version_marker(requires_python)
-                        for requires_python in self.requires_python
-                    )
-                )
+                "({requires_pythons})".format(requires_pythons=" or ".join(python_version_markers))
             )
         return Marker(" and ".join(clauses)) if clauses else None

{pex-2.59.0 → pex-2.59.2}/pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.59.0"
+__version__ = "2.59.2"

{pex-2.59.0 → pex-2.59.2}/tests/integration/cli/commands/test_split_universal_locks.py

@@ -15,6 +15,7 @@ import pytest
 from pex import resolver, targets
 from pex.common import safe_mkdir, safe_open
 from pex.interpreter import PythonInterpreter
+from pex.os import LINUX
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pex import PEX
@@ -35,9 +36,31 @@ else:
     from pex.third_party import colors
 
 
+def assert_split_cowsay_lock(lock_file):
+    # type: (str) -> str
+
+    lock = json_codec.load(lock_file)
+    assert len(lock.locked_resolves) == 2
+    assert len(lock.locked_resolves[0].locked_requirements) == 1
+    assert len(lock.locked_resolves[1].locked_requirements) == 1
+
+    versions_by_project_name = defaultdict(list)  # type: DefaultDict[ProjectName, List[Version]]
+    for locked_resolve in lock.locked_resolves:
+        for locked_requirement in locked_resolve.locked_requirements:
+            versions_by_project_name[locked_requirement.pin.project_name].append(
+                locked_requirement.pin.version
+            )
+    assert len(versions_by_project_name) == 1, "Should only have locked the cowsay project."
+    locked_versions = versions_by_project_name.pop(ProjectName("cowsay"))
+    assert [Version("5"), Version("6")] == sorted(locked_versions)
+
+    return lock_file
+
+
 @pytest.fixture
-def split_requirements_lock(tmpdir):
+def split_requirements_by_python_version_lock(tmpdir):
     # type: (Tempdir) -> str
+
     pex_root = tmpdir.join("pex-root")
     lock_file = tmpdir.join("lock.json")
     run_pex3(
@@ -54,23 +77,7 @@ def split_requirements_lock(tmpdir):
         "-o",
         lock_file,
     ).assert_success()
-
-    lock = json_codec.load(lock_file)
-    assert len(lock.locked_resolves) == 2
-    assert len(lock.locked_resolves[0].locked_requirements) == 1
-    assert len(lock.locked_resolves[1].locked_requirements) == 1
-
-    versions_by_project_name = defaultdict(list)  # type: DefaultDict[ProjectName, List[Version]]
-    for locked_resolve in lock.locked_resolves:
-        for locked_requirement in locked_resolve.locked_requirements:
-            versions_by_project_name[locked_requirement.pin.project_name].append(
-                locked_requirement.pin.version
-            )
-    assert len(versions_by_project_name) == 1, "Should only have locked the cowsay project."
-    locked_versions = versions_by_project_name.pop(ProjectName("cowsay"))
-    assert [Version("5"), Version("6")] == sorted(locked_versions)
-
-    return lock_file
+    return assert_split_cowsay_lock(lock_file)
 
 
 PYTHON2 = sys.executable if sys.version_info[0] == 2 else ensure_python_interpreter(PY27)
@@ -81,7 +88,7 @@ PLATFORM_SYSTEM = targets.current().marker_environment.platform_system
 
 def test_resolve_from_split_lock(
     tmpdir,  # type: Tempdir
-    split_requirements_lock,  # type: str
+    split_requirements_by_python_version_lock,  # type: str
 ):
     # type: (...) -> None
 
@@ -97,7 +104,7 @@ def test_resolve_from_split_lock(
                 "--runtime-pex-root",
                 pex_root,
                 "--lock",
-                split_requirements_lock,
+                split_requirements_by_python_version_lock,
                 "-c",
                 "cowsay",
                 "-o",
@@ -115,7 +122,7 @@ def test_resolve_from_split_lock(
 
 def test_export_from_split_lock(
     tmpdir,  # type: Tempdir
-    split_requirements_lock,  # type: str
+    split_requirements_by_python_version_lock,  # type: str
 ):
     # type: (...) -> None
 
@@ -130,7 +137,7 @@ def test_export_from_split_lock(
             "pip",
             "-o",
             requirements,
-            split_requirements_lock,
+            split_requirements_by_python_version_lock,
             python=python,
         ).assert_success()
 
@@ -145,7 +152,7 @@ def test_export_from_split_lock(
             "pep-751",
             "-o",
             pylock,
-            split_requirements_lock,
+            split_requirements_by_python_version_lock,
             python=python,
         ).assert_success()
 
@@ -377,7 +384,7 @@ def assert_expected_split_repos_and_requirements_lock(lock_file):
 def test_split_repos_and_requirements_lock(
     tmpdir,  # type: Tempdir
     find_links,  # type: str
-    split_requirements_lock,  # type: str
+    split_requirements_by_python_version_lock,  # type: str
 ):
     # type: (...) -> None
 
@@ -466,3 +473,48 @@ def test_split_repos_and_requirements_lock(
         expected_ansicolors_version="1.1.8",
         expected_cowsay_version="5",
     )
+
+
+def test_split_requirements_by_platform(tmpdir):
+    # type: (Tempdir) -> None
+
+    pex_root = tmpdir.join("pex-root")
+    lock_file = tmpdir.join("lock.json")
+    run_pex3(
+        "lock",
+        "create",
+        "--pex-root",
+        pex_root,
+        "--style",
+        "universal",
+        "cowsay<6; platform_system == 'Linux'",
+        "cowsay==6; platform_system != 'Linux'",
+        "--indent",
+        "2",
+        "-o",
+        lock_file,
+    ).assert_success()
+    assert_split_cowsay_lock(lock_file)
+
+    pex = tmpdir.join("pex")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--lock",
+            lock_file,
+            "-c",
+            "cowsay",
+            "-o",
+            pex,
+        ]
+    ).assert_success()
+
+    args = [pex]
+    if not LINUX:
+        # N.B.: For cowsay>=6 you must pass `-t <text>` instead of just `<text>` like older cowsay.
+        args.append("-t")
+    args.append("Moo!")
+    assert b"| Moo! |" in subprocess.check_output(args=args)

{pex-2.59.0 → pex-2.59.2}/tests/integration/resolve/test_issue_1918.py

@@ -117,6 +117,8 @@ def test_redacted_requirement_handling(
         lock,
         "--indent",
         "2",
+        "--pip-log",
+        os.path.join(str(tmpdir), "pip.log"),
     ).assert_success()
     lockfile = json_codec.load(lock)
     assert (
@@ -137,6 +139,10 @@ def test_redacted_requirement_handling(
     assert isinstance(artifact, VCSArtifact)
     assert VCS.Git is artifact.vcs
     assert ArtifactURL.parse(expected_url) == artifact.url
+    if pip_parameters.pip_version is not PipVersion.VENDORED:
+        assert "c965f5b9103c5bd32a1572adb8024ebe83278fb0" == artifact.commit_id
+    else:
+        assert artifact.commit_id is None, "Vendored Pip does not log resolution of commit ids."
 
     pex_root = os.path.join(str(tmpdir), "pex_root")
     result = run_pex_command(