pex 2.54.2__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

pex/environment.py

@@ -15,6 +15,7 @@ from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import Distribution, Requirement, is_wheel
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.inherit_path import InheritPath
+from pex.installed_wheel import InstalledWheel
 from pex.interpreter import PythonInterpreter
 from pex.layout import ensure_installed, identify_layout
 from pex.orderedset import OrderedSet
@@ -26,6 +27,7 @@ from pex.third_party.packaging import specifiers
 from pex.third_party.packaging.tags import Tag
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
+from pex.whl import repacked_whl
 
 if TYPE_CHECKING:
     from typing import (
@@ -66,6 +68,22 @@ def _import_pkg_resources():
             return None, False
 
 
+def _fd_lt(
+    self,  # type: FingerprintedDistribution
+    other,  # type: FingerprintedDistribution
+):
+    # type: (...) -> bool
+    if self.project_name.normalized < other.project_name.normalized:
+        return True
+
+    # Since we want to rank higher versions higher (earlier) we need to reverse the natural
+    # ordering of Version in Distribution which is least to greatest.
+    if self.distribution.metadata.version >= other.distribution.metadata.version:
+        return True
+
+    return self.fingerprint < other.fingerprint
+
+
 @attr.s(frozen=True)
 class _RankedDistribution(object):
     # N.B.: A distribution implements rich comparison with the leading component being the
@@ -77,9 +95,7 @@ class _RankedDistribution(object):
     # The attr project type stub file simply misses this.
     _fd_cmp = attr.cmp_using(  # type: ignore[attr-defined]
         eq=FingerprintedDistribution.__eq__,
-        # Since we want to rank higher versions higher (earlier) we need to reverse the natural
-        # ordering of Version in Distribution which is least to greatest.
-        lt=FingerprintedDistribution.__ge__,
+        lt=_fd_lt,
     )
 
     @classmethod
@@ -291,12 +307,7 @@ class PEXEnvironment(object):
 
     def iter_distributions(self, result_type_wheel_file=False):
         # type: (bool) -> Iterator[FingerprintedDistribution]
-        if result_type_wheel_file:
-            if not self._pex_info.deps_are_wheel_files:
-                raise ResolveError(
-                    "Cannot resolve .whl files from PEX at {pex}; its dependencies are in the "
-                    "form of pre-installed wheel chroots.".format(pex=self.source_pex)
-                )
+        if result_type_wheel_file and self._pex_info.deps_are_wheel_files:
             with TRACER.timed(
                 "Searching dependency cache: {cache}".format(
                     cache=os.path.join(self.source_pex, self._pex_info.internal_cache)
@@ -320,10 +331,17 @@ class PEXEnvironment(object):
             ):
                 for distribution_name, fingerprint in self._pex_info.distributions.items():
                     dist_path = os.path.join(internal_cache, distribution_name)
-                    yield FingerprintedDistribution(
-                        distribution=Distribution.load(dist_path),
-                        fingerprint=fingerprint,
-                    )
+                    if result_type_wheel_file:
+                        yield repacked_whl(
+                            installed_wheel=dist_path,
+                            distribution_name=distribution_name,
+                            fingerprint=fingerprint,
+                            use_system_time=True,
+                        )
+                    else:
+                        yield FingerprintedDistribution(
+                            distribution=Distribution.load(dist_path), fingerprint=fingerprint
+                        )
 
     def _update_candidate_distributions(self, distribution_iter):
         # type: (Iterable[FingerprintedDistribution]) -> None
@@ -768,10 +786,10 @@ class PEXEnvironment(object):
                 current_interpreter = PythonInterpreter.get()
                 pex_warnings.warn(
                     "The legacy `pkg_resources` package cannot be imported by the "
-                    "{interpreter} {version} interpreter at {path}.\n"
+                    "{implementation} {version} interpreter at {path}.\n"
                     "The following distributions need `pkg_resources` to load some legacy "
                     "namespace packages and may fail to work properly:\n{dists}".format(
-                        interpreter=current_interpreter.identity.interpreter,
+                        implementation=current_interpreter.identity.implementation,
                         version=current_interpreter.python,
                         path=current_interpreter.binary,
                         dists=dists,
@@ -803,22 +821,25 @@ class PEXEnvironment(object):
             if dist.location in sys.path:
                 continue
             with TRACER.timed("Activating %s" % dist, V=2):
-                if self._pex_info.inherit_path == InheritPath.FALLBACK:
-                    # Prepend location to sys.path.
-                    #
-                    # This ensures that bundled versions of libraries will be used before system-installed
-                    # versions, in case something is installed in both, helping to favor hermeticity in
-                    # the case of non-hermetic PEX files (i.e. those with inherit_path=True).
-                    #
-                    # If the path is not already in sys.path, site.addsitedir will append (not prepend)
-                    # the path to sys.path. But if the path is already in sys.path, site.addsitedir will
-                    # leave sys.path unmodified, but will do everything else it would do. This is not part
-                    # of its advertised contract (which is very vague), but has been verified to be the
-                    # case by inspecting its source for both cpython 2.7 and cpython 3.7.
-                    sys.path.insert(0, dist.location)
-                else:
-                    sys.path.append(dist.location)
-
-                with TRACER.timed("Adding sitedir", V=2):
-                    site.addsitedir(dist.location)
+                for entry in InstalledWheel.load(dist.location).iter_sys_path_entries():
+                    if self._pex_info.inherit_path == InheritPath.FALLBACK:
+                        # Prepend location to sys.path.
+                        #
+                        # This ensures that bundled versions of libraries will be used before
+                        # system-installed versions, in case something is installed in both, helping
+                        # to favor hermeticity in the case of non-hermetic PEX files (i.e. those
+                        # with inherit_path=True).
+                        #
+                        # If the path is not already in sys.path, site.addsitedir will append (not
+                        # prepend) the path to sys.path. But if the path is already in sys.path,
+                        # site.addsitedir will leave sys.path unmodified, but will do everything
+                        # else it would do. This is not part of its advertised contract (which is
+                        # very vague), but has been verified to be the case by inspecting its source
+                        # for both cpython 2.7 and cpython 3.7.
+                        sys.path.insert(0, entry)
+                    else:
+                        sys.path.append(entry)
+
+                    with TRACER.timed("Adding sitedir", V=2):
+                        site.addsitedir(entry)
         return resolved

pex/finders.py

@@ -15,7 +15,7 @@ from pex.dist_metadata import (
     NamedEntryPoint,
 )
 from pex.executables import is_python_script
-from pex.pep_376 import InstalledWheel
+from pex.installed_wheel import InstalledWheel
 from pex.pep_503 import ProjectName
 from pex.sysconfig import SCRIPT_DIR, script_name
 from pex.typing import TYPE_CHECKING, cast

pex/hashing.py

@@ -1,7 +1,7 @@
 # Copyright 2022 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
 import hashlib
 import os
@@ -241,10 +241,12 @@ def dir_hash(
         def iter_files():
             # type: () -> Iterator[Text]
             for root, dirs, files in os.walk(top, followlinks=True):
-                dirs[:] = [d for d in dirs if dir_filter(os.path.join(root, d))]
+                dirs[:] = [
+                    d for d in dirs if dir_filter(os.path.relpath(os.path.join(root, d), top))
+                ]
                 for f in files:
                     path = os.path.join(root, f)
-                    if file_filter(path):
+                    if file_filter(os.path.relpath(path, top)):
                         yield path
 
         file_paths = sorted(iter_files())
@@ -278,16 +280,26 @@ def zip_hash(
             else zf.namelist()
         )
 
-        dirs = frozenset(name.rstrip("/") for name in namelist if name.endswith("/"))
-        accept_dirs = frozenset(d for d in dirs if dir_filter(os.path.basename(d)))
-        reject_dirs = dirs - accept_dirs
-
+        dirs = set()
+        for name in namelist:
+            if name.endswith("/"):
+                dirname = name.rstrip("/")
+            else:
+                dirname = os.path.dirname(name)
+            while dirname:
+                dirs.add(dirname)
+                dirname = os.path.dirname(dirname)
+
+        accept_dirs = frozenset(
+            d for d in dirs if dir_filter(os.path.relpath(d, relpath) if relpath else d)
+        )
+        reject_dirs = tuple("{dir}/".format(dir=path) for path in (dirs - accept_dirs))
         accept_files = sorted(
             name
             for name in namelist
             if not name.endswith("/")
-            and not any(name.startswith(reject_dir) for reject_dir in reject_dirs)
-            and file_filter(os.path.basename(name))
+            and not name.startswith(reject_dirs)
+            and file_filter(os.path.relpath(name, relpath) if relpath else name)
         )
 
         hashed_names = (
@@ -297,3 +309,53 @@ def zip_hash(
 
         for filename in accept_files:
             update_hash(zf.open(filename, "r"), digest)
+
+
+if __name__ == "__main__":
+    import sys
+    import zipfile
+    from argparse import ArgumentParser
+
+    from pex.common import is_pyc_dir, is_pyc_file
+
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--exclude-dir",
+        dest="exclude_dirs",
+        action="append",
+        default=[],
+    )
+    parser.add_argument("--zip-relpath")
+    parser.add_argument("paths", nargs="+")
+
+    options = parser.parse_args()
+    exclude_dirs = frozenset(options.exclude_dirs)
+
+    for path in options.paths:
+        digest = Sha256()
+        if zipfile.is_zipfile(path):
+            zip_hash(
+                path,
+                digest=digest,
+                relpath=options.zip_relpath,
+                dir_filter=(
+                    lambda dir_path: (
+                        not is_pyc_dir(dir_path) and os.path.basename(dir_path) not in exclude_dirs
+                    )
+                ),
+                file_filter=lambda f: not is_pyc_file(f),
+            )
+        elif os.path.isdir(path):
+            dir_hash(
+                path,
+                digest=digest,
+                dir_filter=(
+                    lambda dir_path: (
+                        not is_pyc_dir(dir_path) and os.path.basename(dir_path) not in exclude_dirs
+                    )
+                ),
+                file_filter=lambda f: not is_pyc_file(f),
+            )
+        else:
+            print("Can only hash zip files or directories. Skipping file", path, file=sys.stderr)
+        print(path, digest.hexdigest(), file=sys.stdout)

pex/installed_wheel.py

@@ -0,0 +1,141 @@
+# Copyright 2025 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import hashlib
+import json
+import os
+
+from pex.typing import TYPE_CHECKING, cast
+from pex.util import CacheHelper
+from pex.wheel import WHEEL, Wheel, WheelMetadataLoadError
+
+if TYPE_CHECKING:
+    from typing import Iterator, Optional, Text, Tuple
+
+    import attr  # vendor:skip
+else:
+    import pex.third_party.attr as attr
+
+
+@attr.s(frozen=True)
+class InstalledWheel(object):
+    class LoadError(Exception):
+        """Indicates an installed wheel was not loadable at a particular path."""
+
+    _LAYOUT_JSON_FILENAME = ".layout.json"
+
+    @classmethod
+    def layout_file(cls, prefix_dir):
+        # type: (str) -> str
+        return os.path.join(prefix_dir, cls._LAYOUT_JSON_FILENAME)
+
+    @classmethod
+    def save(
+        cls,
+        prefix_dir,  # type: str
+        stash_dir,  # type: str
+        record_relpath,  # type: Text
+        root_is_purelib,  # type: bool
+        sys_path_entries,  # type: Tuple[str, ...]
+    ):
+        # type: (...) -> InstalledWheel
+
+        # We currently need the installed wheel chroot hash for PEX-INFO / boot purposes. It is
+        # expensive to calculate; so we do it here 1 time when saving the installed wheel.
+        fingerprint = CacheHelper.dir_hash(prefix_dir, hasher=hashlib.sha256)
+
+        layout = {
+            "stash_dir": stash_dir,
+            "record_relpath": record_relpath,
+            "fingerprint": fingerprint,
+            "root_is_purelib": root_is_purelib,
+            "sys_path_entries": sys_path_entries,
+        }
+        with open(cls.layout_file(prefix_dir), "w") as fp:
+            json.dump(layout, fp, sort_keys=True, separators=(",", ":"))
+        return cls(
+            prefix_dir=prefix_dir,
+            stash_dir=stash_dir,
+            record_relpath=record_relpath,
+            fingerprint=fingerprint,
+            root_is_purelib=root_is_purelib,
+            sys_path_entries=sys_path_entries,
+        )
+
+    @classmethod
+    def load(cls, prefix_dir):
+        # type: (str) -> InstalledWheel
+        layout_file = cls.layout_file(prefix_dir)
+        try:
+            with open(layout_file) as fp:
+                layout = json.load(fp)
+        except (IOError, OSError) as e:
+            raise cls.LoadError(
+                "Failed to load an installed wheel layout from {layout_file}: {err}".format(
+                    layout_file=layout_file, err=e
+                )
+            )
+        if not isinstance(layout, dict):
+            raise cls.LoadError(
+                "The installed wheel layout file at {layout_file} must contain a single top-level "
+                "object, found: {value}.".format(layout_file=layout_file, value=layout)
+            )
+        stash_dir = layout.get("stash_dir")
+        record_relpath = layout.get("record_relpath")
+        if not stash_dir or not record_relpath:
+            raise cls.LoadError(
+                "The installed wheel layout file at {layout_file} must contain an object with both "
+                "`stash_dir` and `record_relpath` attributes, found: {value}".format(
+                    layout_file=layout_file, value=layout
+                )
+            )
+
+        fingerprint = layout.get("fingerprint")
+
+        # N.B.: Caching root_is_purelib was not part of the original InstalledWheel layout data; so
+        # we materialize the property if needed to support older installed wheel chroots.
+        root_is_purelib = layout.get("root_is_purelib")
+        if root_is_purelib is None:
+            try:
+                wheel = WHEEL.load(prefix_dir)
+            except WheelMetadataLoadError as e:
+                raise cls.LoadError(
+                    "Failed to determine if installed wheel at {location} is platform-specific: "
+                    "{err}".format(location=prefix_dir, err=e)
+                )
+            root_is_purelib = wheel.root_is_purelib
+
+        # N.B.: Older versions of Pex installed wheel chroots did not have this field since the
+        # `sys.path` entry was always just the prefix_dir for those.
+        sys_path_entries = layout.get("sys_path_entries", [""])
+
+        return cls(
+            prefix_dir=prefix_dir,
+            stash_dir=cast(str, stash_dir),
+            record_relpath=cast(str, record_relpath),
+            fingerprint=cast("Optional[str]", fingerprint),
+            root_is_purelib=root_is_purelib,
+            sys_path_entries=tuple(sys_path_entries),
+        )
+
+    prefix_dir = attr.ib()  # type: str
+    stash_dir = attr.ib()  # type: str
+    record_relpath = attr.ib()  # type: Text
+    fingerprint = attr.ib()  # type: Optional[str]
+    root_is_purelib = attr.ib()  # type: bool
+    sys_path_entries = attr.ib()  # type: Tuple[str, ...]
+
+    def wheel_file_name(self):
+        # type: () -> str
+        return Wheel.load(self.prefix_dir).wheel_file_name
+
+    def stashed_path(self, *components):
+        # type: (*str) -> str
+        return os.path.join(self.prefix_dir, self.stash_dir, *components)
+
+    def iter_sys_path_entries(self):
+        # type: () -> Iterator[str]
+        for sys_path_entry in self.sys_path_entries:
+            yield os.path.normpath(os.path.join(self.prefix_dir, sys_path_entry))

pex/interpreter.py

@@ -18,7 +18,9 @@ from textwrap import dedent
 from pex import third_party
 from pex.cache.dirs import InterpreterDir
 from pex.common import safe_mkdtemp, safe_rmtree
+from pex.exceptions import production_assert
 from pex.executor import Executor
+from pex.interpreter_implementation import InterpreterImplementation
 from pex.jobs import Job, Retain, SpawnedJob, execute_parallel
 from pex.orderedset import OrderedSet
 from pex.os import WINDOWS, is_exe
@@ -61,18 +63,6 @@ if TYPE_CHECKING:
     InterpreterOrError = Union["PythonInterpreter", InterpreterIdentificationError]
 
 
-def calculate_binary_name(
-    platform_python_implementation, python_version=None  # type: Optional[Tuple[int, ...]]
-):
-    # type: (...) -> str
-    name = "python"
-    if platform_python_implementation == "PyPy":
-        name = "pypy"
-    if not python_version:
-        return name
-    return "{name}{version}".format(name=name, version=".".join(map(str, python_version)))
-
-
 class SitePackagesDir(object):
     def __init__(self, path):
         # type: (str) -> None
@@ -404,11 +394,11 @@ class PythonIdentity(object):
         )
 
     @classmethod
-    def _find_interpreter_name(cls, python_tag):
-        # type: (str) -> str
-        for abbr, interpreter in cls.ABBR_TO_INTERPRETER_NAME.items():
-            if python_tag.startswith(abbr):
-                return interpreter
+    def _find_implementation(cls, python_tag):
+        # type: (str) -> InterpreterImplementation.Value
+        for implementation in InterpreterImplementation.values():
+            if python_tag.startswith(implementation.abbr):
+                return implementation
         raise ValueError("Unknown interpreter: {}".format(python_tag))
 
     def __init__(
@@ -431,9 +421,12 @@ class PythonIdentity(object):
         configured_macosx_deployment_target,  # type: Optional[str]
     ):
         # type: (...) -> None
-        # N.B.: We keep this mapping to support historical values for `distribution` and
-        # `requirement` properties.
-        self._interpreter_name = self._find_interpreter_name(python_tag)
+
+        self._implementation = self._find_implementation(python_tag)
+        production_assert(
+            not pypy_version or self._implementation is InterpreterImplementation.PYPY
+        )
+        self._pypy_version = pypy_version
 
         self._binary = binary
         self._prefix = prefix
@@ -447,7 +440,6 @@ class PythonIdentity(object):
         self._abi_tag = abi_tag
         self._platform_tag = platform_tag
         self._version = version
-        self._pypy_version = pypy_version
         self._supported_tags = CompatibilityTags(tags=supported_tags)
         self._env_markers = env_markers
         self._configured_macosx_deployment_target = configured_macosx_deployment_target
@@ -491,7 +483,7 @@ class PythonIdentity(object):
             env_markers=self._env_markers.as_dict(),
             configured_macosx_deployment_target=self._configured_macosx_deployment_target,
         )
-        return json.dumps(values, sort_keys=True)
+        return json.dumps(values, sort_keys=True, separators=(",", ":"))
 
     @property
     def binary(self):
@@ -570,7 +562,7 @@ class PythonIdentity(object):
     @property
     def is_pypy(self):
         # type: () -> bool
-        return bool(self._pypy_version)
+        return self._implementation is InterpreterImplementation.PYPY
 
     @property
     def version_str(self):
@@ -593,9 +585,9 @@ class PythonIdentity(object):
         return self._configured_macosx_deployment_target
 
     @property
-    def interpreter(self):
-        # type: () -> str
-        return self._interpreter_name
+    def implementation(self):
+        # type: () -> InterpreterImplementation.Value
+        return self._implementation
 
     def iter_supported_platforms(self):
         # type: () -> Iterator[Platform]
@@ -614,9 +606,8 @@ class PythonIdentity(object):
 
     def binary_name(self, version_components=2):
         # type: (int) -> str
-        return calculate_binary_name(
-            platform_python_implementation=self._interpreter_name,
-            python_version=self._version[:version_components] if version_components > 0 else None,
+        return self._implementation.calculate_binary_name(
+            version=self._version[:version_components] if version_components > 0 else None
         )
 
     def hashbang(self):
@@ -636,8 +627,8 @@ class PythonIdentity(object):
         # type: () -> str
         # N.B.: Kept as distinct from __repr__ to support legacy str(identity) used by Pants v1 when
         # forming cache locations.
-        return "{interpreter_name}-{major}.{minor}.{patch}".format(
-            interpreter_name=self._interpreter_name,
+        return "{implementation}-{major}.{minor}.{patch}".format(
+            implementation=self._implementation,
             major=self._version[0],
             minor=self._version[1],
             patch=self._version[2],
@@ -1438,13 +1429,13 @@ class PythonInterpreter(object):
         # python<major>.<minor> is present in any given <prefix>/bin/ directory; so the algorithm
         # gets a hit on 1st try for CPython binaries incurring ~no extra overhead.
 
+        implementation = self._identity.implementation
         version = self._identity.version
         abi_tag = self._identity.abi_tag
 
-        prefix = "pypy" if self.is_pypy else "python"
-        suffixes = ("{}.{}".format(version[0], version[1]), str(version[0]), "")
+        versions = version[:2], version[:1], None
         candidate_binaries = tuple(
-            script_name("{}{}".format(prefix, suffix)) for suffix in suffixes
+            script_name(implementation.calculate_binary_name(version)) for version in versions
         )
 
         def iter_base_candidate_binary_paths(interpreter):
@@ -1534,13 +1525,19 @@ class PythonInterpreter(object):
             self._supported_platforms = frozenset(self._identity.iter_supported_platforms())
         return self._supported_platforms
 
-    def shebang(self, args=None):
-        # type: (Optional[Text]) -> Text
+    def shebang(
+        self,
+        args=None,  # type: Optional[Text]
+        encoding_line="",  # type: str
+    ):
+        # type: (...) -> Text
         """Return the contents of an appropriate shebang for this interpreter and args.
 
         The shebang will include the leading `#!` but will not include a trailing new line character.
         """
-        return create_shebang(self._binary, python_args=args)
+        return create_shebang(
+            adjust_to_final_path(self._binary), python_args=args, encoding_line=encoding_line
+        )
 
     def create_isolated_cmd(
         self,
@@ -1646,6 +1643,7 @@ def create_shebang(
     python_exe,  # type: Text
     python_args=None,  # type: Optional[Text]
     max_shebang_length=MAX_SHEBANG_LENGTH,  # type: int
+    encoding_line="",  # type: str
 ):
     # type: (...) -> Text
     """Return the contents of an appropriate shebang for the given Python interpreter and args.
@@ -1671,6 +1669,7 @@ def create_shebang(
         dedent(
             """\
             #!/bin/sh
+            {encoding_line}
             # N.B.: This python script executes via a /bin/sh re-exec as a hack to work around a
             # potential maximum shebang length of {max_shebang_length} bytes on this system which
             # the python interpreter `exec`ed below would violate.
@@ -1678,6 +1677,10 @@ def create_shebang(
             '''
             """
         )
-        .format(max_shebang_length=max_shebang_length, python=python)
+        .format(
+            encoding_line=encoding_line.rstrip(),
+            max_shebang_length=max_shebang_length,
+            python=python,
+        )
         .strip()
     )