pex 2.64.1__tar.gz → 2.65.0__tar.gz

{pex-2.64.1 → pex-2.65.0}/CHANGES.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+## 2.65.0
+
+This release adds support for PEX scies using CPython free-threaded builds. Most such scies should
+be able to auto-detect when a free-threaded CPython is needed, but new `--scie-pbs-free-threaded`
+and `--scie-pbs-debug` options have been added to explicitly request the desired PBS CPython build
+as well.
+
+* Support free-threaded PEX scies. (#2967)
+
 ## 2.64.1
 
 This release is a follow-up to 2.64.0 to fix a regression in locks for credentialed VCS

{pex-2.64.1 → pex-2.65.0}/PKG-INFO

@@ -1,15 +1,15 @@
 Metadata-Version: 2.4
 Name: pex
-Version: 2.64.1
+Version: 2.65.0
 Summary: The PEX packaging toolchain.
 Home-page: https://github.com/pex-tool/pex
-Download-URL: https://github.com/pex-tool/pex/releases/download/v2.64.1/pex
+Download-URL: https://github.com/pex-tool/pex/releases/download/v2.65.0/pex
 Author: The PEX developers
 Author-email: developers@pex-tool.org
 License-Expression: Apache-2.0
-Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.64.1/CHANGES.md
+Project-URL: Changelog, https://github.com/pex-tool/pex/blob/v2.65.0/CHANGES.md
 Project-URL: Documentation, https://docs.pex-tool.org/
-Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.64.1
+Project-URL: Source, https://github.com/pex-tool/pex/tree/v2.65.0
 Keywords: package,executable,virtualenv,lock,freeze
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers

{pex-2.64.1 → pex-2.65.0}/docker/base/Dockerfile

@@ -5,12 +5,14 @@ FROM ubuntu:24.04
 # See: https://github.com/pyenv/pyenv/wiki#suggested-build-environment
 # Additionally, some sdists need cargo to build native extensions and we need add-apt-repository
 # from software-properties-common to install the deadsnakes ppa and ssh for integration tests to
-# use.
+# use. We also install clang since its the compiler toolchain configured for PBS CPythons used in
+# some tests.
 RUN apt update && \
   DEBIAN_FRONTEND=noninteractive apt upgrade --yes && \
   DEBIAN_FRONTEND=noninteractive apt install --yes --no-install-recommends \
     build-essential \
     cargo \
+    clang \
     curl \
     git \
     libbz2-dev \
@@ -22,6 +24,8 @@ RUN apt update && \
     libssl-dev \
     libxml2-dev \
     libxmlsec1-dev \
+    python3.12-dev \
+    python3.12-venv \
     software-properties-common \
     ssh \
     tk-dev \

{pex-2.64.1 → pex-2.65.0}/docker/base/install_pythons.sh

@@ -35,7 +35,6 @@ PYENV_VERSIONS=(
   2.7.18
   3.5.10
   3.6.15
-  3.12.12
   3.13.9
   pypy2.7-7.3.20
   pypy3.5-7.0.0

{pex-2.64.1 → pex-2.65.0}/pex/scie/__init__.py

@@ -222,6 +222,32 @@ def register_options(parser):
             "to the patch level."
         ),
     )
+    parser.add_argument(
+        "--scie-pbs-free-threaded",
+        "--no-scie-pbs-free-threaded",
+        dest="scie_pbs_free_threaded",
+        default=False,
+        type=bool,
+        action=HandleBoolAction,
+        help=(
+            "Should the Python Standalone Builds CPython distributions be free-threaded. If left "
+            "unspecified or otherwise turned off, creating a scie from a PEX with free-threaded "
+            "abi wheels will automatically turn this option on. Note that this option is not "
+            "compatible with `--scie-pbs-stripped`."
+        ),
+    )
+    parser.add_argument(
+        "--scie-pbs-debug",
+        "--no-scie-pbs-debug",
+        dest="scie_pbs_debug",
+        default=False,
+        type=bool,
+        action=HandleBoolAction,
+        help=(
+            "Should the Python Standalone Builds CPython distributions be debug builds. Note that "
+            "this option is not compatible with `--scie-pbs-stripped`."
+        ),
+    )
     parser.add_argument(
         "--scie-pbs-stripped",
         "--no-scie-pbs-stripped",
@@ -232,7 +258,8 @@ def register_options(parser):
         help=(
             "Should the Python Standalone Builds CPython distributions used be stripped of debug "
             "symbols or not. For Linux and Windows particularly, the stripped distributions are "
-            "less than half the size of the distributions that ship with debug symbols."
+            "less than half the size of the distributions that ship with debug symbols. Note that"
+            "this option is not compatible with `--scie-pbs-free-threaded` or `--scie-pbs-debug`."
         ),
     )
     parser.add_argument(
@@ -318,6 +345,10 @@ def render_options(options):
     if options.python_version:
         args.append("--scie-python-version")
         args.append(".".join(map(str, options.python_version)))
+    if options.pbs_free_threaded:
+        args.append("--scie-pbs-free-threaded")
+    if options.pbs_debug:
+        args.append("--scie-pbs-debug")
     if options.pbs_stripped:
         args.append("--scie-pbs-stripped")
     for hash_algorithm in options.hash_algorithms:
@@ -398,6 +429,12 @@ def extract_options(options):
                 )
             )
 
+    if options.scie_pbs_stripped and (options.scie_pbs_free_threaded or options.scie_pbs_debug):
+        raise ValueError(
+            "Python Standalone Builds does not release stripped distributions for debug or "
+            "free-threaded builds."
+        )
+
     science_binary = None  # type: Optional[Union[File, Url]]
     if options.scie_science_binary:
         url_info = urlparse.urlparse(options.scie_science_binary)
@@ -420,6 +457,8 @@ def extract_options(options):
         pbs_release=options.scie_pbs_release,
         pypy_release=options.scie_pypy_release,
         python_version=python_version,
+        pbs_free_threaded=options.scie_pbs_free_threaded,
+        pbs_debug=options.scie_pbs_debug,
         pbs_stripped=options.scie_pbs_stripped,
         hash_algorithms=tuple(options.scie_hash_algorithms),
         science_binary=science_binary,

{pex-2.64.1 → pex-2.65.0}/pex/scie/model.py

@@ -303,6 +303,8 @@ class ScieOptions(object):
     python_version = attr.ib(
         default=None
     )  # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]]
+    pbs_free_threaded = attr.ib(default=False)  # type: bool
+    pbs_debug = attr.ib(default=False)  # type: bool
     pbs_stripped = attr.ib(default=False)  # type: bool
     hash_algorithms = attr.ib(default=())  # type: Tuple[str, ...]
     science_binary = attr.ib(default=None)  # type: Optional[Union[File, Url]]

{pex-2.64.1 → pex-2.65.0}/pex/scie/science.py

@@ -24,6 +24,7 @@ from pex.hashing import Sha256
 from pex.os import is_exe
 from pex.pep_440 import Version
 from pex.pex import PEX
+from pex.pex_info import PexInfo
 from pex.result import Error, try_
 from pex.scie.model import (
     File,
@@ -37,6 +38,7 @@ from pex.scie.model import (
 )
 from pex.sysconfig import SysPlatform
 from pex.third_party.packaging.specifiers import SpecifierSet
+from pex.third_party.packaging.utils import parse_wheel_filename
 from pex.third_party.packaging.version import InvalidVersion
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
@@ -66,7 +68,7 @@ class Manifest(object):
 
 
 SCIENCE_RELEASES_URL = "https://github.com/a-scie/lift/releases"
-MIN_SCIENCE_VERSION = Version("0.14.0")
+MIN_SCIENCE_VERSION = Version("0.15.0")
 SCIENCE_REQUIREMENT = SpecifierSet("~={min_version}".format(min_version=MIN_SCIENCE_VERSION))
 
 
@@ -104,6 +106,16 @@ class Filenames(Enum["Filenames.Value"]):
 Filenames.seal()
 
 
+def _is_free_threaded_pex(pex_info):
+    # type: (PexInfo) -> bool
+    for distribution in pex_info.distributions:
+        _, _, _, tags = parse_wheel_filename(os.path.basename(distribution))
+        for tag in tags:
+            if tag.abi.startswith(("cp", "abi3")) and "t" in tag.abi:
+                return True
+    return False
+
+
 def create_manifests(
     configuration,  # type: ScieConfiguration
     name,  # type: str
@@ -270,6 +282,7 @@ def create_manifests(
     }
 
     configure_binding_args = [Filenames.PEX.placeholder, Filenames.CONFIGURE_BINDING.placeholder]
+    pbs_free_threaded = _is_free_threaded_pex(pex_info) or configuration.options.pbs_free_threaded
     for interpreter in configuration.interpreters:
         lift = lift_template.copy()
 
@@ -285,10 +298,17 @@ def create_manifests(
             interpreter.platform.qualified_file_name("{name}-lift.toml".format(name=name)),
         )
 
+        version_str = interpreter.version_str
+        if Provider.PythonBuildStandalone is interpreter.provider:
+            if configuration.options.pbs_debug:
+                version_str += "d"
+            if pbs_free_threaded:
+                version_str += "t"
+
         interpreter_config = {
             "id": "python-distribution",
             "provider": interpreter.provider.value,
-            "version": interpreter.version_str,
+            "version": version_str,
             "lazy": configuration.options.style is ScieStyle.LAZY,
         }
         if interpreter.release:
@@ -297,7 +317,9 @@ def create_manifests(
             interpreter_config["base_url"] = "/".join(
                 (configuration.options.assets_base_url, "providers", str(interpreter.provider))
             )
-        if Provider.PythonBuildStandalone is interpreter.provider:
+        if Provider.PythonBuildStandalone is interpreter.provider and not (
+            configuration.options.pbs_debug or pbs_free_threaded
+        ):
             interpreter_config.update(
                 flavor=(
                     "install_only_stripped"

{pex-2.64.1 → pex-2.65.0}/pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.64.1"
+__version__ = "2.65.0"

{pex-2.64.1 → pex-2.65.0}/tests/integration/scie/test_pex_scie.py

@@ -10,7 +10,7 @@ import re
 import shutil
 import sys
 from textwrap import dedent
-from typing import Optional
+from typing import Dict, Optional
 
 import pytest
 
@@ -20,7 +20,7 @@ from pex.executables import chmod_plus_x, is_script
 from pex.fetcher import URLFetcher
 from pex.layout import Layout
 from pex.orderedset import OrderedSet
-from pex.os import is_exe
+from pex.os import WINDOWS, is_exe
 from pex.pip.version import PipVersion
 from pex.scie import ScieStyle
 from pex.sysconfig import SysPlatform
@@ -331,7 +331,7 @@ def test_specified_science_binary(tmpdir):
 
     local_science_binary = os.path.join(str(tmpdir), "science")
     with open(local_science_binary, "wb") as write_fp, URLFetcher().get_body_stream(
-        "https://github.com/a-scie/lift/releases/download/v0.14.0/{binary}".format(
+        "https://github.com/a-scie/lift/releases/download/v0.15.0/{binary}".format(
             binary=SysPlatform.CURRENT.qualified_binary_name("science")
         )
     ) as read_fp:
@@ -375,7 +375,7 @@ def test_specified_science_binary(tmpdir):
         cached_science_binaries
     ), "Expected the local science binary to be used but not cached."
     assert (
-        "0.14.0"
+        "0.15.0"
         == subprocess.check_output(args=[local_science_binary, "--version"]).decode("utf-8").strip()
     )
 
@@ -1259,3 +1259,92 @@ def test_scie_eager_no_ptex(tmpdir):
 
     assert_ptex(expect_included=False, lazy=False)
     assert_ptex(expect_included=True, lazy=True)
+
+
+@skip_if_no_provider
+@pytest.mark.skipif(
+    IS_PYPY or sys.version_info[:2] < (3, 13),
+    reason="Free-threaded CPython is only available for Python >=3.13.",
+)
+def test_free_threaded_scie_requested(tmpdir):
+    # type: (Tempdir) -> None
+
+    def build_and_run_scie(*extra_args):
+        # type: (*str) -> Dict[str, Any]
+
+        scie = tmpdir.join("scie")
+        run_pex_command(
+            args=["--scie", "eager", "--scie-only", "-o", scie] + list(extra_args)
+        ).assert_success()
+        data = json.loads(
+            subprocess.check_output(
+                args=[
+                    scie,
+                    "-c",
+                    dedent(
+                        """\
+                        import json
+                        import sys
+                        import sysconfig
+
+
+                        json.dump(
+                            {
+                                "debug": sysconfig.get_config_var("Py_DEBUG"),
+                                "free-threaded": sysconfig.get_config_var("Py_GIL_DISABLED"),
+                            },
+                            sys.stdout,
+                        )
+                        """
+                    ),
+                ]
+            )
+        )
+        assert isinstance(data, dict)
+        return data
+
+    assert {"debug": 0, "free-threaded": 0} == build_and_run_scie()
+    assert {"debug": 0, "free-threaded": 1} == build_and_run_scie("--scie-pbs-free-threaded")
+    if not WINDOWS:
+        assert {"debug": 1, "free-threaded": 0} == build_and_run_scie("--scie-pbs-debug")
+        assert {"debug": 1, "free-threaded": 1} == build_and_run_scie(
+            "--scie-pbs-debug", "--scie-pbs-free-threaded"
+        )
+
+
+@skip_if_no_provider
+@pytest.mark.skipif(
+    IS_PYPY or sys.version_info[:2] < (3, 13),
+    reason="Free-threaded CPython is only available for Python >=3.13.",
+)
+def test_free_threaded_scie_auto_detected(tmpdir):
+    # type: (Tempdir) -> None
+
+    # N.B.: We need to run Pex with a free-threaded Python to get a free-threaded psutil wheel.
+    free_threaded_python = tmpdir.join("pythont")
+    run_pex_command(
+        args=[
+            "--scie",
+            "eager",
+            "--scie-only",
+            "--scie-pbs-free-threaded",
+            "-o",
+            free_threaded_python,
+        ]
+    ).assert_success()
+
+    scie = tmpdir.join("scie")
+    run_pex_command(
+        args=["--scie", "eager", "psutil", "--scie-only", "-o", scie], python=free_threaded_python
+    ).assert_success()
+
+    assert (
+        b"1"
+        == subprocess.check_output(
+            args=[
+                scie,
+                "-c",
+                "import psutil, sysconfig; print(sysconfig.get_config_var('Py_GIL_DISABLED'))",
+            ]
+        ).strip()
+    )