relenv 0.21.2__py3-none-any.whl → 0.22.1__py3-none-any.whl

relenv/build/windows.py

@@ -1,8 +1,11 @@
 # Copyright 2022-2025 Broadcom.
 # SPDX-License-Identifier: Apache-2
+# mypy: ignore-errors
 """
 The windows build process.
 """
+from __future__ import annotations
+
 import glob
 import json
 import logging
@@ -11,24 +14,34 @@ import pathlib
 import shutil
 import sys
 import tarfile
+import time
+from typing import IO, MutableMapping, Union
 
 from .common import (
+    Dirs,
     builds,
     create_archive,
-    download_url,
-    extract_archive,
+    get_dependency_version,
     install_runtime,
-    MODULE_DIR,
     patch_file,
-    runcmd,
     update_ensurepip,
+    update_sbom_checksums,
+)
+from ..common import (
+    WIN32,
+    arches,
+    MODULE_DIR,
+    download_url,
+    extract_archive,
+    runcmd,
 )
-from ..common import arches, WIN32, Version
 
 log = logging.getLogger(__name__)
 
 ARCHES = arches[WIN32]
 
+EnvMapping = MutableMapping[str, str]
+
 if sys.platform == WIN32:
     import ctypes
 
@@ -36,7 +49,7 @@ if sys.platform == WIN32:
     kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)
 
 
-def populate_env(env, dirs):
+def populate_env(env: EnvMapping, dirs: Dirs) -> None:
     """
     Make sure we have the correct environment variables set.
 
@@ -48,7 +61,7 @@ def populate_env(env, dirs):
     env["MSBUILDDISABLENODEREUSE"] = "1"
 
 
-def update_props(source, old, new):
+def update_props(source: pathlib.Path, old: str, new: str) -> None:
     """
     Overwrite a dependency string for Windows PCBuild.
 
@@ -63,7 +76,7 @@ def update_props(source, old, new):
     patch_file(source / "PCbuild" / "get_externals.bat", old, new)
 
 
-def get_externals_source(externals_dir, url):
+def get_externals_source(externals_dir: pathlib.Path, url: str) -> None:
     """
     Download external source code dependency.
 
@@ -80,7 +93,7 @@ def get_externals_source(externals_dir, url):
         log.exception("Failed to remove temporary file")
 
 
-def get_externals_bin(source_root, url):
+def get_externals_bin(source_root: pathlib.Path, url: str) -> None:
     """
     Download external binary dependency.
 
@@ -90,13 +103,25 @@ def get_externals_bin(source_root, url):
     pass
 
 
-def update_sqlite(dirs, env):
+def update_sqlite(dirs: Dirs, env: EnvMapping) -> None:
     """
     Update the SQLITE library.
     """
-    version = "3.50.4.0"
-    url = "https://sqlite.org/2025/sqlite-autoconf-3500400.tar.gz"
-    sha256 = "a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18"
+    # Try to get version from JSON
+    sqlite_info = get_dependency_version("sqlite", "win32")
+    if sqlite_info:
+        version = sqlite_info["version"]
+        url_template = sqlite_info["url"]
+        sha256 = sqlite_info["sha256"]
+        sqliteversion = sqlite_info.get("sqliteversion", "3500400")
+        # Format the URL with sqliteversion (the 7-digit version number)
+        url = url_template.format(version=sqliteversion)
+    else:
+        # Fallback to hardcoded values
+        version = "3.50.4.0"
+        url = "https://sqlite.org/2025/sqlite-autoconf-3500400.tar.gz"
+        sha256 = "a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18"
+        sqliteversion = "3500400"
     ref_loc = f"cpe:2.3:a:sqlite:sqlite:{version}:*:*:*:*:*:*:*"
     target_dir = dirs.source / "externals" / f"sqlite-{version}"
     target_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -104,7 +129,7 @@ def update_sqlite(dirs, env):
         update_props(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", f"sqlite-{version}")
         get_externals_source(externals_dir=dirs.source / "externals", url=url)
         # # we need to fix the name of the extracted directory
-        extracted_dir = dirs.source / "externals" / "sqlite-autoconf-3500400"
+        extracted_dir = dirs.source / "externals" / f"sqlite-autoconf-{sqliteversion}"
         shutil.move(str(extracted_dir), str(target_dir))
     # Update externals.spdx.json with the correct version, url, and hash
     # This became a thing in 3.12
@@ -122,13 +147,43 @@ def update_sqlite(dirs, env):
             json.dump(data, f, indent=2)
 
 
-def update_xz(dirs, env):
+def update_xz(dirs: Dirs, env: EnvMapping) -> None:
     """
     Update the XZ library.
+
+    COMPATIBILITY NOTE: We use config.h from XZ 5.4.7 for all XZ versions.
+    Starting with XZ 5.5.0, the project removed Visual Studio .vcxproj files
+    and switched to CMake. Python's build system (PCbuild/liblzma.vcxproj)
+    still expects MSBuild-compatible builds, so we maintain a compatibility
+    shim at relenv/_resources/xz/config.h.
+
+    When updating XZ versions, verify compatibility by checking:
+    1. Build completes without compiler errors
+    2. test_xz_lzma_functionality passes
+    3. No new HAVE_* defines required in src/liblzma source files
+    4. No removed HAVE_* defines that config.h references
+
+    If compatibility breaks, you have two options:
+    - Use CMake to generate new config.h for Windows (see discussion at
+      https://discuss.python.org/t/building-python-from-source-on-windows-using-a-custom-version-of-xz/74717)
+    - Update relenv/_resources/xz/config.h manually from newer XZ source
+
+    See also: relenv/_resources/xz/readme.md
     """
-    version = "5.6.2"
-    url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz"
-    sha256 = "8bfd20c0e1d86f0402f2497cfa71c6ab62d4cd35fd704276e3140bfb71414519"
+    # Try to get version from JSON
+    # Note: Windows may use a different XZ version than Linux/Darwin due to MSBuild compatibility
+    xz_info = get_dependency_version("xz", "win32")
+    if xz_info:
+        version = xz_info["version"]
+        url_template = xz_info["url"]
+        sha256 = xz_info["sha256"]
+        url = url_template.format(version=version)
+    else:
+        # Fallback to hardcoded values
+        # Note: Using 5.6.2 for MSBuild compatibility (5.5.0+ removed MSBuild support)
+        version = "5.6.2"
+        url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz"
+        sha256 = "8bfd20c0e1d86f0402f2497cfa71c6ab62d4cd35fd704276e3140bfb71414519"
     ref_loc = f"cpe:2.3:a:tukaani:xz:{version}:*:*:*:*:*:*:*"
     target_dir = dirs.source / "externals" / f"xz-{version}"
     target_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -158,14 +213,23 @@ def update_xz(dirs, env):
             json.dump(data, f, indent=2)
 
 
-def update_expat(dirs, env):
+def update_expat(dirs: Dirs, env: EnvMapping) -> None:
     """
     Update the EXPAT library.
     """
     # Patch <src>/Modules/expat/refresh.sh. When the SBOM is created, refresh.sh
     # is scanned for the expat version, even though it doesn't run on Windows.
-    version = "2.7.3"
-    hash = "821ac9710d2c073eaf13e1b1895a9c9aa66c1157a99635c639fbff65cdbdd732"
+
+    # Try to get version from JSON
+    expat_info = get_dependency_version("expat", "win32")
+    if expat_info:
+        version = expat_info["version"]
+        hash = expat_info["sha256"]
+    else:
+        # Fallback to hardcoded values
+        version = "2.7.3"
+        hash = "821ac9710d2c073eaf13e1b1895a9c9aa66c1157a99635c639fbff65cdbdd732"
+
     url = f'https://github.com/libexpat/libexpat/releases/download/R_{version.replace(".", "_")}/expat-{version}.tar.xz'
     bash_refresh = dirs.source / "Modules" / "expat" / "refresh.sh"
     old = r'expected_libexpat_tag="R_\d+_\d+_\d"'
@@ -181,137 +245,43 @@ def update_expat(dirs, env):
     # Copy *.h and *.c to expat directory
     expat_lib_dir = dirs.source / "Modules" / "expat" / f"expat-{version}" / "lib"
     expat_dir = dirs.source / "Modules" / "expat"
+    updated_files = []
     for file in glob.glob(str(expat_lib_dir / "*.h")):
-        if expat_dir / os.path.basename(file):
-            (expat_dir / os.path.basename(file)).unlink()
+        target = expat_dir / os.path.basename(file)
+        if target.exists():
+            target.unlink()
         shutil.move(file, str(expat_dir))
+        updated_files.append(target)
     for file in glob.glob(str(expat_lib_dir / "*.c")):
-        if expat_dir / os.path.basename(file):
-            (expat_dir / os.path.basename(file)).unlink()
+        target = expat_dir / os.path.basename(file)
+        if target.exists():
+            target.unlink()
         shutil.move(file, str(expat_dir))
-    # Update sbom.spdx.json with the correct hashes. This became a thing in 3.12
-    # python Tools/build/generate_sbom.py doesn't work because it requires a git
-    # repository, so we have to do it manually.
-    if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]:
-        checksums = {
-            "Modules/expat/expat.h": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "a4395dd0589a97aab0904f7a5f5dc5781a086aa2",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "610b844bbfa3ec955772cc825db4d4db470827d57adcb214ad372d0eaf00e591",
-                },
-            ],
-            "Modules/expat/expat_external.h": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "8fdf2e79a7ab46a3c76c74ed7e5fe641cbef308d",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "ffb960af48b80935f3856a16e87023524b104f7fc1e58104f01db88ba7bfbcc9",
-                },
-            ],
-            "Modules/expat/internal.h": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "7dce7d98943c5db33ae05e54801dcafb4547b9dd",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "6bfe307d52e7e4c71dbc30d3bd902a4905cdd83bbe4226a7e8dfa8e4c462a157",
-                },
-            ],
-            "Modules/expat/refresh.sh": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "71812ca27328697a8dcae1949cd638717538321a",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "64fd1368de41e4ebc14593c65f5a676558aed44bd7d71c43ae05d06f9086d3b0",
-                },
-            ],
-            "Modules/expat/xmlparse.c": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "4c81a1f04fc653877c63c834145c18f93cd95f3e",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "04a379615f476d55f95ca1853107e20627b48ca4afe8d0fd5981ac77188bf0a6",
-                },
-            ],
-            "Modules/expat/xmlrole.h": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "ac2964cca107f62dd133bfd4736a9a17defbc401",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "92e41f373b67f6e0dcd7735faef3c3f1e2c17fe59e007e6b74beef6a2e70fa88",
-                },
-            ],
-            "Modules/expat/xmltok.c": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "1e2d35d90a1c269217f83d3bdf3c71cc22cb4c3f",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "98d0fc735041956cc2e7bbbe2fb8f03130859410e0aee5e8015f406a37c02a3c",
-                },
-            ],
-            "Modules/expat/xmltok.h": [
-                {
-                    "algorithm": "SHA1",
-                    "checksumValue": "d126831eaa5158cff187a8c93f4bc1c8118f3b17",
-                },
-                {
-                    "algorithm": "SHA256",
-                    "checksumValue": "91bf003a725a675761ea8d92cebc299a76fd28c3a950572f41bc7ce5327ee7b5",
-                },
-            ],
-        }
-        spdx_json = dirs.source / "Misc" / "sbom.spdx.json"
-        with open(str(spdx_json), "r") as f:
-            data = json.load(f)
-            for file in data["files"]:
-                if file["fileName"] in checksums.keys():
-                    print(file["fileName"])
-                    file["checksums"] = checksums[file["fileName"]]
-        with open(str(spdx_json), "w") as f:
-            json.dump(data, f, indent=2)
+        updated_files.append(target)
+
+    # Touch all updated files to ensure MSBuild rebuilds them
+    # (The original files may have newer timestamps)
+    now = time.time()
+    for target_file in updated_files:
+        os.utime(target_file, (now, now))
+
+    # Update SBOM with correct checksums for updated expat files
+    # Map SBOM file names to actual file paths
+    files_to_update = {}
+    for target_file in updated_files:
+        # SBOM uses relative paths from Python source root
+        relative_path = f"Modules/expat/{target_file.name}"
+        files_to_update[relative_path] = target_file
+
+    # Also include refresh.sh which was patched
+    bash_refresh = dirs.source / "Modules" / "expat" / "refresh.sh"
+    if bash_refresh.exists():
+        files_to_update["Modules/expat/refresh.sh"] = bash_refresh
 
+    update_sbom_checksums(dirs.source, files_to_update)
 
-def update_expat_check(env):
-    """
-    Check if the given python version should get an updated libexpat.
-
-    Patch libexpat on these versions and below:
-      - 3.9.23
-      - 3.10.18
-      - 3.11.13
-      - 3.12.11
-      - 3.13.7
-    """
-    relenv_version = Version(env["RELENV_PY_VERSION"])
-    if relenv_version.minor == 9 and relenv_version.micro <= 23:
-        return True
-    elif relenv_version.minor == 10 and relenv_version.micro <= 18:
-        return True
-    elif relenv_version.minor == 11 and relenv_version.micro <= 13:
-        return True
-    elif relenv_version.minor == 12 and relenv_version.micro <= 11:
-        return True
-    elif relenv_version.minor == 13 and relenv_version.micro <= 7:
-        return True
-    return False
-
-
-def build_python(env, dirs, logfp):
+
+def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None:
     """
     Run the commands to build Python.
 
@@ -328,27 +298,11 @@ def build_python(env, dirs, logfp):
     externals_dir = dirs.source / "externals"
     externals_dir.mkdir(parents=True, exist_ok=True)
 
-    # SQLITE
-    # TODO: Python 3.12 started creating an SBOM. We're doing something wrong
-    # TODO: updating sqlite so SBOM creation is failing. Gating here until we
-    # TODO: fix this. Here's the original gate:
-    # if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12"]:
-    if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11"]:
-        update_sqlite(dirs=dirs, env=env)
-
-    # XZ-Utils
-    # TODO: Python 3.12 started creating an SBOM. We're doing something wrong
-    # TODO: updating XZ so SBOM creation is failing. Gating here until we fix
-    # TODO: this. Here's the original gate:
-    # if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13", "3.14"]:
-    if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11"]:
-        update_xz(dirs=dirs, env=env)
-
-    # TODO: This was my attempt to fix the expat error during build... it failed
-    # TODO: so we're just skipping for now.
-    if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11"]:
-        if update_expat_check(env=env):
-            update_expat(dirs=dirs, env=env)
+    update_sqlite(dirs=dirs, env=env)
+
+    update_xz(dirs=dirs, env=env)
+
+    update_expat(dirs=dirs, env=env)
 
     arch_to_plat = {
         "amd64": "x64",
@@ -447,7 +401,7 @@ build.add(
 )
 
 
-def finalize(env, dirs, logfp):
+def finalize(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None:
     """
     Finalize sitecustomize, relenv runtime, and pip for Windows.
 
@@ -469,7 +423,7 @@ def finalize(env, dirs, logfp):
     python = dirs.prefix / "Scripts" / "python.exe"
     runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp)
 
-    def runpip(pkg):
+    def runpip(pkg: Union[str, os.PathLike[str]]) -> None:
         # XXX Support cross pip installs on windows
         env = os.environ.copy()
         target = None

relenv/buildenv.py

@@ -1,15 +1,21 @@
-# Copyright 2023-2025 Broadcom.
+# Copyright 2022-2025 Broadcom.
 # SPDX-License-Identifier: Apache-2.0
 """
 Helper for building libraries to install into a relenv environment.
 """
+from __future__ import annotations
+
+import argparse
 import json
 import logging
+import os
 import sys
+from typing import Any, cast
 
 from .common import (
     MACOS_DEVELOPMENT_TARGET,
-    RelenvException,
+    PlatformError,
+    RelenvEnvironmentError,
     get_toolchain,
     get_triplet,
 )
@@ -17,7 +23,9 @@ from .common import (
 log = logging.getLogger()
 
 
-def setup_parser(subparsers):
+def setup_parser(
+    subparsers: argparse._SubParsersAction[argparse.ArgumentParser],
+) -> None:
     """
     Setup the subparser for the ``relenv buildenv`` command.
 
@@ -36,30 +44,33 @@ def setup_parser(subparsers):
     )
 
 
-def is_relenv():
+def is_relenv() -> bool:
     """
     True when we are in a relenv environment.
     """
     return hasattr(sys, "RELENV")
 
 
-def buildenv(relenv_path=None):
+def buildenv(
+    relenv_path: str | os.PathLike[str] | None = None,
+) -> dict[str, str]:
     """
     Relenv build environment variable mapping.
     """
     if not relenv_path:
         if not is_relenv():
-            raise RelenvException("Not in a relenv environment")
-        relenv_path = sys.RELENV
+            raise RelenvEnvironmentError("Not in a relenv environment")
+        relenv_path = cast(str | os.PathLike[str], cast(Any, sys).RELENV)
 
     if sys.platform != "linux":
-        raise RelenvException("buildenv is only supported on Linux")
+        raise PlatformError("buildenv is only supported on Linux")
 
     toolchain = get_toolchain()
     if not toolchain:
-        raise RelenvException("buildenv is only supported on Linux")
+        raise PlatformError("buildenv is only supported on Linux")
 
     triplet = get_triplet()
+    sysroot = f"{toolchain}/{triplet}/sysroot"
     env = {
         "RELENV_BUILDENV": "1",
         "TOOLCHAIN_PATH": f"{toolchain}",
@@ -67,30 +78,50 @@ def buildenv(relenv_path=None):
         "RELENV_PATH": f"{relenv_path}",
         "CC": f"{toolchain}/bin/{triplet}-gcc",
         "CXX": f"{toolchain}/bin/{triplet}-g++",
-        "CFLAGS": f"-I{relenv_path}/include -I{toolchain}/sysroot/usr/include",
+        "CFLAGS": (
+            f"--sysroot={sysroot} "
+            f"-fPIC "
+            f"-I{relenv_path}/include "
+            f"-I{sysroot}/usr/include"
+        ),
         "CXXFLAGS": (
+            f"--sysroot={sysroot} "
+            f"-fPIC "
             f"-I{relenv_path}/include "
-            f"-I{toolchain}/{triplet}/sysroot/usr/include "
-            f"-L{relenv_path}/lib -L{toolchain}/{triplet}/sysroot/lib "
+            f"-I{sysroot}/usr/include "
+            f"-L{relenv_path}/lib -L{sysroot}/lib "
             f"-Wl,-rpath,{relenv_path}/lib"
         ),
         "CPPFLAGS": (
-            f"-I{relenv_path}/include " f"-I{toolchain}/{triplet}/sysroot/usr/include"
+            f"--sysroot={sysroot} "
+            f"-fPIC "
+            f"-I{relenv_path}/include "
+            f"-I{sysroot}/usr/include"
         ),
         "CMAKE_CFLAGS": (
-            f"-I{relenv_path}/include " f"-I{toolchain}/{triplet}/sysroot/usr/include"
+            f"--sysroot={sysroot} "
+            f"-fPIC "
+            f"-I{relenv_path}/include "
+            f"-I{sysroot}/usr/include"
         ),
         "LDFLAGS": (
-            f"-L{relenv_path}/lib -L{toolchain}/{triplet}/sysroot/lib "
+            f"--sysroot={sysroot} "
+            f"-L{relenv_path}/lib -L{sysroot}/lib "
             f"-Wl,-rpath,{relenv_path}/lib"
         ),
+        "CRATE_CC_NO_DEFAULTS": "1",
+        "OPENSSL_DIR": f"{relenv_path}",
+        "OPENSSL_INCLUDE_DIR": f"{relenv_path}/include",
+        "OPENSSL_LIB_DIR": f"{relenv_path}/lib",
+        "PKG_CONFIG_PATH": f"{relenv_path}/lib/pkgconfig",
+        "RUSTFLAGS": f"-L {relenv_path}/lib -C link-arg=-Wl,-rpath,{relenv_path}/lib",
     }
-    if sys.platform == "dawin":
+    if sys.platform == "darwin":
         env["MACOS_DEVELOPMENT_TARGET"] = MACOS_DEVELOPMENT_TARGET
     return env
 
 
-def main(args):
+def main(args: argparse.Namespace) -> None:
     """
     The entrypoint into the ``relenv buildenv`` command.
 

relenv/check.py

@@ -1,18 +1,23 @@
-# Copyright 2025 Broadcom.
+# Copyright 2022-2025 Broadcom.
 # SPDX-License-Identifier: Apache-2.0
 """
 Check the integrety of a relenv environment.
 """
+from __future__ import annotations
+
+import argparse
 import logging
 import pathlib
 import sys
 
-from . import relocate
+from relenv import relocate
 
-log = logging.getLogger()
+log: logging.Logger = logging.getLogger(__name__)
 
 
-def setup_parser(subparsers):
+def setup_parser(
+    subparsers: argparse._SubParsersAction[argparse.ArgumentParser],
+) -> None:
     """
     Setup the subparser for the ``relenv check`` command.
 
@@ -23,7 +28,7 @@ def setup_parser(subparsers):
     subparser.set_defaults(func=main)
 
 
-def main(args):
+def main(args: argparse.Namespace) -> None:
     """
     The entrypoint into the ``relenv check`` command.