relenv 0.19.4__tar.gz → 0.20.0__tar.gz

{relenv-0.19.4/relenv.egg-info → relenv-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: relenv
-Version: 0.19.4
+Version: 0.20.0
 Project-URL: Source Code, https://github.com/saltstack/relative-environment-for-python
 Project-URL: Documentation, https://relenv.readthedocs.io/en/latest/
 Project-URL: Changelog, https://relenv.readthedocs.io/en/latest/changelog.html

{relenv-0.19.4 → relenv-0.20.0}/relenv/__main__.py

@@ -6,7 +6,7 @@ The entrypoint into relenv.
 
 from argparse import ArgumentParser
 
-from . import build, buildenv, check, create, fetch, toolchain
+from . import build, buildenv, check, create, fetch, pyversions, toolchain
 from .common import __version__
 
 
@@ -34,6 +34,7 @@ def setup_cli():
         fetch,
         check,
         buildenv,
+        pyversions,
     ]
     for mod in modules_to_setup:
         mod.setup_parser(subparsers)
@@ -47,10 +48,9 @@ def main():
     """
     parser = setup_cli()
     args = parser.parse_args()
-    # args.func(args)
-    try:
+    if hasattr(args, "func"):
         args.func(args)
-    except AttributeError:
+    else:
         parser.print_help()
         parser.exit(1, "\nNo subcommand given...\n\n")
 

{relenv-0.19.4 → relenv-0.20.0}/relenv/build/__init__.py

@@ -11,6 +11,8 @@ import signal
 from . import linux, darwin, windows
 from .common import builds, CHECK_VERSIONS_SUPPORT
 
+from ..pyversions import python_versions, Version
+
 from ..common import build_arch
 
 
@@ -26,13 +28,6 @@ def platform_module():
         return windows
 
 
-def platform_versions():
-    """
-    Return the right module based on `sys.platform`.
-    """
-    return list(builds.builds[sys.platform].keys())
-
-
 def setup_parser(subparsers):
     """
     Setup the subparser for the ``build`` command.
@@ -63,8 +58,7 @@ def setup_parser(subparsers):
     )
     build_subparser.add_argument(
         "--python",
-        default=platform_versions()[0],
-        choices=platform_versions(),
+        default="3.10.17",
         type=str,
         help="The python version [default: %(default)s]",
     )
@@ -146,8 +140,34 @@ def main(args):
         print(f"Unsupported platform: {sys.platform}")
         sys.exit(1)
 
+    requested = Version(args.python)
+
+    if requested.micro:
+        pyversions = python_versions()
+        if requested not in pyversions:
+            print(f"Unknown version {requested}")
+            strversions = "\n".join([str(_) for _ in pyversions])
+            print(f"Known versions are:\n{strversions}")
+            sys.exit(1)
+        build_version = requested
+    else:
+        pyversions = python_versions(args.python)
+        build_version = pyversions[0]
+
+    # print(pyversions)
+    # print(pyversions[0].major)
+    # print(pyversions[0].minor)
+    # print(pyversions[0].micro)
+    # print(pyversions[0].pre)
+    # print(pyversions[0].post)
+    # print(pyversions)
+    print(f"Build Python {build_version}")
+
     # XXX
-    build = builds.builds[sys.platform][args.python]
+    build = builds.builds[sys.platform]
+    build.version = str(build_version)
+    build.dirs.version = str(build_version)
+    build.recipies["python"]["download"].version = str(build_version)
 
     if args.check_versions:
         if not CHECK_VERSIONS_SUPPORT:

{relenv-0.19.4 → relenv-0.20.0}/relenv/build/common.py

@@ -78,7 +78,9 @@ RELENV_PTH = (
 
 
 SYSCONFIGDATA = """
-import pathlib, sys, platform, os
+import pathlib, sys, platform, os, logging
+
+log = logging.getLogger(__name__)
 
 def build_arch():
     machine = platform.machine()
@@ -112,7 +114,22 @@ else:
     DATA_DIR = DEFAULT_DATA_DIR
 
 buildroot = pydir.parent.parent
-toolchain = DATA_DIR / "toolchain" / get_triplet()
+
+if sys.platform == "linux":
+    toolchain = ""
+    ppbt = None
+    try:
+        import ppbt
+    except ImportError:
+        pass
+    if ppbt:
+        env = ppbt.environ(auto_extract=True)
+        toolchain = pathlib.Path(env["TOOLCHAIN_PATH"])
+    else:
+        print("ppbt package not installed")
+else:
+    toolchain = DATA_DIR / "toolchain" / get_triplet()
+
 build_time_vars = {}
 for key in _build_time_vars:
     val = _build_time_vars[key]
@@ -768,9 +785,9 @@ class Builds:
         else:
             build = Builder(*args, **kwargs)
         if platform not in self.builds:
-            self.builds[platform] = {build.version: build}
+            self.builds[platform] = build
         else:
-            self.builds[platform][build.version] = build
+            self.builds[platform] = build
         return build
 
 
@@ -809,10 +826,6 @@ class Builder:
         self.build_arch = build_arch()
         self.build_triplet = get_triplet(self.build_arch)
         self.arch = arch
-        self.triplet = get_triplet(self.arch)
-        self.version = version
-        # XXX Refactor WorkDirs, Dirs and Builder so as not to duplicate logic
-        self.prefix = self.dirs.build / f"{self.version}-{self.triplet}"
         self.sources = self.dirs.src
         self.downloads = self.dirs.download
 
@@ -823,6 +836,7 @@ class Builder:
 
         self.build_default = build_default
         self.populate_env = populate_env
+        self.version = version
         self.toolchains = get_toolchain(root=self.dirs.root)
         self.set_arch(self.arch)
 
@@ -855,13 +869,19 @@ class Builder:
         :type arch: str
         """
         self.arch = arch
-        self.triplet = get_triplet(self.arch)
-        self.prefix = self.dirs.build / f"{self.version}-{self.triplet}"
         if sys.platform in ["darwin", "win32"]:
             self.toolchain = None
         else:
             self.toolchain = get_toolchain(self.arch, self.dirs.root)
 
+    @property
+    def triplet(self):
+        return get_triplet(self.arch)
+
+    @property
+    def prefix(self):
+        return self.dirs.build / f"{self.version}-{self.triplet}"
+
     @property
     def _triplet(self):
         if sys.platform == "darwin":
@@ -974,6 +994,7 @@ class Builder:
             env["RELENV_NATIVE_PY"] = str(native_root / "bin" / "python3")
 
         self.populate_env(env, dirs)
+
         _ = dirs.to_dict()
         for k in _:
             log.info("Directory %s %s", k, _[k])
@@ -1191,10 +1212,11 @@ class Builder:
         :rtype: list
         """
         fail = []
-        if self.toolchain and not self.toolchain.exists():
-            fail.append(
-                f"Toolchain for {self.arch} does not exist. Please use relenv toolchain to obtain a toolchain."
-            )
+        if sys.platform == "linux":
+            if not self.toolchain or not self.toolchain.exists():
+                fail.append(
+                    f"Toolchain for {self.arch} does not exist. Please pip install ppbt."
+                )
         return fail
 
     def __call__(
@@ -1470,6 +1492,9 @@ def finalize(env, dirs, logfp):
         env["RELENV_CROSS"] = dirs.prefix
         python = env["RELENV_NATIVE_PY"]
     logfp.write("\nRUN ENSURE PIP\n")
+
+    env.pop("RELENV_BUILDENV")
+
     runcmd(
         [str(python), "-m", "ensurepip"],
         env=env,
@@ -1578,9 +1603,8 @@ def create_archive(tarfp, toarchive, globs, logfp=None):
     :param logfp: A pointer to the log file
     :type logfp: file
     """
-    if logfp is None:
-        log.info("Current directory %s", os.getcwd())
-        log.info("Creating archive %s", tarfp.name)
+    log.debug("Current directory %s", os.getcwd())
+    log.debug("Creating archive %s", tarfp.name)
     for root, _dirs, files in os.walk(toarchive):
         relroot = pathlib.Path(root).relative_to(toarchive)
         for f in files:
@@ -1591,9 +1615,7 @@ def create_archive(tarfp, toarchive, globs, logfp=None):
                     matches = True
                     break
             if matches:
-                if logfp is None:
-                    log.info("Adding %s", relpath)
+                log.debug("Adding %s", relpath)
                 tarfp.add(relpath, relpath, recursive=False)
             else:
-                if logfp is None:
-                    log.info("Skipping %s", relpath)
+                log.debug("Skipping %s", relpath)

{relenv-0.19.4 → relenv-0.20.0}/relenv/build/darwin.py

@@ -72,7 +72,7 @@ def build_python(env, dirs, logfp):
     runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp)
 
 
-build = builds.add("darwin", populate_env=populate_env, version="3.10.17")
+build = builds.add("darwin", populate_env=populate_env)
 
 build.add(
     "openssl",
@@ -128,18 +128,3 @@ build.add(
         "python",
     ],
 )
-
-build = build.copy(
-    version="3.11.11", checksum="acf539109b024d3c5f1fc63d6e7f08cd294ba56d"
-)
-builds.add("darwin", builder=build)
-
-build = build.copy(
-    version="3.12.9", checksum="465d8a664e63dc5aa1f0d90cd1d0000a970ee2fb"
-)
-builds.add("darwin", builder=build)
-
-build = build.copy(
-    version="3.13.5", checksum="dbf3aed444cbb2221eabfb52688aa371423aa0ba"
-)
-builds.add("darwin", builder=build)

{relenv-0.19.4 → relenv-0.20.0}/relenv/build/linux.py

@@ -445,7 +445,7 @@ def build_python(env, dirs, logfp):
     # runcmd([str(python), "-m", "ensurepip", "-U"], env=env, stderr=logfp, stdout=logfp)
 
 
-build = builds.add("linux", populate_env=populate_env, version="3.10.17")
+build = builds.add("linux", populate_env=populate_env)
 
 build.add(
     "openssl",
@@ -651,18 +651,3 @@ build.add(
         "openssl-fips-module",
     ],
 )
-
-build = build.copy(
-    version="3.11.11", checksum="acf539109b024d3c5f1fc63d6e7f08cd294ba56d"
-)
-builds.add("linux", builder=build)
-
-build = build.copy(
-    version="3.12.9", checksum="465d8a664e63dc5aa1f0d90cd1d0000a970ee2fb"
-)
-builds.add("linux", builder=build)
-
-build = build.copy(
-    version="3.13.5", checksum="dbf3aed444cbb2221eabfb52688aa371423aa0ba"
-)
-builds.add("linux", builder=build)

{relenv-0.19.4 → relenv-0.20.0}/relenv/build/windows.py

@@ -130,7 +130,7 @@ def build_python(env, dirs, logfp):
     )
 
 
-build = builds.add("win32", populate_env=populate_env, version="3.10.17")
+build = builds.add("win32", populate_env=populate_env)
 
 build.add(
     "python",
@@ -210,18 +210,3 @@ build.add(
     build_func=finalize,
     wait_on=["python"],
 )
-
-build = build.copy(
-    version="3.11.11", checksum="acf539109b024d3c5f1fc63d6e7f08cd294ba56d"
-)
-builds.add("win32", builder=build)
-
-build = build.copy(
-    version="3.12.9", checksum="465d8a664e63dc5aa1f0d90cd1d0000a970ee2fb"
-)
-builds.add("win32", builder=build)
-
-build = build.copy(
-    version="3.13.5", checksum="dbf3aed444cbb2221eabfb52688aa371423aa0ba"
-)
-builds.add("win32", builder=build)

{relenv-0.19.4 → relenv-0.20.0}/relenv/buildenv.py

@@ -3,10 +3,16 @@
 """
 Helper for building libraries to install into a relenv environment.
 """
+import json
 import logging
 import sys
 
-from .common import MACOS_DEVELOPMENT_TARGET, RelenvException, get_triplet, work_dirs
+from .common import (
+    MACOS_DEVELOPMENT_TARGET,
+    RelenvException,
+    get_toolchain,
+    get_triplet,
+)
 
 log = logging.getLogger()
 
@@ -22,6 +28,12 @@ def setup_parser(subparsers):
         "buildenv", description="Relenv build environment"
     )
     subparser.set_defaults(func=main)
+    subparser.add_argument(
+        "--json",
+        default=False,
+        action="store_true",
+        help=("Output json to stdout instead of export statments"),
+    )
 
 
 def is_relenv():
@@ -43,9 +55,11 @@ def buildenv(relenv_path=None):
     if sys.platform != "linux":
         raise RelenvException("buildenv is only supported on Linux")
 
-    dirs = work_dirs()
+    toolchain = get_toolchain()
+    if not toolchain:
+        raise RelenvException("buildenv is only supported on Linux")
+
     triplet = get_triplet()
-    toolchain = dirs.toolchain / get_triplet()
     env = {
         "RELENV_BUILDENV": "1",
         "TOOLCHAIN_PATH": f"{toolchain}",
@@ -90,12 +104,13 @@ def main(args):
     if sys.platform != "linux":
         log.error("buildenv is only supported on Linux.")
 
-    # dirs = work_dirs()
-    # triplet = get_triplet()
-    # toolchain = dirs.toolchain / get_triplet()
+    if args.json:
+        print(json.dumps(buildenv()))
+        sys.exit(0)
 
     script = ""
     for k, v in buildenv().items():
         script += f'export {k}="{v}"\n'
 
     print(script)
+    sys.exit(0)

{relenv-0.19.4 → relenv-0.20.0}/relenv/common.py

@@ -18,7 +18,7 @@ import threading
 import time
 
 # relenv package version
-__version__ = "0.19.4"
+__version__ = "0.20.0"
 
 MODULE_DIR = pathlib.Path(__file__).resolve().parent
 
@@ -28,6 +28,8 @@ DARWIN = "darwin"
 
 MACOS_DEVELOPMENT_TARGET = "10.15"
 
+REQUEST_HEADERS = {"User-Agent": f"relenv {__version__}"}
+
 CHECK_HOSTS = (
     "packages.broadcom.com/artifactory/saltproject-generic",
     "repo.saltproject.io",
@@ -160,6 +162,7 @@ class WorkDirs:
 
     def __init__(self, root):
         self.root = root
+        self.data = DATA_DIR
         self.toolchain_config = work_dir("toolchain", self.root)
         self.toolchain = work_dir("toolchain", DATA_DIR)
         self.build = work_dir("build", DATA_DIR)
@@ -224,10 +227,18 @@ def get_toolchain(arch=None, root=None):
     :return: The directory holding the toolchain
     :rtype: ``pathlib.Path``
     """
-    dirs = work_dirs(root)
-    if arch:
-        return dirs.toolchain / "{}-linux-gnu".format(arch)
-    return dirs.toolchain
+    if sys.platform != "linux":
+        return DATA_DIR
+    ppbt = None
+
+    try:
+        import ppbt
+    except ImportError:
+        pass
+
+    if ppbt:
+        env = ppbt.environ(auto_extract=True)
+        return pathlib.Path(env["TOOLCHAIN_PATH"])
 
 
 def get_triplet(machine=None, plat=None):
@@ -342,17 +353,30 @@ def get_download_location(url, dest):
     return os.path.join(dest, os.path.basename(url))
 
 
-def check_url(url, timeout=30):
+def check_url(url, timestamp=None, timeout=30):
     """
     Check that the url returns a 200.
     """
     # Late import so we do not import hashlib before runtime.bootstrap is called.
+    import time
     import urllib.request
 
+    headers = dict(REQUEST_HEADERS)
+    req = urllib.request.Request(url)
+
+    if timestamp:
+        headers["If-Modified-Since"] = time.strftime(
+            "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp)
+        )
+
+    for k, v in headers.items():
+        req.add_header(k, v)
+
     fin = None
     try:
-        fin = urllib.request.urlopen(url, timeout=timeout)
-    except Exception:
+        fin = urllib.request.urlopen(req, timeout=timeout)
+    except Exception as exc:
+        print(exc)
         return False
     finally:
         if fin:
@@ -405,6 +429,62 @@ def fetch_url(url, fp, backoff=3, timeout=30):
     log.info("Download complete %s", url)
 
 
+def fetch_url_content(url, backoff=3, timeout=30):
+    """
+    Fetch the contents of a url.
+
+    This method will store the contents in the given file like object.
+    """
+    # Late import so we do not import hashlib before runtime.bootstrap is called.
+    import gzip
+    import io
+    import urllib.error
+    import urllib.request
+
+    fp = io.BytesIO()
+
+    last = time.time()
+    if backoff < 1:
+        backoff = 1
+    n = 0
+    while n < backoff:
+        n += 1
+        try:
+            fin = urllib.request.urlopen(url, timeout=timeout)
+        except (
+            urllib.error.HTTPError,
+            urllib.error.URLError,
+            http.client.RemoteDisconnected,
+        ) as exc:
+            if n >= backoff:
+                raise RelenvException(f"Error fetching url {url} {exc}")
+            log.debug("Unable to connect %s", url)
+            time.sleep(n * 10)
+    log.info("url opened %s", url)
+    try:
+        total = 0
+        size = 1024 * 300
+        block = fin.read(size)
+        while block:
+            total += size
+            if time.time() - last > 10:
+                log.info("%s > %d", url, total)
+                last = time.time()
+            fp.write(block)
+            block = fin.read(10240)
+    finally:
+        fin.close()
+        # fp.close()
+    log.info("Download complete %s", url)
+    fp.seek(0)
+    info = fin.info()
+    if "content-encoding" in info:
+        if info["content-encoding"] == "gzip":
+            log.debug("Found gzipped content")
+            fp = gzip.GzipFile(fileobj=fp)
+    return fp.read().decode()
+
+
 def download_url(url, dest, verbose=True, backoff=3, timeout=60):
     """
     Download the url to the provided destination.

{relenv-0.19.4 → relenv-0.20.0}/relenv/create.py

@@ -10,7 +10,6 @@ import pathlib
 import sys
 import tarfile
 
-from .build import platform_versions
 from .common import RelenvException, arches, archived_build, build_arch
 
 
@@ -61,8 +60,7 @@ def setup_parser(subparsers):
     )
     subparser.add_argument(
         "--python",
-        default=platform_versions()[0],
-        choices=platform_versions(),
+        default="3.10.17",
         type=str,
         help="The python version [default: %(default)s]",
     )
@@ -89,7 +87,7 @@ def create(name, dest=None, arch=None, version=None):
         writeto = pathlib.Path(name).resolve()
 
     if version is None:
-        version = platform_versions()[0]
+        version = "3.10.17"
 
     if pathlib.Path(writeto).exists():
         raise CreateException("The requested path already exists.")

{relenv-0.19.4 → relenv-0.20.0}/relenv/fetch.py

@@ -7,7 +7,7 @@ The ``relenv fetch`` command.
 import os
 import sys
 
-from .build import platform_module, platform_versions
+from .build import platform_module
 from .common import (
     CHECK_HOSTS,
     DATA_DIR,
@@ -39,9 +39,7 @@ def setup_parser(subparsers):
     )
     subparser.add_argument(
         "--python",
-        default=platform_versions()[0],
-        # Allow fetching of older versions
-        # choices=platform_versions(),
+        default="3.10.17",
         type=str,
         help="The python version [default: %(default)s]",
     )

relenv-0.20.0/relenv/manifest.py

@@ -0,0 +1,34 @@
+# Copyright 2025 Broadcom.
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Relenv manifest.
+"""
+import hashlib
+import os
+import sys
+
+
+def manifest(root=None):
+    """
+    List all the file in a relenv and their hashes.
+    """
+    if root is None:
+        root = getattr(sys, "RELENV", os.getcwd())
+    for root, dirs, files in os.walk(root):
+        for file in files:
+            hsh = hashlib.sha256()
+            try:
+                with open(root + os.path.sep + file, "rb") as fp:
+                    while True:
+                        chunk = fp.read(9062)
+                        if not chunk:
+                            break
+                        hsh.update(chunk)
+            except OSError:
+                pass
+            print(f"{root + os.path.sep + file} => {hsh.hexdigest()}")
+
+
+if __name__ == "__main__":
+    manifest()