librelane 3.0.0.dev29__tar.gz → 3.0.0.dev32__tar.gz

{librelane-3.0.0.dev29 → librelane-3.0.0.dev32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: librelane
-Version: 3.0.0.dev29
+Version: 3.0.0.dev32
 Summary: An infrastructure for implementing chip design flows
 Home-page: https://github.com/librelane/librelane
 License: Apache-2.0

{librelane-3.0.0.dev29 → librelane-3.0.0.dev32}/librelane/__main__.py

@@ -22,7 +22,6 @@ import shutil
 import marshal
 import tempfile
 import traceback
-import subprocess
 from textwrap import dedent
 from functools import partial
 from typing import Any, Dict, Sequence, Tuple, Type, Optional, List
@@ -275,20 +274,12 @@ def run_included_example(
         if os.path.isdir(final_path):
             print(f"A directory named {value} already exists.", file=sys.stderr)
             ctx.exit(1)
-        # 1. Copy the files
-        shutil.copytree(
-            example_path,
-            final_path,
-            symlinks=False,
-        )
-
-        # 2. Make files writable
-        if os.name == "posix":
-            subprocess.check_call(["chmod", "-R", "755", final_path])
 
+        # 1. Copy the files
+        common.recreate_tree(example_path, final_path)
         config_file = glob.glob(os.path.join(final_path, "config.*"))[0]
 
-        # 3. Run
+        # 2. Run
         run(
             ctx,
             config_files=[config_file],
@@ -318,28 +309,38 @@ def cli_in_container(
     if not value:
         return
 
-    docker_mounts = list(ctx.params.get("docker_mounts") or ())
-    docker_tty: bool = ctx.params.get("docker_tty", True)
+    mounts = list(ctx.params.get("docker_mounts") or ())
+    tty: bool = ctx.params.get("docker_tty", True)
     pdk_root = ctx.params.get("pdk_root")
-    argv = sys.argv[sys.argv.index("--dockerized") + 1 :]
+
+    try:
+        containerized_index = sys.argv.index("--dockerized")
+    except ValueError:
+        containerized_index = sys.argv.index("--containerized")
+
+    argv = sys.argv[containerized_index + 1 :]
 
     final_argv = ["zsh"]
     if len(argv) != 0:
-        final_argv = ["librelane"] + argv
+        final_argv = ["python3", "-m", "librelane"] + argv
 
-    docker_image = os.getenv(
+    container_image = os.getenv(
         "LIBRELANE_IMAGE_OVERRIDE", f"ghcr.io/librelane/librelane:{__version__}"
     )
 
     try:
         run_in_container(
-            docker_image,
+            container_image,
             final_argv,
             pdk_root=pdk_root,
-            other_mounts=docker_mounts,
-            tty=docker_tty,
+            other_mounts=mounts,
+            tty=tty,
         )
+    except ValueError as e:
+        err(e)
+        ctx.exit(1)
     except Exception as e:
+        traceback.print_exc(file=sys.stderr)
         err(e)
         ctx.exit(1)
 
@@ -374,25 +375,28 @@ o = partial(option, show_default=True)
     "Containerization options",
     o(
         "--docker-mount",
+        "--container-mount",
         "-m",
         "docker_mounts",
         multiple=True,
-        is_eager=True,  # docker options should be processed before anything else
+        is_eager=True,  # container options should be processed before anything else
         default=(),
-        help="Used to mount more directories in dockerized mode. If a valid directory is specified, it will be mounted in the same path in the container. Otherwise, the value of the option will be passed to the Docker-compatible container engine verbatim. Must be passed before --dockerized, has no effect if --dockerized is not set.",
+        help="Used to mount more directories in dockerized mode. If a valid directory is specified, it will be mounted in the same path in the container. Otherwise, the value of the option will be passed to the container engine verbatim. Must be passed before --containerized/--dockerized, has no effect if not set.",
     ),
     o(
         "--docker-tty/--docker-no-tty",
-        is_eager=True,  # docker options should be processed before anything else
+        "--container-tty/--container-no-tty",
+        is_eager=True,  # container options should be processed before anything else
         default=True,
-        help="Controls the allocation of a virtual terminal by passing -t to the Docker-compatible container engine invocation. Must be passed before --dockerized, has no effect if --dockerized is not set.",
+        help="Controls the allocation of a virtual terminal by passing -t to the Docker-compatible container engine invocation. Must be passed before --containerized/--dockerized, has no effect if not set.",
     ),
     o(
         "--dockerized",
+        "--containerized",
         default=False,
         is_flag=True,
         is_eager=True,  # docker options should be processed before anything else
-        help="Run the remaining flags using a Docker container. Some caveats apply. Must precede all options except --docker-mount, --docker-tty/--docker-no-tty.",
+        help="Run the remaining flags using a containerized version of LibreLane. Some caveats apply. Must precede all options except --{docker,container}-mount, --{docker,container}-[no-]tty.",
         callback=cli_in_container,
     ),
 )

{librelane-3.0.0.dev29 → librelane-3.0.0.dev32}/librelane/common/__init__.py

@@ -41,6 +41,7 @@ from .misc import (
     format_size,
     format_elapsed_time,
     Filter,
+    recreate_tree,
     get_latest_file,
     process_list_file,
     count_occurences,

{librelane-3.0.0.dev29 → librelane-3.0.0.dev32}/librelane/common/misc.py

@@ -21,6 +21,7 @@ import re
 import glob
 import gzip
 import yaml
+import shutil
 import typing
 import pathlib
 import fnmatch
@@ -328,6 +329,37 @@ class Filter(object):
                 yield input
 
 
+def recreate_tree(
+    source: AnyPath,
+    target: AnyPath,
+):
+    """
+    This function attempts to recreate a file tree from a source path in another
+    target path.
+
+    Permissions are not copied over. Symlinks and hardlinks are followed.
+
+    Directories are not recreated unless they contain files as (grand)children.
+
+    If the source and target are the same, the function returns early and does
+    nothing.
+
+    :param source: The source file tree to replicate
+    :param target: The target path to recreate the file tree within
+    """
+    source = os.path.abspath(source)
+    target = os.path.abspath(target)
+    if os.path.exists(target) and os.path.samefile(source, target):
+        return
+    for dirname, _, files in os.walk(source):
+        for file in files:
+            resolved = os.path.join(dirname, file)
+            resolved_target = os.path.join(target, os.path.relpath(resolved, source))
+            os.makedirs(os.path.dirname(resolved_target), exist_ok=True)
+            with open(resolved, "rb") as fi, open(resolved_target, "wb") as fo:
+                shutil.copyfileobj(fi, fo)
+
+
 def get_latest_file(in_path: Union[str, os.PathLike], filename: str) -> Optional[Path]:
     """
     :param in_path: A directory to search in
@@ -394,7 +426,7 @@ def _get_process_limit() -> int:
 
 def gzopen(filename: AnyPath, mode="rt") -> IO[Any]:
     """
-    This method (tries to?) emulate the gzopen from the Linux Standard Base,
+    This function (tries to?) emulate the gzopen from the Linux Standard Base,
     specifically this part:
 
     If path refers to an uncompressed file, and mode refers to a read mode,

{librelane-3.0.0.dev29 → librelane-3.0.0.dev32}/librelane/container.py

@@ -18,7 +18,6 @@ import re
 import uuid
 import shlex
 import pathlib
-import tempfile
 import subprocess
 from typing import List, NoReturn, Sequence, Optional, Union, Tuple
 
@@ -27,15 +26,15 @@ import semver
 
 from .common import mkdirp
 from .logging import err, info, warn
-from .env_info import OSInfo
+from .env_info import ContainerInfo, OSInfo
 
-CONTAINER_ENGINE = os.getenv("OPENLANE_CONTAINER_ENGINE", "docker")
+__file_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 
 def permission_args(osinfo: OSInfo) -> List[str]:
     if (
         osinfo.kernel == "Linux"
-        and osinfo.container_info is not None
+        and isinstance(osinfo.container_info, ContainerInfo)
         and osinfo.container_info.engine == "docker"
         and not osinfo.container_info.rootless
     ):
@@ -70,9 +69,9 @@ def gui_args(osinfo: OSInfo) -> List[str]:
     return args
 
 
-def image_exists(image: str) -> bool:
+def image_exists(ce_path: str, image: str) -> bool:
     images = (
-        subprocess.check_output([CONTAINER_ENGINE, "images", image])
+        subprocess.check_output([ce_path, "images", image])
         .decode("utf8")
         .rstrip()
         .split("\n")[1:]
@@ -116,14 +115,14 @@ def remote_manifest_exists(image: str) -> bool:
     return True
 
 
-def ensure_image(image: str) -> bool:
-    if image_exists(image):
+def ensure_image(ce_path: str, image: str) -> bool:
+    if image_exists(ce_path, image):
         return True
 
     try:
-        subprocess.check_call([CONTAINER_ENGINE, "pull", image])
+        subprocess.check_call([ce_path, "pull", image])
     except subprocess.CalledProcessError:
-        err(f"Failed to pull image {image} from the container registries.")
+        err(f"Failed to pull image '{image}' from the container registries.")
         return False
 
     return True
@@ -150,6 +149,22 @@ def sanitize_path(path: Union[str, os.PathLike]) -> Tuple[str, str]:
     return (abspath, mountable_path)
 
 
+def container_version_error(input: str, against: str) -> Optional[str]:
+    if input == "UNKNOWN":
+        return (
+            "Could not determine version for %s. You may encounter unexpected issues."
+        )
+    if semver.compare(input, against) < 0:
+        return f"Your %s version ({input}) is out of date. You may encounter unexpected issues."
+    return None
+
+
+def ubuntu_version_at_least(current: str, minimum: str) -> bool:
+    if current == "UNKNOWN":
+        return False
+    return tuple(map(int, current.split("."))) >= tuple(map(int, minimum.split(".")))
+
+
 def run_in_container(
     image: str,
     args: Sequence[str],
@@ -169,20 +184,31 @@ def run_in_container(
             f"Unsupported host operating system '{osinfo.kernel}'. You may encounter unexpected issues."
         )
 
-    if osinfo.container_info is None:
+    if not isinstance(osinfo.container_info, ContainerInfo):
         raise FileNotFoundError("No compatible container engine found.")
 
-    if osinfo.container_info.engine.lower() == "docker":
-        if semver.compare(osinfo.container_info.version, "25.0.5") < 0:
+    ce_path = osinfo.container_info.path
+    assert ce_path is not None
+
+    engine_name = osinfo.container_info.engine.lower()
+    if engine_name == "docker":
+        if error := container_version_error(osinfo.container_info.version, "25.0.5"):
+            warn(error % engine_name)
+    elif engine_name == "podman":
+        if osinfo.distro.lower() == "ubuntu" and not ubuntu_version_at_least(
+            osinfo.distro_version, "24.04"
+        ):
             warn(
-                f"Your Docker engine version ({osinfo.container_info.version}) is out of date. You may encounter unexpected issues."
+                "Versions of Podman for Ubuntu before Ubuntu 24.04 are generally pretty buggy. We recommend using Docker instead if possible."
             )
+        elif error := container_version_error(osinfo.container_info.version, "4.1.0"):
+            warn(error % engine_name)
     else:
         warn(
-            f"Unsupported container engine '{osinfo.container_info.engine}'. You may encounter unexpected issues."
+            f"Unsupported container engine referenced by '{osinfo.container_info.path}'. You may encounter unexpected issues."
         )
 
-    if not ensure_image(image):
+    if not ensure_image(ce_path, image):
         raise ValueError(f"Failed to use image '{image}'.")
 
     terminal_args = ["-i"]
@@ -222,15 +248,6 @@ def run_in_container(
         mount_args += ["-v", f"{from_cwd}:{to_cwd}"]
     mount_args += ["-w", to_cwd]
 
-    tempdir = tempfile.mkdtemp("librelane_docker")
-
-    mount_args += [
-        "-v",
-        f"{tempdir}:/tmp",
-        "-e",
-        "TMPDIR=/tmp",
-    ]
-
     if other_mounts is not None:
         for mount in other_mounts:
             if os.path.isdir(mount):
@@ -242,9 +259,13 @@ def run_in_container(
 
     container_id = str(uuid.uuid4())
 
+    if os.getenv("_MOUNT_HOST_LIBRELANE") == "1":
+        host_librelane_pythonpath = os.path.dirname(__file_dir__)
+        mount_args += ["-v", f"{host_librelane_pythonpath}:/host_librelane"]
+
     cmd = (
         [
-            CONTAINER_ENGINE,
+            ce_path,
             "run",
             "--rm",
             "--name",
@@ -261,4 +282,4 @@ def run_in_container(
     info("Running containerized command:")
     print(shlex.join(cmd))
 
-    os.execlp(CONTAINER_ENGINE, *cmd)
+    os.execlp(ce_path, *cmd)

{librelane-3.0.0.dev29 → librelane-3.0.0.dev32}/librelane/env_info.py

@@ -23,17 +23,16 @@ import os
 import re
 import sys
 import json
+import shutil
 import tempfile
 import platform
 import subprocess
 
 try:
-    from typing import Optional, Dict, List  # noqa: F401
+    from typing import Union, Optional, Dict, List  # noqa: F401
 except ImportError:
     pass
 
-CONTAINER_ENGINE = os.getenv("OPENLANE_CONTAINER_ENGINE", "docker")
-
 
 class StringRepresentable(object):
     def __str__(self):
@@ -44,6 +43,7 @@ class StringRepresentable(object):
 
 
 class ContainerInfo(StringRepresentable):
+    path = None  # type: Optional[str]
     engine = "UNKNOWN"  # type: str
     version = "UNKNOWN"  # type: str
     conmon = False  # type: bool
@@ -54,63 +54,84 @@ class ContainerInfo(StringRepresentable):
         self.version = "UNKNOWN"
         self.conmon = False
         self.rootless = False
+        self.seccomp = False
+        self.selinux = False
+        self.apparmor = False
 
     @staticmethod
     def get():
-        # type: () -> Optional['ContainerInfo']
+        # type: () -> Union[ContainerInfo, str]
+        cinfo = ContainerInfo()
+        # Here are the rules:
+        # 1. If LIBRELANE_CONTAINER_ENGINE exists, use that uncritically.
+        # 2. Else, if OPENLANE_CONTAINER_ENGINE exists, use that uncritically.
+        # 3. Else, if "docker" is in PATH, always use it.
+        # 4. Else, see if "podman" is in PATH, and use THAT.
+        # 5. If none exist, halt and return early.
+
+        container_engine = os.getenv(
+            "LIBRELANE_CONTAINER_ENGINE", os.getenv("OPENLANE_CONTAINER_ENGINE")
+        )
+        if container_engine is None or container_engine == "":
+            container_engine = shutil.which("docker")
+            if container_engine is None:
+                container_engine = shutil.which("podman")
+                if container_engine is None:
+                    return "no compatible container engine found in PATH (tried docker, podman)"
         try:
-            cinfo = ContainerInfo()
+            info_str = subprocess.check_output(
+                [container_engine, "info", "--format", "{{json .}}"]
+            ).decode("utf8")
+        except Exception as e:
+            return "failed to get container engine info: %s" % str(e)
+        cinfo.path = container_engine
 
+        try:
+            info = json.loads(info_str)
+        except Exception as e:
+            return "result from '%s info' was not valid JSON: %s" % (
+                container_engine,
+                str(e),
+            )
+
+        if (
+            info.get("Docker Root Dir") is not None
+            or info.get("DockerRootDir") is not None
+        ):
+            cinfo.engine = "docker"
+
+            # Get Version
             try:
-                info_str = subprocess.check_output(
-                    [CONTAINER_ENGINE, "info", "--format", "{{json .}}"]
-                ).decode("utf8")
-            except Exception as e:
-                raise Exception("Failed to get Docker info: %s" % str(e)) from None
+                version_output = (
+                    subprocess.check_output([container_engine, "--version"])
+                    .decode("utf8")
+                    .strip()
+                )
+                cinfo.version = re.split(r"\s", version_output)[2].strip(",")
+            except Exception:
+                pass
 
-            try:
-                info = json.loads(info_str)
-            except Exception as e:
-                raise Exception(
-                    "Result from 'docker info' was not valid JSON: %s" % str(e)
-                ) from None
-
-            if info.get("host") is not None:
-                if info["host"].get("conmon") is not None:
-                    cinfo.conmon = True
-                    if (
-                        info["host"].get("remoteSocket") is not None
-                        and "podman" in info["host"]["remoteSocket"]["path"]
-                    ):
-                        cinfo.engine = "podman"
-
-                        cinfo.version = info["version"]["Version"]
-            elif (
-                info.get("Docker Root Dir") is not None
-                or info.get("DockerRootDir") is not None
-            ):
-                cinfo.engine = "docker"
-
-                # Get Version
-                try:
-                    version_output = (
-                        subprocess.check_output([CONTAINER_ENGINE, "--version"])
-                        .decode("utf8")
-                        .strip()
-                    )
-                    cinfo.version = re.split(r"\s", version_output)[2].strip(",")
-                except Exception:
-                    print("Could not extract Docker version.", file=sys.stderr)
-
-                security_options = info.get("SecurityOptions")
-                for option in security_options:
-                    if "rootless" in option:
-                        cinfo.rootless = True
-
-            return cinfo
-        except Exception as e:
-            print(e, file=sys.stderr)
-            return None
+            security_options = info.get("SecurityOptions")
+            for option in security_options:
+                if "rootless" in option:
+                    cinfo.rootless = True
+        elif info.get("host") is not None:
+            host = info["host"]
+            conmon = host.get("conmon")
+            remote_socket = host.get("remoteSocket")
+            security = host.get("security")
+            if conmon is not None:
+                cinfo.conmon = True
+            if remote_socket is not None and "podman" in remote_socket["path"]:
+                cinfo.engine = "podman"
+                cinfo.version = info["version"]["Version"]
+            if security is not None:
+                cinfo.rootless = security.get("rootless", False)
+                cinfo.apparmor = security.get("apparmorEnabled", False)
+                cinfo.seccomp = security.get("seccompEnabled", False)
+                cinfo.selinux = security.get("selinuxEnabled", False)
+
+        return cinfo
 
 
 class NixInfo(StringRepresentable):
@@ -127,73 +148,65 @@ class NixInfo(StringRepresentable):
 
     @staticmethod
     def get():
-        # type: () -> Optional['NixInfo']
+        # type: () -> Union[NixInfo, str]
         ninfo = NixInfo()
+        if shutil.which("nix") is None:
+            return "nix not found in PATH"
         try:
-            try:
-                version_str = subprocess.check_output(
-                    ["nix", "--version"], encoding="utf8"
-                )
-                ninfo.version_string = version_str.strip()
-            except Exception as e:
-                raise Exception("Failed to get Nix info: %s" % str(e)) from None
+            version_str = subprocess.check_output(["nix", "--version"], encoding="utf8")
+            ninfo.version_string = version_str.strip()
+        except Exception as e:
+            return "could not get nix version: %s" % str(e)
 
-            try:
-                channels = {}
-                channels_raw = subprocess.check_output(
-                    ["nix-channel", "--list"], encoding="utf8"
-                )
-                for channel in channels_raw.splitlines():
-                    name, url = channel.split(maxsplit=1)
-                    channels[name] = url
-                ninfo.channels = channels
-            except Exception as e:
+        try:
+            channels = {}
+            channels_raw = subprocess.check_output(
+                ["nix-channel", "--list"], encoding="utf8"
+            )
+            for channel in channels_raw.splitlines():
+                name, url = channel.split(maxsplit=1)
+                channels[name] = url
+            ninfo.channels = channels
+        except Exception:
+            pass
+
+        with tempfile.TemporaryDirectory(prefix="librelane_env_report_") as d:
+            with open(os.path.join(d, "flake.nix"), "w") as f:
+                f.write("{}")
+            nix_command = subprocess.run(
+                ["nix", "eval"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                cwd=d,
+                encoding="utf8",
+            )
+            nix_command_result = nix_command.stdout
+            if "'nix-command'" in nix_command_result:
+                pass
+            elif "'flakes'" in nix_command_result:
+                ninfo.nix_command = True
+            elif "lacks attribute" in nix_command_result:
+                ninfo.nix_command = True
+                ninfo.flakes = True
+            else:
                 print(
-                    "Failed to get nix channels: %s" % str(e),
+                    "'nix flake' returned unexpected output: %s" % nix_command_result,
                     file=sys.stderr,
                 )
 
-            with tempfile.TemporaryDirectory(prefix="librelane_env_report_") as d:
-                with open(os.path.join(d, "flake.nix"), "w") as f:
-                    f.write("{}")
-                nix_command = subprocess.run(
-                    ["nix", "eval"],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.STDOUT,
-                    cwd=d,
-                    encoding="utf8",
-                )
-                nix_command_result = nix_command.stdout
-                if "'nix-command'" in nix_command_result:
-                    pass
-                elif "'flakes'" in nix_command_result:
-                    ninfo.nix_command = True
-                elif "lacks attribute" in nix_command_result:
-                    ninfo.nix_command = True
-                    ninfo.flakes = True
-                else:
-                    print(
-                        "'nix flake' returned unexpected output: %s"
-                        % nix_command_result,
-                        file=sys.stderr,
-                    )
-
-            return ninfo
-        except Exception as e:
-            print(e, file=sys.stderr)
-            return None
+        return ninfo
 
 
 class OSInfo(StringRepresentable):
     kernel = ""  # type: str
     kernel_version = ""  # type: str
     supported = False  # type: bool
-    distro = None  # type: Optional[str]
-    distro_version = None  # type: Optional[str]
+    distro = "UNKNOWN"  # type: str
+    distro_version = "UNKNOWN"  # type: str
     python_version = ""  # type: str
     python_path = []  # type: List[str]
-    container_info = None  # type: Optional[ContainerInfo]
-    nix_info = None  # type: Optional[NixInfo]
+    container_info = None  # type: Union[ContainerInfo, str]
+    nix_info = None  # type: Union[NixInfo, str]
 
     def __init__(self):
         self.kernel = platform.system()
@@ -201,8 +214,8 @@ class OSInfo(StringRepresentable):
             platform.release()
         )  # Unintuitively enough, it's the kernel's release
         self.supported = self.kernel in ["Darwin", "Linux"]
-        self.distro = None
-        self.distro_version = None
+        self.distro = "UNKNOWN"
+        self.distro_version = "UNKNOWN"
         self.python_version = platform.python_version()
         self.python_path = sys.path.copy()
         self.tkinter = False
@@ -212,8 +225,8 @@ class OSInfo(StringRepresentable):
             self.tkinter = True
         except ImportError:
             pass
-        self.container_info = None
-        self.nix_info = None
+        self.container_info = ""
+        self.nix_info = ""
 
     @staticmethod
     def get():
@@ -253,13 +266,14 @@ class OSInfo(StringRepresentable):
 
                     config[key] = value
 
-                osinfo.distro = config.get("ID") or config.get("DISTRIB_ID")
-                osinfo.distro_version = config.get("VERSION_ID") or config.get(
-                    "DISTRIB_RELEASE"
+                osinfo.distro = (
+                    config.get("ID") or config.get("DISTRIB_ID") or "UNKNOWN"
+                )
+                osinfo.distro_version = (
+                    config.get("VERSION_ID")
+                    or config.get("DISTRIB_RELEASE")
+                    or "UNKNOWN"
                 )
-
-            else:
-                print("Failed to get distribution info.", file=sys.stderr)
 
         osinfo.container_info = ContainerInfo.get()
         osinfo.nix_info = NixInfo.get()