librelane 2.4.0.dev11__py3-none-any.whl → 2.4.1__py3-none-any.whl

librelane/__main__.py

@@ -22,7 +22,6 @@ import shutil
 import marshal
 import tempfile
 import traceback
-import subprocess
 from textwrap import dedent
 from functools import partial
 from typing import Any, Dict, Sequence, Tuple, Type, Optional, List
@@ -278,20 +277,12 @@ def run_included_example(
         if os.path.isdir(final_path):
             print(f"A directory named {value} already exists.", file=sys.stderr)
             ctx.exit(1)
-        # 1. Copy the files
-        shutil.copytree(
-            example_path,
-            final_path,
-            symlinks=False,
-        )
-
-        # 2. Make files writable
-        if os.name == "posix":
-            subprocess.check_call(["chmod", "-R", "755", final_path])
 
+        # 1. Copy the files
+        common.recreate_tree(example_path, final_path)
         config_file = glob.glob(os.path.join(final_path, "config.*"))[0]
 
-        # 3. Run
+        # 2. Run
         run(
             ctx,
             config_files=[config_file],
@@ -321,28 +312,38 @@ def cli_in_container(
     if not value:
         return
 
-    docker_mounts = list(ctx.params.get("docker_mounts") or ())
-    docker_tty: bool = ctx.params.get("docker_tty", True)
+    mounts = list(ctx.params.get("docker_mounts") or ())
+    tty: bool = ctx.params.get("docker_tty", True)
     pdk_root = ctx.params.get("pdk_root")
-    argv = sys.argv[sys.argv.index("--dockerized") + 1 :]
+
+    try:
+        containerized_index = sys.argv.index("--dockerized")
+    except ValueError:
+        containerized_index = sys.argv.index("--containerized")
+
+    argv = sys.argv[containerized_index + 1 :]
 
     final_argv = ["zsh"]
     if len(argv) != 0:
-        final_argv = ["librelane"] + argv
+        final_argv = ["python3", "-m", "librelane"] + argv
 
-    docker_image = os.getenv(
+    container_image = os.getenv(
         "LIBRELANE_IMAGE_OVERRIDE", f"ghcr.io/librelane/librelane:{__version__}"
     )
 
     try:
         run_in_container(
-            docker_image,
+            container_image,
             final_argv,
             pdk_root=pdk_root,
-            other_mounts=docker_mounts,
-            tty=docker_tty,
+            other_mounts=mounts,
+            tty=tty,
         )
+    except ValueError as e:
+        err(e)
+        ctx.exit(1)
     except Exception as e:
+        traceback.print_exc(file=sys.stderr)
         err(e)
         ctx.exit(1)
 
@@ -377,25 +378,28 @@ o = partial(option, show_default=True)
     "Containerization options",
     o(
         "--docker-mount",
+        "--container-mount",
         "-m",
         "docker_mounts",
         multiple=True,
-        is_eager=True,  # docker options should be processed before anything else
+        is_eager=True,  # container options should be processed before anything else
         default=(),
-        help="Used to mount more directories in dockerized mode. If a valid directory is specified, it will be mounted in the same path in the container. Otherwise, the value of the option will be passed to the Docker-compatible container engine verbatim. Must be passed before --dockerized, has no effect if --dockerized is not set.",
+        help="Used to mount more directories in dockerized mode. If a valid directory is specified, it will be mounted in the same path in the container. Otherwise, the value of the option will be passed to the container engine verbatim. Must be passed before --containerized/--dockerized, has no effect if not set.",
     ),
     o(
         "--docker-tty/--docker-no-tty",
-        is_eager=True,  # docker options should be processed before anything else
+        "--container-tty/--container-no-tty",
+        is_eager=True,  # container options should be processed before anything else
         default=True,
-        help="Controls the allocation of a virtual terminal by passing -t to the Docker-compatible container engine invocation. Must be passed before --dockerized, has no effect if --dockerized is not set.",
+        help="Controls the allocation of a virtual terminal by passing -t to the Docker-compatible container engine invocation. Must be passed before --containerized/--dockerized, has no effect if not set.",
     ),
     o(
         "--dockerized",
+        "--containerized",
         default=False,
         is_flag=True,
         is_eager=True,  # docker options should be processed before anything else
-        help="Run the remaining flags using a Docker container. Some caveats apply. Must precede all options except --docker-mount, --docker-tty/--docker-no-tty.",
+        help="Run the remaining flags using a containerized version of LibreLane. Some caveats apply. Must precede all options except --{docker,container}-mount, --{docker,container}-[no-]tty.",
         callback=cli_in_container,
     ),
 )

librelane/common/__init__.py

@@ -41,8 +41,10 @@ from .misc import (
     format_size,
     format_elapsed_time,
     Filter,
+    recreate_tree,
     get_latest_file,
     process_list_file,
+    count_occurences,
     _get_process_limit,
 )
 from .types import (

librelane/common/drc.py

@@ -208,6 +208,7 @@ class DRC:
         from lxml import etree as ET
 
         with ET.xmlfile(out, encoding="utf8", buffered=False) as xf:
+            xf.write_declaration()
             with xf.element("report-database"):
                 # 1. Cells
                 with xf.element("cells"):

librelane/common/misc.py

@@ -11,16 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import io
 import os
 import re
 import glob
 import gzip
+import shutil
 import typing
 import fnmatch
 import pathlib
 import unicodedata
 from math import inf
 from typing import (
+    IO,
     Any,
     Generator,
     Iterable,
@@ -314,6 +317,37 @@ class Filter(object):
                 yield input
 
 
+def recreate_tree(
+    source: AnyPath,
+    target: AnyPath,
+):
+    """
+    This function attempts to recreate a file tree from a source path in another
+    target path.
+
+    Permissions are not copied over. Symlinks and hardlinks are followed.
+
+    Directories are not recreated unless they contain files as (grand)children.
+
+    If the source and target are the same, the function returns early and does
+    nothing.
+
+    :param source: The source file tree to replicate
+    :param target: The target path to recreate the file tree within
+    """
+    source = os.path.abspath(source)
+    target = os.path.abspath(target)
+    if os.path.exists(target) and os.path.samefile(source, target):
+        return
+    for dirname, _, files in os.walk(source):
+        for file in files:
+            resolved = os.path.join(dirname, file)
+            resolved_target = os.path.join(target, os.path.relpath(resolved, source))
+            os.makedirs(os.path.dirname(resolved_target), exist_ok=True)
+            with open(resolved, "rb") as fi, open(resolved_target, "wb") as fo:
+                shutil.copyfileobj(fi, fo)
+
+
 def get_latest_file(in_path: Union[str, os.PathLike], filename: str) -> Optional[Path]:
     """
     :param in_path: A directory to search in
@@ -378,9 +412,9 @@ def _get_process_limit() -> int:
     return int(os.getenv("_OPENLANE_MAX_CORES", os.cpu_count() or 1))
 
 
-def gzopen(filename, mode="rt"):
+def gzopen(filename: AnyPath, mode="rt") -> IO[Any]:
     """
-    This method (tries to?) emulate the gzopen from the Linux Standard Base,
+    This function (tries to?) emulate the gzopen from the Linux Standard Base,
     specifically this part:
 
     If path refers to an uncompressed file, and mode refers to a read mode,
@@ -388,6 +422,11 @@ def gzopen(filename, mode="rt"):
     for reading directly from the file without any decompression.
 
     gzip.open does not have this behavior.
+
+    :param filename: The full path to the uncompressed or gzipped file.
+    :param mode: "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+        binary mode, or "rt", "wt", "xt" or "at" for text mode.
+    :returns: An I/O wrapper that may very slightly based on the mode.
     """
     try:
         g = gzip.open(filename, mode=mode)
@@ -400,3 +439,18 @@ def gzopen(filename, mode="rt"):
     except gzip.BadGzipFile:
         g.close()
         return open(filename, mode=mode)
+
+
+def count_occurences(fp: io.TextIOWrapper, pattern: str = "") -> int:
+    """
+    Counts the occurences of a certain string in a stream, line-by-line, without
+    necessarily loading the entire file into memory.
+
+    Equivalent to: ``grep -c 'pattern' <file>`` (but without regex support).
+
+    :param fp: the text stream
+    :param pattern: the substring to search for. if set to "", it will simply
+        count the lines in the file.
+    :returns: the number of matching lines
+    """
+    return sum(pattern in line for line in fp)

librelane/config/__main__.py

@@ -14,7 +14,6 @@
 import os
 import sys
 import json
-import functools
 from decimal import Decimal
 
 import click
@@ -111,9 +110,7 @@ def create_config(
                 print("At least one source RTL file is required.", file=sys.stderr)
                 exit(1)
     source_rtl_key = "VERILOG_FILES"
-    if not functools.reduce(
-        lambda acc, x: acc and (x.endswith(".sv") or x.endswith(".v")), source_rtl, True
-    ):
+    if not all(file.endswith(".sv") or file.endswith(".v") for file in source_rtl):
         print(
             "Only Verilog/SystemVerilog files are supported by create-config.",
             file=sys.stderr,

librelane/container.py

@@ -18,7 +18,6 @@ import re
 import uuid
 import shlex
 import pathlib
-import tempfile
 import subprocess
 from typing import List, NoReturn, Sequence, Optional, Union, Tuple
 
@@ -27,15 +26,15 @@ import semver
 
 from .common import mkdirp
 from .logging import err, info, warn
-from .env_info import OSInfo
+from .env_info import ContainerInfo, OSInfo
 
-CONTAINER_ENGINE = os.getenv("OPENLANE_CONTAINER_ENGINE", "docker")
+__file_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 
 def permission_args(osinfo: OSInfo) -> List[str]:
     if (
         osinfo.kernel == "Linux"
-        and osinfo.container_info is not None
+        and isinstance(osinfo.container_info, ContainerInfo)
         and osinfo.container_info.engine == "docker"
         and not osinfo.container_info.rootless
     ):
@@ -70,9 +69,9 @@ def gui_args(osinfo: OSInfo) -> List[str]:
     return args
 
 
-def image_exists(image: str) -> bool:
+def image_exists(ce_path: str, image: str) -> bool:
     images = (
-        subprocess.check_output([CONTAINER_ENGINE, "images", image])
+        subprocess.check_output([ce_path, "images", image])
         .decode("utf8")
         .rstrip()
         .split("\n")[1:]
@@ -116,14 +115,14 @@ def remote_manifest_exists(image: str) -> bool:
     return True
 
 
-def ensure_image(image: str) -> bool:
-    if image_exists(image):
+def ensure_image(ce_path: str, image: str) -> bool:
+    if image_exists(ce_path, image):
         return True
 
     try:
-        subprocess.check_call([CONTAINER_ENGINE, "pull", image])
+        subprocess.check_call([ce_path, "pull", image])
     except subprocess.CalledProcessError:
-        err(f"Failed to pull image {image} from the container registries.")
+        err(f"Failed to pull image '{image}' from the container registries.")
         return False
 
     return True
@@ -150,6 +149,22 @@ def sanitize_path(path: Union[str, os.PathLike]) -> Tuple[str, str]:
     return (abspath, mountable_path)
 
 
+def container_version_error(input: str, against: str) -> Optional[str]:
+    if input == "UNKNOWN":
+        return (
+            "Could not determine version for %s. You may encounter unexpected issues."
+        )
+    if semver.compare(input, against) < 0:
+        return f"Your %s version ({input}) is out of date. You may encounter unexpected issues."
+    return None
+
+
+def ubuntu_version_at_least(current: str, minimum: str) -> bool:
+    if current == "UNKNOWN":
+        return False
+    return tuple(map(int, current.split("."))) >= tuple(map(int, minimum.split(".")))
+
+
 def run_in_container(
     image: str,
     args: Sequence[str],
@@ -169,20 +184,31 @@ def run_in_container(
             f"Unsupported host operating system '{osinfo.kernel}'. You may encounter unexpected issues."
         )
 
-    if osinfo.container_info is None:
+    if not isinstance(osinfo.container_info, ContainerInfo):
         raise FileNotFoundError("No compatible container engine found.")
 
-    if osinfo.container_info.engine.lower() == "docker":
-        if semver.compare(osinfo.container_info.version, "25.0.5") < 0:
+    ce_path = osinfo.container_info.path
+    assert ce_path is not None
+
+    engine_name = osinfo.container_info.engine.lower()
+    if engine_name == "docker":
+        if error := container_version_error(osinfo.container_info.version, "25.0.5"):
+            warn(error % engine_name)
+    elif engine_name == "podman":
+        if osinfo.distro.lower() == "ubuntu" and not ubuntu_version_at_least(
+            osinfo.distro_version, "24.04"
+        ):
             warn(
-                f"Your Docker engine version ({osinfo.container_info.version}) is out of date. You may encounter unexpected issues."
+                "Versions of Podman for Ubuntu before Ubuntu 24.04 are generally pretty buggy. We recommend using Docker instead if possible."
             )
+        elif error := container_version_error(osinfo.container_info.version, "4.1.0"):
+            warn(error % engine_name)
     else:
         warn(
-            f"Unsupported container engine '{osinfo.container_info.engine}'. You may encounter unexpected issues."
+            f"Unsupported container engine referenced by '{osinfo.container_info.path}'. You may encounter unexpected issues."
         )
 
-    if not ensure_image(image):
+    if not ensure_image(ce_path, image):
         raise ValueError(f"Failed to use image '{image}'.")
 
     terminal_args = ["-i"]
@@ -222,15 +248,6 @@ def run_in_container(
         mount_args += ["-v", f"{from_cwd}:{to_cwd}"]
     mount_args += ["-w", to_cwd]
 
-    tempdir = tempfile.mkdtemp("librelane_docker")
-
-    mount_args += [
-        "-v",
-        f"{tempdir}:/tmp",
-        "-e",
-        "TMPDIR=/tmp",
-    ]
-
     if other_mounts is not None:
         for mount in other_mounts:
             if os.path.isdir(mount):
@@ -242,9 +259,13 @@ def run_in_container(
 
     container_id = str(uuid.uuid4())
 
+    if os.getenv("_MOUNT_HOST_LIBRELANE") == "1":
+        host_librelane_pythonpath = os.path.dirname(__file_dir__)
+        mount_args += ["-v", f"{host_librelane_pythonpath}:/host_librelane"]
+
     cmd = (
         [
-            CONTAINER_ENGINE,
+            ce_path,
             "run",
             "--rm",
             "--name",
@@ -261,4 +282,4 @@ def run_in_container(
     info("Running containerized command:")
     print(shlex.join(cmd))
 
-    os.execlp(CONTAINER_ENGINE, *cmd)
+    os.execlp(ce_path, *cmd)