ciel 0.21.0.dev0__py3-none-any.whl

ciel/build/__init__.py

@@ -0,0 +1,267 @@
+# Copyright 2022-2023 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import uuid
+import pathlib
+import tarfile
+import tempfile
+import importlib
+import subprocess
+from typing import Optional, List, Dict
+
+import click
+import zstandard as zstd
+from rich.console import Console
+from rich.progress import Progress
+
+from ..github import (
+    GitHubSession,
+    get_commit_date,
+    volare_repo,
+)
+from ..common import (
+    Version,
+    mkdirp,
+    resolve_version,
+    date_to_iso8601,
+)
+from ..click_common import (
+    opt_push,
+    opt_build,
+    opt_pdk_root,
+    opt_token,
+)
+from ..families import Family
+
+
+def build(
+    pdk_root: str,
+    pdk: str,
+    version: str,
+    jobs: int = 1,
+    sram: bool = True,  # Deprecated
+    clear_build_artifacts: bool = True,
+    include_libraries: Optional[List[str]] = None,
+    use_repo_at: Optional[List[str]] = None,
+):
+    use_repos = {}
+    if use_repo_at is not None:
+        for repo in use_repo_at:
+            name, path = repo.split("=")
+            use_repos[name] = os.path.abspath(path)
+
+    if pdk not in Family.by_name:
+        raise Exception(f"Unsupported PDK family '{pdk}'.")
+
+    kwargs = {
+        "pdk_root": pdk_root,
+        "version": version,
+        "jobs": jobs,
+        "clear_build_artifacts": clear_build_artifacts,
+        "include_libraries": include_libraries,
+        "using_repos": use_repos,
+    }
+
+    build_module = importlib.import_module(f".{pdk}", package=__name__)
+    build_function = build_module.build
+    build_function(**kwargs)
+
+
+@click.command("build")
+@opt_token
+@opt_pdk_root
+@opt_build
+@click.option(
+    "-f",
+    "--metadata-file",
+    "tool_metadata_file_path",
+    default=None,
+    help="Explicitly define a tool metadata file instead of searching for a metadata file",
+)
+@click.argument("version", required=False)
+def build_cmd(
+    include_libraries,
+    jobs,
+    pdk_root,
+    pdk,
+    clear_build_artifacts,
+    tool_metadata_file_path,
+    version,
+    use_repo_at,
+):
+    """
+    Builds the requested PDK.
+
+    Parameters: <version> (Optional)
+
+    If a version is not given, and you run this in the top level directory of
+    tools with a tool_metadata.yml file, for example OpenLane or DFFRAM,
+    the appropriate version will be enabled automatically.
+    """
+    if include_libraries == ():
+        include_libraries = None
+
+    console = Console()
+    try:
+        version = resolve_version(version, tool_metadata_file_path)
+    except Exception as e:
+        console.print(f"Could not determine open_pdks version: {e}")
+        exit(-1)
+
+    build(
+        pdk_root=pdk_root,
+        pdk=pdk,
+        version=version,
+        jobs=jobs,
+        clear_build_artifacts=clear_build_artifacts,
+        include_libraries=include_libraries,
+        use_repo_at=use_repo_at,
+    )
+
+
+def push(
+    pdk_root,
+    pdk,
+    version,
+    *,
+    owner=volare_repo.owner,
+    repository=volare_repo.name,
+    pre=False,
+    push_libraries=None,
+    session: Optional[GitHubSession] = None,
+):
+    family = Family.by_name[pdk]
+
+    if session is None:
+        session = GitHubSession()
+    if session.github_token is None:
+        raise TypeError("No GitHub token was provided.")
+
+    console = Console()
+
+    if push_libraries is None or len(push_libraries) == 0:
+        push_libraries = family.all_libraries
+    library_list = set(push_libraries)
+
+    version_object = Version(version, pdk)
+    version_directory = version_object.get_dir(pdk_root)
+    if not os.path.isdir(version_directory):
+        raise FileNotFoundError(f"Version {version} not found.")
+
+    tempdir = tempfile.gettempdir()
+    tarball_directory = os.path.join(tempdir, "cielo", f"{uuid.uuid4()}", version)
+    mkdirp(tarball_directory)
+
+    final_tarballs = []
+
+    with Progress() as progress:
+        collections: Dict[str, List[str]] = {"common": []}
+        path_it = pathlib.Path(version_directory).glob("**/*")
+        for path in path_it:
+            if not path.is_file():
+                continue
+            relative = os.path.relpath(path, version_directory)
+            path_components = relative.split(os.sep)
+            if path_components[1] == "libs.ref":
+                lib = path_components[2]
+                if lib not in library_list:
+                    continue
+                collections[lib] = collections.get(lib) or []
+                collections[lib].append(str(path))
+            else:
+                collections["common"].append(str(path))
+
+        for name, files in collections.items():
+            tarball_path = os.path.join(tarball_directory, f"{name}.tar.zst")
+            task = progress.add_task(f"Compressing {name}…", total=len(files))
+            with zstd.open(tarball_path, mode="wb") as stream:
+                with tarfile.TarFile(fileobj=stream, mode="w") as tf:
+                    for i, file in enumerate(files):
+                        progress.update(task, completed=i + 1)
+                        path_in_tarball = os.path.relpath(file, version_directory)
+                        tf.add(file, arcname=path_in_tarball)
+            console.log(f"\nCompressed to {tarball_path}.")
+            progress.remove_task(task)
+            final_tarballs.append(tarball_path)
+
+    tag = f"{pdk}-{version}"
+
+    # If someone wants to rewrite this to not use ghr, please, by all means.
+    console.log("Starting upload…")
+
+    body = f"{pdk} variants built using cielo"
+    date = get_commit_date(version, family.repo, session)
+    if date is not None:
+        body = f"{pdk} variants (released on {date_to_iso8601(date)})"
+
+    for tarball_path in final_tarballs:
+        subprocess.check_call(
+            [
+                "ghr",
+                "-owner",
+                owner,
+                "-repository",
+                repository,
+                "-token",
+                session.github_token,
+                "-body",
+                body,
+                "-commitish",
+                "releases",
+                "-replace",
+            ]
+            + (["-prerelease"] if pre else [])
+            + [
+                tag,
+                tarball_path,
+            ]
+        )
+    console.log("Done.")
+
+
+@click.command("push", hidden=True)
+@opt_token
+@opt_pdk_root
+@opt_push
+@click.argument("version")
+def push_cmd(
+    owner,
+    repository,
+    pre,
+    pdk_root,
+    pdk,
+    version,
+    push_libraries,
+):
+    """
+    For maintainers: Package and release a build to the public.
+
+    Requires ghr: github.com/tcnksm/ghr
+
+    Parameters: <version> (required)
+    """
+    console = Console()
+    try:
+        push(
+            pdk_root,
+            pdk,
+            version,
+            owner=owner,
+            repository=repository,
+            pre=pre,
+            push_libraries=push_libraries,
+        )
+    except Exception as e:
+        console.print(f"[red]Failed to push version: {e}")
+        exit(-1)

ciel/build/common.py

@@ -0,0 +1,98 @@
+# Copyright 2022-2023 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import shutil
+import subprocess
+
+from cielo.github import GitHubSession
+
+
+def open_pdks_fix_makefile(at_path: str):
+    backup_path = f"{at_path}.bak"
+    shutil.move(at_path, backup_path)
+
+    fix_fi = False
+
+    with open(backup_path, "r") as file_in, open(at_path, "w") as file_out:
+        for line in file_in:
+            if "_COMMIT = `" in line:
+                line = line.replace("_COMMIT = ", "_COMMIT=")
+            if fix_fi:
+                file_out.write(line.replace("fi", "fi ; \\"))
+                fix_fi = False
+            else:
+                file_out.write(line)
+            if "_COMMIT=`" in line:
+                fix_fi = True
+
+
+def patch_open_pdks(at_path: str):
+    """
+    This functions applies various patches based on the current version of
+    open_pdks in use.
+    """
+    head = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=at_path, encoding="utf8"
+    ).strip()
+
+    def is_ancestor(commit: str):
+        nonlocal head, at_path
+        return (
+            subprocess.call(
+                ["git", "merge-base", "--is-ancestor", commit, head],
+                stdout=open(os.devnull, "w"),
+                stderr=open(os.devnull, "w"),
+                cwd=at_path,
+            )
+            == 0
+        )
+
+    can_build = is_ancestor(
+        "c74daac794c83327e54b91cbaf426f722574665c"
+    )  # First one with --with-reference
+    if not can_build:
+        print(
+            f"Commit {head} cannot be built using Cielo: the minimum version of open_pdks buildable with Cielo is 1.0.381."
+        )
+        exit(-1)
+
+    gf180mcu_sources_ok = is_ancestor(
+        "c1e2118846fd216b2c065a216950e75d2d67ccb8"
+    )  # gf180mcu sources fix
+    if not gf180mcu_sources_ok:
+        print(
+            "Patching gf180mcu Makefile.in…",
+        )
+        open_pdks_fix_makefile(os.path.join(at_path, "gf180mcu", "Makefile.in"))
+
+    download_script_ok = is_ancestor(
+        "ebffedd16788db327af050ac01c3fb1558ebffd1"
+    )  # download script fix
+    if not download_script_ok:
+        print("Replacing download.sh…")
+        session = GitHubSession()
+        r = session.get(
+            "https://raw.githubusercontent.com/RTimothyEdwards/open_pdks/ebffedd16788db327af050ac01c3fb1558ebffd1/scripts/download.sh"
+        )
+        with open(os.path.join(at_path, "scripts", "download.sh"), "wb") as f:
+            f.write(r.content)
+
+    sky130_sources_ok = is_ancestor(
+        "274040274a7dfb5fd2c69e0e9c643f80507df3fe"
+    )  # sky130 sources fix
+    if not sky130_sources_ok:
+        print(
+            "Patching sky130 Makefile.in…",
+        )
+        open_pdks_fix_makefile(os.path.join(at_path, "sky130", "Makefile.in"))

ciel/build/gf180mcu.py

@@ -0,0 +1,263 @@
+# Copyright 2022-2023 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import io
+import json
+import shlex
+import shutil
+import subprocess
+from datetime import datetime
+from typing import Optional, List, Tuple, Dict
+from concurrent.futures import ThreadPoolExecutor
+
+import pcpp
+from rich.console import Console
+from rich.progress import Progress
+
+from .git_multi_clone import GitMultiClone
+from .common import patch_open_pdks
+from ..families import Family
+from ..github import opdks_repo
+from ..common import (
+    Version,
+    get_cielo_dir,
+    mkdirp,
+)
+
+MAGIC_DEFAULT_TAG = "085131b090cb511d785baf52a10cf6df8a657d44"
+
+
+def get_open_pdks(
+    version, build_directory, jobs=1, repo_path=None
+) -> Tuple[str, Optional[str], Optional[str]]:
+    try:
+        console = Console()
+
+        open_pdks_repo = None
+        if repo_path is None:
+            with Progress() as progress:
+                with ThreadPoolExecutor(max_workers=jobs) as executor:
+                    gmc = GitMultiClone(build_directory, progress)
+                    open_pdks_future = executor.submit(
+                        GitMultiClone.clone,
+                        gmc,
+                        opdks_repo.link,
+                        version,
+                        default_branch="master",
+                    )
+                    open_pdks_repo = open_pdks_future.result()
+                    repo_path = open_pdks_repo.path
+
+            console.log(f"Done fetching {open_pdks_repo.name}.")
+        else:
+            console.log(f"Using open_pdks at {repo_path} unaltered.")
+
+        patch_open_pdks(repo_path)
+
+        try:
+            json_raw = open(f"{repo_path}/gf180mcu/gf180mcu.json").read()
+            cpp = pcpp.Preprocessor()
+            cpp.line_directive = None
+            cpp.parse(json_raw)
+            json_str = None
+            with io.StringIO() as sio:
+                cpp.write(sio)
+                json_str = sio.getvalue()
+            manifest = json.loads(json_str)
+            reference_commits = manifest["reference"]
+            print(f"Reference commits: {reference_commits}")
+        except FileNotFoundError:
+            console.log("Warning: Couldn't find open_pdks/sky130 JSON manifest.")
+        except json.JSONDecodeError:
+            console.log("Warning: Failed to parse open_pdks/sky130 JSON manifest..")
+        except KeyError:
+            console.log(
+                "Warning: Failed to extract reference commits from open_pdks/sky130 JSON manifest."
+            )
+
+        return repo_path
+
+    except subprocess.CalledProcessError as e:
+        print(e)
+        print(e.stderr)
+        exit(-1)
+
+
+LIB_FLAG_MAP = {
+    "gf180mcu_fd_pr": "--enable-primitive-gf180mcu",
+    "gf180mcu_fd_sc_mcu7t5v0": "--enable-sc-7t5v0-gf180mcu",
+    "gf180mcu_fd_sc_mcu9t5v0": "--enable-sc-9t5v0-gf180mcu",
+    "gf180mcu_fd_io": "--enable-io-gf180mcu",
+    "gf180mcu_fd_ip_sram": "--enable-sram-gf180mcu",
+    "gf180mcu_osu_sc_gp12t3v3": "--enable-osu-sc-gf180mcu",
+    "gf180mcu_osu_sc_gp9t3v3": "--enable-osu-sc-gf180mcu",
+}
+
+
+def build_variants(
+    magic_bin, include_libraries, build_directory, open_pdks_path, log_dir, jobs=1
+):
+    try:
+        pdk_root_abs = os.path.abspath(build_directory)
+        console = Console()
+
+        def run_sh(script, log_to):
+            output_file = open(log_to, "w")
+            try:
+                subprocess.check_call(
+                    ["sh", "-c", script],
+                    cwd=open_pdks_path,
+                    stdout=output_file,
+                    stderr=output_file,
+                    stdin=open(os.devnull),
+                )
+            except subprocess.CalledProcessError as e:
+                console.log(
+                    f"An error occurred while building the PDK. Check {log_to} for more information."
+                )
+                raise e
+
+        library_flags = set([LIB_FLAG_MAP[library] for library in include_libraries])
+        library_flags_disable = set(
+            [
+                LIB_FLAG_MAP[library].replace("enable", "disable")
+                for library in LIB_FLAG_MAP
+                if library not in include_libraries
+            ]
+        )
+        magic_dirname = os.path.dirname(magic_bin)
+
+        configuration_flags = ["--enable-gf180mcu-pdk", "--with-reference"] + list(
+            library_flags.union(library_flags_disable)
+        )
+        console.log(f"Configuring with flags {shlex.join(configuration_flags)}")
+
+        with console.status("Configuring open_pdks…"):
+            run_sh(
+                f"""
+                    set -e
+                    export PATH="{magic_dirname}:$PATH"
+                    ./configure {shlex.join(configuration_flags)}
+                """,
+                log_to=os.path.join(log_dir, "config.log"),
+            )
+        console.log("Configured open_pdks.")
+
+        with console.status("Building variants using open_pdks…"):
+            run_sh(
+                f"""
+                    set -e
+                    export LC_ALL=en_US.UTF-8
+                    export PATH="{magic_dirname}:$PATH"
+                    make -j{jobs}
+                    make 'SHARED_PDKS_PATH={pdk_root_abs}' install
+                """,
+                log_to=os.path.join(log_dir, "install.log"),
+            )
+        console.log("Built PDK variants.")
+        with console.status("Cleaning build artifacts…"):
+            run_sh(
+                """
+                set -e
+                rm -rf sources
+                """,
+                log_to=os.path.join(log_dir, "clean.log"),
+            )
+        console.log("Cleaned build artifacts.")
+
+        console.log("Done.")
+
+    except subprocess.CalledProcessError as e:
+        print(e)
+        print(e.stderr)
+        exit(-1)
+
+
+def install_gf180mcu(build_directory, pdk_root, version):
+    console = Console()
+    with console.status("Adding build to list of installed versions…"):
+        version_directory = Version(version, "gf180mcu").get_dir(pdk_root)
+        if (
+            os.path.exists(version_directory)
+            and len(os.listdir(version_directory)) != 0
+        ):
+            backup_path = version_directory
+            it = 0
+            while os.path.exists(backup_path) and len(os.listdir(backup_path)) != 0:
+                it += 1
+                backup_path = Version(f"{version}.bk{it}", "gf180mcu").get_dir(pdk_root)
+            console.log(
+                f"Build already found at {version_directory}, moving to {backup_path}…"
+            )
+            shutil.move(version_directory, backup_path)
+
+        console.log("Copying…")
+        mkdirp(version_directory)
+
+        gf180mcu_family = Family.by_name["gf180mcu"]
+
+        for variant in gf180mcu_family.variants:
+            variant_build_path = os.path.join(build_directory, variant)
+            variant_install_path = os.path.join(version_directory, variant)
+            if os.path.isdir(variant_build_path):
+                shutil.copytree(variant_build_path, variant_install_path)
+
+    console.log("Done.")
+
+
+def build(
+    pdk_root: str,
+    version: str,
+    jobs: int = 1,
+    clear_build_artifacts: bool = True,
+    include_libraries: Optional[List[str]] = None,
+    using_repos: Optional[Dict[str, str]] = None,
+):
+    family = Family.by_name["gf180mcu"]
+    library_set = family.resolve_libraries(include_libraries)
+
+    if using_repos is None:
+        using_repos = {}
+
+    timestamp = datetime.now().strftime("build_gf180mcu-%Y-%m-%d-%H-%M-%S")
+    build_directory = os.path.join(
+        get_cielo_dir(pdk_root, "gf180mcu"), "build", version
+    )
+    log_dir = os.path.join(build_directory, "logs", timestamp)
+    mkdirp(log_dir)
+
+    console = Console()
+    console.log(f"Logging to '{log_dir}'…")
+
+    open_pdks_path = get_open_pdks(
+        version, build_directory, jobs, using_repos.get("open_pdks")
+    )
+
+    magic_bin = shutil.which("magic")
+    if magic_bin is None:
+        print("Magic is either not installed or not in PATH.")
+        exit(-1)
+
+    build_variants(
+        magic_bin,
+        library_set,
+        build_directory,
+        open_pdks_path,
+        log_dir,
+        jobs,
+    )
+    install_gf180mcu(build_directory, pdk_root, version)
+
+    if clear_build_artifacts:
+        shutil.rmtree(build_directory)