ciel 0.21.0.dev0__py3-none-any.whl

ciel/build/git_multi_clone.py

@@ -0,0 +1,220 @@
+# Copyright 2022 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import re
+import shutil
+import subprocess
+from typing import Optional
+
+from rich.progress import Progress
+
+from ..common import mkdirp
+
+
+class Repository(object):
+    @classmethod
+    def from_path(Self, path):
+        name = os.path.basename(path)
+        url = subprocess.check_output(
+            ["git", "remote", "get-url", "origin"], stderr=subprocess.PIPE
+        ).strip()
+
+        remote_branch_info = open(
+            os.path.join(path, ".git", "refs", "remotes", "origin", "HEAD")
+        ).read()
+        remote_branch = os.path.basename(remote_branch_info)
+
+        return Self(name, url, path, remote_branch)
+
+    def __init__(self, name, url, path, default_branch="main"):
+        path = os.path.abspath(path)
+
+        self.name = name
+        self.url = url
+        self.path = path
+        self.default_branch = default_branch
+
+    def clone_if_not_exist(self, callback=None):
+        if os.path.exists(self.path):
+            self.pristine()
+            self.pull(callback)
+        else:
+            self.clone(callback)
+
+    def clone(self, callback=None):
+        try:
+            shutil.rmtree(self.path)
+        except FileNotFoundError:
+            pass
+
+        callback(0, f"Cloning {self.name} to {self.path}…")
+
+        process = subprocess.Popen(
+            ["git", "clone", "--progress", self.url, self.path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        assert process.stderr is not None
+
+        ro_rx = re.compile(r"Receiving objects:\s*(\d+)%")
+
+        # Python Moment
+        buffer = ""
+        while True:
+            bytes_read = process.stderr.read(1)
+            if len(bytes_read) == 0:
+                break
+            char_read = bytes_read.decode("utf8")
+            if char_read in ["\n", "\r"]:
+                match = ro_rx.search(buffer)
+                if match is not None:
+                    if callback is not None:
+                        callback(int(match[1]))
+                buffer = ""
+            else:
+                buffer += char_read
+
+        process.wait()
+
+    def pristine(self):
+        subprocess.check_output(
+            ["git", "clean", "-fdX"], cwd=self.path, stderr=subprocess.PIPE
+        )
+
+        subprocess.check_output(
+            ["git", "reset", "--hard", "HEAD"], cwd=self.path, stderr=subprocess.PIPE
+        )
+
+    def pull(self, callback=None):
+        subprocess.check_output(
+            ["git", "checkout", "-f", self.default_branch],
+            cwd=self.path,
+            stderr=subprocess.PIPE,
+        )
+        process = subprocess.Popen(
+            ["git", "pull", "--no-recurse-submodules", "--progress"],
+            cwd=self.path,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        assert process.stderr is not None
+        callback(0, f"Updating {self.name} at {self.path}…")
+
+        ro_rx = re.compile(r"Receiving objects:\s*(\d+)%")
+
+        # Python Moment #2
+        buffer = ""
+        while True:
+            bytes_read = process.stderr.read(1)
+            if len(bytes_read) == 0:
+                break
+            char_read = bytes_read.decode("utf8")
+            if char_read in ["\n", "\r"]:
+                match = ro_rx.search(buffer)
+                if match is not None:
+                    if callback is not None:
+                        callback(int(match[1]))
+                buffer = ""
+            else:
+                buffer += char_read
+
+        process.wait()
+        if callback is not None:
+            callback(100)
+
+    def checkout_commit(self, commit: str):
+        subprocess.check_output(
+            ["git", "checkout", "-f", self.default_branch],
+            cwd=self.path,
+            stderr=subprocess.PIPE,
+        )
+        try:
+            subprocess.check_output(
+                ["git", "branch", "-f", "-D", "current"],
+                cwd=self.path,
+                stderr=subprocess.PIPE,
+            )
+        except Exception:
+            pass
+        subprocess.check_output(
+            ["git", "checkout", "-f", "-b", "current", commit],
+            cwd=self.path,
+            stderr=subprocess.PIPE,
+        )
+
+    def init_submodule(self, submodule: Optional[str] = None, callback=None):
+        cmd = ["git", "submodule", "update", "--init", "--progress"]
+        if submodule is not None:
+            cmd.append(submodule)
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=self.path,
+        )
+
+        ro_rx = re.compile(r"Receiving objects:\s*(\d+)%")
+
+        assert process.stderr is not None, "Process doesn't have a stderr channel"
+
+        # Python Moment #3
+        buffer = ""
+        while True:
+            bytes_read = process.stderr.read(1)
+            if len(bytes_read) == 0:
+                break
+            char_read = bytes_read.decode("utf8")
+            if char_read in ["\n", "\r"]:
+                match = ro_rx.search(buffer)
+                if match is not None:
+                    if callback is not None:
+                        callback(int(match[1]))
+                buffer = ""
+            else:
+                buffer += char_read
+
+        process.wait()
+
+
+class GitMultiClone(object):
+    progress: Progress
+
+    def __init__(self, folder, progress):
+        super().__init__()
+        self.folder = os.path.abspath(folder)
+        mkdirp(self.folder)
+        self.progress = progress
+
+    def clone(
+        self, repo_url: str, commit: str, default_branch: str = "main"
+    ) -> Repository:
+        current_task = self.progress.add_task("", total=100)
+        name = os.path.basename(repo_url)
+        path = os.path.join(self.folder, name)
+        r = Repository(name, repo_url, path, default_branch=default_branch)
+        r.clone_if_not_exist(
+            lambda x, y=None: self.progress.update(
+                current_task, completed=x, description=y
+            )
+        )
+        r.checkout_commit(commit)
+        return r
+
+    def clone_submodule(self, repo: Repository, submodule: str):
+        current_task = self.progress.add_task(
+            f"Updating submodule {submodule}…", total=100
+        )
+        repo.init_submodule(
+            submodule, lambda x: self.progress.update(current_task, completed=x)
+        )

ciel/build/ihp_sg13g2.py

@@ -0,0 +1,148 @@
+# Copyright 2022-2023 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import shutil
+import subprocess
+from datetime import datetime
+from typing import Optional, List, Tuple, Dict
+from concurrent.futures import ThreadPoolExecutor
+
+from rich.console import Console
+from rich.progress import Progress
+
+from .git_multi_clone import GitMultiClone
+from ..families import Family
+from ..github import ihp_repo
+from ..common import (
+    Version,
+    get_cielo_dir,
+    mkdirp,
+)
+
+
+def get_ihp(
+    version, build_directory, jobs=1, repo_path=None
+) -> Tuple[str, Optional[str], Optional[str]]:
+    try:
+        console = Console()
+
+        if repo_path is None:
+            with Progress() as progress:
+                with ThreadPoolExecutor(max_workers=jobs) as executor:
+                    gmc = GitMultiClone(build_directory, progress)
+                    ihp_future = executor.submit(
+                        GitMultiClone.clone,
+                        gmc,
+                        ihp_repo.link,
+                        version,
+                    )
+                    repo = ihp_future.result()
+                    current_task = progress.add_task("Updating submodules…", total=100)
+                    repo.init_submodule(
+                        callback=lambda x: progress.update(current_task, completed=x)
+                    )
+                    repo_path = repo.path
+            console.log(f"Done fetching {ihp_repo.name}.")
+        else:
+            console.log(f"Using IHP-Open-PDK at {repo_path} unaltered.")
+
+        return repo_path
+
+    except subprocess.CalledProcessError as e:
+        print(e)
+        print(e.stderr)
+        exit(-1)
+
+
+def build_ihp(build_directory, ihp_path):
+    # """Build"""
+    try:
+        shutil.rmtree(os.path.join(build_directory, "ihp-sg13g2"))
+    except FileNotFoundError:
+        pass
+    shutil.copytree(
+        os.path.join(ihp_path, "ihp-sg13g2"),
+        os.path.join(build_directory, "ihp-sg13g2"),
+        ignore=lambda dir, files: (
+            files if ".git" in os.path.split(dir) else [".git", ".DS_Store"]
+        ),
+    )
+
+
+def install_ihp(build_directory, pdk_root, version):
+    console = Console()
+    with console.status("Adding build to list of installed versions…"):
+        ihp_sg13g2_family = Family.by_name["ihp_sg13g2"]
+
+        version_directory = Version(version, "ihp_sg13g2").get_dir(pdk_root)
+        if (
+            os.path.exists(version_directory)
+            and len(os.listdir(version_directory)) != 0
+        ):
+            backup_path = version_directory
+            it = 0
+            while os.path.exists(backup_path) and len(os.listdir(backup_path)) != 0:
+                it += 1
+                backup_path = Version(f"{version}.bk{it}", "ihp_sg13g2").get_dir(
+                    pdk_root
+                )
+            console.log(
+                f"Build already found at {version_directory}, moving to {backup_path}…"
+            )
+            shutil.move(version_directory, backup_path)
+
+        console.log("Copying…")
+        mkdirp(version_directory)
+
+        for variant in ihp_sg13g2_family.variants:
+            variant_build_path = os.path.join(build_directory, variant)
+            variant_install_path = os.path.join(version_directory, variant)
+            if os.path.isdir(variant_build_path):
+                shutil.copytree(variant_build_path, variant_install_path)
+
+    console.log("Done.")
+
+
+def build(
+    pdk_root: str,
+    version: str,
+    jobs: int = 1,
+    clear_build_artifacts: bool = True,
+    include_libraries: Optional[List[str]] = None,
+    using_repos: Optional[Dict[str, str]] = None,
+):
+    console = Console()
+    if include_libraries is not None:
+        console.log(
+            "Note: all libraries will be acquired as part of the trivial PDK build."
+        )
+
+    if using_repos is None:
+        using_repos = {}
+
+    build_directory = os.path.join(
+        get_cielo_dir(pdk_root, "ihp_sg13g2"), "build", version
+    )
+    timestamp = datetime.now().strftime("build_ihp-sg13g2-%Y-%m-%d-%H-%M-%S")
+    log_dir = os.path.join(build_directory, "logs", timestamp)
+    mkdirp(log_dir)
+
+    console.log(f"Logging to '{log_dir}'…")
+
+    ihp_path = get_ihp(version, build_directory, jobs, using_repos.get("ihp"))
+    build_ihp(build_directory, ihp_path)
+    install_ihp(build_directory, pdk_root, version)
+
+    if clear_build_artifacts:
+        shutil.rmtree(build_directory)