siliconcompiler 0.31.0__py3-none-any.whl → 0.32.0__py3-none-any.whl

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.31.0'
+version = '0.32.0'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc.py

@@ -99,6 +99,7 @@ def main():
 
         # Print Job Summary
         chip.summary()
+        chip.snapshot()
     except SiliconCompilerError:
         return 1
     except Exception as e:

siliconcompiler/apps/sc_install.py

@@ -101,6 +101,19 @@ def print_machine_info():
     print("Scripts:  ", _get_tool_script_dir())
 
 
+def __print_summary(successful, failed):
+    max_len = 64
+    print("#"*max_len)
+    if successful:
+        msg = f"Installed: {', '.join(sorted(successful))}"
+        print(f"# {msg}")
+
+    if failed:
+        msg = f"Failed to install: {failed}"
+        print(f"# {msg}")
+    print("#"*max_len)
+
+
 def _get_tool_script_dir():
     return Path(siliconcompiler.__file__).parent / "toolscripts"
 
@@ -242,6 +255,7 @@ To system debugging information (this should only be used to debug):
                 args.tool.extend(tool_groups[group])
 
     tools_handled = set()
+    tools_completed = set()
     for tool in args.tool:
         if tool in tools_handled:
             continue
@@ -250,9 +264,14 @@ To system debugging information (this should only be used to debug):
             show_tool(tool, tools[tool])
         else:
             if not install_tool(tool, tools[tool], args.build_dir, args.prefix):
+                __print_summary(tools_completed, tool)
                 return 1
+            else:
+                tools_completed.add(tool)
 
     if not args.show:
+        __print_summary(tools_completed, None)
+
         msgs = []
         for env, path in (
                 ("PATH", "bin"),
@@ -262,7 +281,6 @@ To system debugging information (this should only be used to debug):
                 os.path.expandvars(os.path.expanduser(p))
                 for p in os.getenv(env, "").split(":")
             ]
-            print(envs)
             if check_path not in envs:
                 msgs.extend([
                     "",

siliconcompiler/apps/utils/summarize.py

@@ -37,7 +37,7 @@ def main():
         return 1
 
     # Print Job Summary
-    chip.summary(generate_image=False, generate_html=False)
+    chip.summary()
 
     return 0
 

siliconcompiler/core.py

@@ -26,7 +26,6 @@ from siliconcompiler import _metadata
 from siliconcompiler import NodeStatus, SiliconCompilerError
 from siliconcompiler.report import _show_summary_table
 from siliconcompiler.report import _generate_summary_image, _open_summary_image
-from siliconcompiler.report import _generate_html_report, _open_html_report
 from siliconcompiler.report import Dashboard
 from siliconcompiler import package as sc_package
 import glob
@@ -211,6 +210,8 @@ class Chip:
 
         self.logger.addHandler(file_handler)
 
+        return file_handler
+
     ###########################################################################
     def _init_logger(self, step=None, index=None, in_run=False):
 
@@ -2889,7 +2890,7 @@ class Chip:
         return self._dash
 
     ###########################################################################
-    def summary(self, show_all_indices=False, generate_image=True, generate_html=False):
+    def summary(self, show_all_indices=False):
         '''
         Prints a summary of the compilation manifest.
 
@@ -2902,13 +2903,6 @@ class Chip:
             show_all_indices (bool): If True, displays metrics for all indices
                 of each step. If False, displays metrics only for winning
                 indices.
-            generate_image (bool): If True, generates a summary image featuring
-                a layout screenshot and a subset of metrics. Requires that the
-                current job has an ending node that generated a PNG file.
-            generate_html (bool): If True, generates an HTML report featuring a
-                metrics summary table and manifest tree view. The report will
-                include a layout screenshot if the current job has an ending node
-                that generated a PNG file.
 
         Examples:
             >>> chip.summary()
@@ -2921,36 +2915,36 @@ class Chip:
 
         _show_summary_table(self, flow, nodes_to_execute, show_all_indices=show_all_indices)
 
-        # Create a report for the Chip object which can be viewed in a web browser.
-        # Place report files in the build's root directory.
-        work_dir = self.getworkdir()
-        if os.path.isdir(work_dir):
-            # Mark file paths where the reports can be found if they were generated.
-            results_img = os.path.join(work_dir, f'{self.design}.png')
-            results_html = os.path.join(work_dir, 'report.html')
-
-            for path in (results_img, results_html):
-                if os.path.exists(path):
-                    os.remove(path)
-
-            if generate_image:
-                _generate_summary_image(self, results_img)
-
-            if generate_html:
-                _generate_html_report(self, flow, nodes_to_execute, results_html)
-
-            # dashboard does not generate any data
-            self.logger.info(f'Dashboard at "sc-dashboard -cfg {work_dir}/{self.design}.pkg.json"')
-
-            # Try to open the results and layout only if '-nodisplay' is not set.
-            # Priority: PNG > HTML > dashboard.
-            if not self.get('option', 'nodisplay'):
-                if os.path.isfile(results_img):
-                    _open_summary_image(results_img)
-                elif os.path.isfile(results_html):
-                    _open_html_report(self, results_html)
-                else:
-                    self._dashboard(wait=False)
+        # dashboard does not generate any data
+        self.logger.info('Dashboard at "sc-dashboard '
+                         f'-cfg {self.getworkdir()}/{self.design}.pkg.json"')
+
+    ###########################################################################
+    def snapshot(self, path=None, display=True):
+        '''
+        Creates a snapshot image of the job
+
+        Args:
+            path (str): Path to generate the image at, if not provided will default to
+                <job>/<design>.png
+            display (bool): If True, will open the image for viewing. If :keypath:`option,nodisplay`
+                is True, this argument will be ignored.
+
+        Examples:
+            >>> chip.snapshot()
+            Creates a snapshot image in the default location
+        '''
+
+        if not path:
+            path = os.path.join(self.getworkdir(), f'{self.design}.png')
+
+        if os.path.exists(path):
+            os.remove(path)
+
+        _generate_summary_image(self, path)
+
+        if os.path.isfile(path) and not self.get('option', 'nodisplay') and display:
+            _open_summary_image(path)
 
     ###########################################################################
     def clock(self, pin, period, jitter=0, mode='global'):

siliconcompiler/flows/_common.py

@@ -1,4 +1,5 @@
 from siliconcompiler.tools.surelog import parse as surelog_parse
+from siliconcompiler.tools.slang import elaborate as slang_preprocess
 from siliconcompiler.tools.chisel import convert as chisel_convert
 from siliconcompiler.tools.bambu import convert as bambu_convert
 from siliconcompiler.tools.bluespec import convert as bluespec_convert
@@ -7,6 +8,8 @@ from siliconcompiler.tools.sv2v import convert as sv2v_convert
 
 from siliconcompiler.tools.builtin import concatenate
 
+from siliconcompiler.tools.slang import has_pyslang
+
 
 def _make_docs(chip):
     from siliconcompiler.targets import freepdk45_demo
@@ -20,9 +23,12 @@ def _make_docs(chip):
     chip.use(freepdk45_demo)
 
 
-def __get_frontends(allow_system_verilog):
+def __get_frontends(allow_system_verilog, use_surelog=False):
+    parser = surelog_parse
+    if not use_surelog and has_pyslang():
+        parser = slang_preprocess
     systemverilog_frontend = [
-        ('import.verilog', surelog_parse)
+        ('import.verilog', parser)
     ]
     if not allow_system_verilog:
         systemverilog_frontend.append(('import.convert', sv2v_convert))
@@ -36,7 +42,7 @@ def __get_frontends(allow_system_verilog):
     }
 
 
-def setup_multiple_frontends(flow, allow_system_verilog=False):
+def setup_multiple_frontends(flow, allow_system_verilog=False, use_surelog=False):
     '''
     Sets of multiple frontends if different frontends are required.
 
@@ -45,7 +51,7 @@ def setup_multiple_frontends(flow, allow_system_verilog=False):
 
     concat_nodes = []
     flowname = flow.design
-    for _, pipe in __get_frontends(allow_system_verilog).items():
+    for _, pipe in __get_frontends(allow_system_verilog, use_surelog=use_surelog).items():
         prev_step = None
         for step, task in pipe:
             flow.node(flowname, step, task)

siliconcompiler/{package.py → package/__init__.py}

@@ -1,25 +1,20 @@
 import os
-import requests
-import tarfile
-import zipfile
-from git import Repo, GitCommandError
 from urllib.parse import urlparse
 import importlib
-import shutil
 import re
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler.utils import default_cache_dir, _resolve_env_vars
 import json
 from importlib.metadata import distributions, distribution
 import functools
-import fasteners
 import time
 from pathlib import Path
-from io import BytesIO
 
 from github import Github
 import github.Auth
 
+from siliconcompiler.utils import get_plugins
+
 
 def get_cache_path(chip):
     cache_path = chip.get('option', 'cachedir')
@@ -33,10 +28,45 @@ def get_cache_path(chip):
     return cache_path
 
 
-def _path(chip, package, download_handler):
-    if package in chip._packages:
-        return chip._packages[package]
+def get_download_cache_path(chip, package, ref):
+    cache_path = get_cache_path(chip)
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path, exist_ok=True)
+
+    if ref is None:
+        raise SiliconCompilerError(f'Reference is required for cached data: {package}', chip=chip)
+
+    return \
+        os.path.join(cache_path, f'{package}-{ref}'), \
+        os.path.join(cache_path, f'{package}-{ref}.lock')
+
+
+def _file_path_resolver(chip, package, path, ref, url, fetch):
+    return os.path.abspath(path.replace('file://', ''))
+
+
+def _python_path_resolver(chip, package, path, ref, url, fetch):
+    return path_from_python(chip, url.netloc)
 
+
+def _get_path_resolver(path):
+    url = urlparse(path)
+
+    for resolver in get_plugins("path_resolver"):
+        func = resolver(url)
+        if func:
+            return func, url
+
+    if url.scheme == "file":
+        return _file_path_resolver, url
+
+    if url.scheme == "python":
+        return _python_path_resolver, url
+
+    raise ValueError(f"{path} is not supported")
+
+
+def _path(chip, package, fetch):
     # Initially try retrieving data source from schema
     data = {}
     data['path'] = chip.get('package', 'source', package, 'path')
@@ -48,94 +78,45 @@ def _path(chip, package, download_handler):
 
     data['path'] = _resolve_env_vars(chip, data['path'], None, None)
 
-    url = urlparse(data['path'])
-
-    # check network drive for package data source
-    if data['path'].startswith('file://') or os.path.exists(data['path']):
-        path = os.path.abspath(data['path'].replace('file://', ''))
-        chip.logger.info(f'Found {package} data at {path}')
-        chip._packages[package] = path
-        return path
-    elif data['path'].startswith('python://'):
-        path = path_from_python(chip, url.netloc)
-        chip.logger.info(f'Found {package} data at {path}')
-        chip._packages[package] = path
-        return path
-
-    # location of the python package
-    cache_path = get_cache_path(chip)
-    if not os.path.exists(cache_path):
-        os.makedirs(cache_path, exist_ok=True)
-    project_id = f'{package}-{data.get("ref")}'
-    if url.scheme not in ['git', 'git+https', 'https', 'git+ssh', 'ssh'] or not project_id:
-        raise SiliconCompilerError(
-            f'Could not find data path in package {package}: {data["path"]}',
-            chip=chip)
-
-    data_path = os.path.join(cache_path, project_id)
-
-    if download_handler:
-        download_handler(chip,
-                         package,
-                         data,
-                         url,
-                         data_path,
-                         os.path.join(cache_path, f'{project_id}.lock'))
-
-    if os.path.exists(data_path):
-        if package not in chip._packages:
-            chip.logger.info(f'Saved {package} data to {data_path}')
-            chip._packages[package] = data_path
-        return data_path
+    if os.path.exists(data['path']):
+        # Path is already a path
+        return os.path.abspath(data['path'])
 
-    raise SiliconCompilerError(f'Extracting {package} data to {data_path} failed',
-                               chip=chip)
+    path_resolver, url = _get_path_resolver(data['path'])
 
+    return path_resolver(chip, package, data['path'], data['ref'], url, fetch)
 
-def path(chip, package):
+
+def path(chip, package, fetch=True):
     """
     Compute data source data path
     Additionally cache data source data if possible
     Parameters:
         package (str): Name of the data source
+        fetch (bool): Flag to indicate that the path should be fetched
     Returns:
         path: Location of data source on the local system
     """
 
-    return _path(chip, package, __download_data)
-
+    if package not in chip._packages:
+        changed = False
+        data_path = _path(chip, package, fetch)
 
-def __download_data(chip, package, data, url, data_path, data_path_lock):
-    data_lock = fasteners.InterProcessLock(data_path_lock)
+        if isinstance(data_path, tuple) and len(data_path) == 2:
+            data_path, changed = data_path
 
-    _aquire_data_lock(data_path, data_lock)
+        if os.path.exists(data_path):
+            if package not in chip._packages and changed:
+                chip.logger.info(f'Saved {package} data to {data_path}')
+            else:
+                chip.logger.info(f'Found {package} data at {data_path}')
 
-    # check cached package data source
-    if os.path.exists(data_path):
-        chip.logger.info(f'Found cached {package} data at {data_path}')
-        if url.scheme in ['git', 'git+https', 'ssh', 'git+ssh']:
-            try:
-                repo = Repo(data_path)
-                if repo.untracked_files or repo.index.diff("HEAD"):
-                    chip.logger.warning('The repo of the cached data is dirty.')
-                _release_data_lock(data_lock)
-                chip._packages[package] = data_path
-                return
-            except GitCommandError:
-                chip.logger.warning('Deleting corrupted cache data.')
-                shutil.rmtree(path)
-        else:
-            _release_data_lock(data_lock)
             chip._packages[package] = data_path
-            return
-
-    # download package data source
-    if url.scheme in ['git', 'git+https', 'ssh', 'git+ssh']:
-        clone_synchronized(chip, package, data, data_path)
-    elif url.scheme == 'https':
-        extract_from_url(chip, package, data, data_path)
+        else:
+            raise SiliconCompilerError(f'Unable to locate {package} data in {data_path}',
+                                       chip=chip)
 
-    _release_data_lock(data_lock)
+    return chip._packages[package]
 
 
 def __get_filebased_lock(data_lock):
@@ -143,7 +124,7 @@ def __get_filebased_lock(data_lock):
     return Path(f'{base}.sc_lock')
 
 
-def _aquire_data_lock(data_path, data_lock):
+def aquire_data_lock(data_path, data_lock):
     # Wait a maximum of 10 minutes for other processes to finish
     max_seconds = 10 * 60
     try:
@@ -168,7 +149,7 @@ def _aquire_data_lock(data_path, data_lock):
                                'please delete it.')
 
 
-def _release_data_lock(data_lock):
+def release_data_lock(data_lock):
     # Check if file based locking method was used
     lock_file = __get_filebased_lock(data_lock)
     if lock_file.exists():
@@ -178,100 +159,6 @@ def _release_data_lock(data_lock):
     data_lock.release()
 
 
-def clone_synchronized(chip, package, data, data_path):
-    url = urlparse(data['path'])
-    try:
-        clone_from_git(chip, package, data, data_path)
-    except GitCommandError as e:
-        if 'Permission denied' in repr(e):
-            if url.scheme in ['ssh', 'git+ssh']:
-                chip.logger.error('Failed to authenticate. Please setup your git ssh.')
-            elif url.scheme in ['git', 'git+https']:
-                chip.logger.error('Failed to authenticate. Please use a token or ssh.')
-        else:
-            chip.logger.error(str(e))
-
-
-def clone_from_git(chip, package, data, repo_path):
-    url = urlparse(data['path'])
-    if url.scheme in ['git', 'git+https'] and url.username:
-        chip.logger.warning('Your token is in the data source path and will be stored in the '
-                            'schema. If you do not want this set the env variable GIT_TOKEN '
-                            'or use ssh for authentication.')
-    if url.scheme in ['git+ssh', 'ssh']:
-        chip.logger.info(f'Cloning {package} data from {url.netloc}:{url.path[1:]}')
-        # Git requires the format git@github.com:org/repo instead of git@github.com/org/repo
-        repo = Repo.clone_from(f'{url.netloc}:{url.path[1:]}',
-                               repo_path,
-                               recurse_submodules=True)
-    else:
-        if os.environ.get('GIT_TOKEN') and not url.username:
-            url = url._replace(netloc=f'{os.environ.get("GIT_TOKEN")}@{url.hostname}')
-        url = url._replace(scheme='https')
-        chip.logger.info(f'Cloning {package} data from {url.geturl()}')
-        repo = Repo.clone_from(url.geturl(), repo_path, recurse_submodules=True)
-    chip.logger.info(f'Checking out {data["ref"]}')
-    repo.git.checkout(data["ref"])
-    for submodule in repo.submodules:
-        submodule.update(init=True)
-
-
-def extract_from_url(chip, package, data, data_path):
-    url = urlparse(data['path'])
-    data_url = data.get('path')
-    headers = {}
-    if os.environ.get('GIT_TOKEN') or url.username:
-        headers['Authorization'] = f'token {os.environ.get("GIT_TOKEN") or url.username}'
-    if "github" in data_url:
-        headers['Accept'] = 'application/octet-stream'
-    data_url = data['path']
-    if data_url.endswith('/'):
-        data_url = f"{data_url}{data['ref']}.tar.gz"
-    chip.logger.info(f'Downloading {package} data from {data_url}')
-    response = requests.get(data_url, stream=True, headers=headers)
-    if not response.ok:
-        raise SiliconCompilerError(f'Failed to download {package} data source.', chip=chip)
-
-    fileobj = BytesIO(response.content)
-    try:
-        with tarfile.open(fileobj=fileobj, mode='r|gz') as tar_ref:
-            tar_ref.extractall(path=data_path)
-    except tarfile.ReadError:
-        fileobj.seek(0)
-        # Try as zip
-        with zipfile.ZipFile(fileobj) as zip_ref:
-            zip_ref.extractall(path=data_path)
-
-    if 'github' in url.netloc and len(os.listdir(data_path)) == 1:
-        # Github inserts one folder at the highest level of the tar file
-        # this compensates for this behavior
-        gh_url = urlparse(data_url)
-
-        repo = gh_url.path.split('/')[2]
-
-        ref = gh_url.path.split('/')[-1]
-        if repo.endswith('.git'):
-            ref = data['ref']
-        elif ref.endswith('.tar.gz'):
-            ref = ref[0:-7]
-        elif ref.endswith('.tgz'):
-            ref = ref[0:-4]
-        else:
-            ref = ref.split('.')[0]
-
-        if ref.startswith('v'):
-            ref = ref[1:]
-
-        github_folder = f"{repo}-{ref}"
-
-        if github_folder in os.listdir(data_path):
-            # This moves all files one level up
-            git_path = os.path.join(data_path, github_folder)
-            for data_file in os.listdir(git_path):
-                shutil.move(os.path.join(git_path, data_file), data_path)
-            os.removedirs(git_path)
-
-
 def path_from_python(chip, python_package, append_path=None):
     try:
         module = importlib.import_module(python_package)

siliconcompiler/package/git.py

@@ -0,0 +1,84 @@
+import shutil
+
+import os.path
+
+from git import Repo, GitCommandError
+from fasteners import InterProcessLock
+
+from siliconcompiler.package import get_download_cache_path
+from siliconcompiler.package import aquire_data_lock, release_data_lock
+
+
+def get_resolver(url):
+    if url.scheme in ("git", "git+https", "git+ssh", "ssh"):
+        return git_resolver
+    return None
+
+
+def git_resolver(chip, package, path, ref, url, fetch):
+    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
+
+    if not fetch:
+        return data_path, False
+
+    # Acquire lock
+    data_lock = InterProcessLock(data_path_lock)
+    aquire_data_lock(data_path, data_lock)
+
+    if os.path.exists(data_path):
+        try:
+            repo = Repo(data_path)
+            if repo.untracked_files or repo.index.diff("HEAD"):
+                chip.logger.warning('The repo of the cached data is dirty.')
+            release_data_lock(data_lock)
+            return data_path, False
+        except GitCommandError:
+            chip.logger.warning('Deleting corrupted cache data.')
+            shutil.rmtree(data_path)
+
+    clone_synchronized(chip, package, path, ref, url, data_path)
+
+    release_data_lock(data_lock)
+
+    return data_path, True
+
+
+def clone_synchronized(chip, package, path, ref, url, data_path):
+    try:
+        clone_from_git(chip, package, path, ref, url, data_path)
+    except GitCommandError as e:
+        if 'Permission denied' in repr(e):
+            if url.scheme in ['ssh', 'git+ssh']:
+                chip.logger.error('Failed to authenticate. Please setup your git ssh.')
+            elif url.scheme in ['git', 'git+https']:
+                chip.logger.error('Failed to authenticate. Please use a token or ssh.')
+        else:
+            chip.logger.error(str(e))
+
+
+def clone_from_git(chip, package, path, ref, url, data_path):
+    if url.scheme in ['git', 'git+https'] and url.username:
+        chip.logger.warning('Your token is in the data source path and will be stored in the '
+                            'schema. If you do not want this set the env variable GIT_TOKEN '
+                            'or use ssh for authentication.')
+    if url.scheme in ['git+ssh']:
+        chip.logger.info(f'Cloning {package} data from {url.netloc}:{url.path[1:]}')
+        # Git requires the format git@github.com:org/repo instead of git@github.com/org/repo
+        repo = Repo.clone_from(f'{url.netloc}/{url.path[1:]}',
+                               data_path,
+                               recurse_submodules=True)
+    elif url.scheme in ['ssh']:
+        chip.logger.info(f'Cloning {package} data from {path}')
+        repo = Repo.clone_from(path,
+                               data_path,
+                               recurse_submodules=True)
+    else:
+        if os.environ.get('GIT_TOKEN') and not url.username:
+            url = url._replace(netloc=f'{os.environ.get("GIT_TOKEN")}@{url.hostname}')
+        url = url._replace(scheme='https')
+        chip.logger.info(f'Cloning {package} data from {url.geturl()}')
+        repo = Repo.clone_from(url.geturl(), data_path, recurse_submodules=True)
+    chip.logger.info(f'Checking out {ref}')
+    repo.git.checkout(ref)
+    for submodule in repo.submodules:
+        submodule.update(init=True)