siliconcompiler 0.32.0__py3-none-any.whl → 0.32.2__py3-none-any.whl

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.32.0'
+version = '0.32.2'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/_common.py

@@ -98,11 +98,28 @@ def pick_manifest(chip, src_file=None):
     if chip.get('option', 'jobname') not in all_manifests[chip.design]:
         jobname = list(all_manifests[chip.design].keys())[0]
 
+    step, index = chip.get('arg', 'step'), chip.get('arg', 'index')
+    if step and not index:
+        all_nodes = list(all_manifests[chip.design][jobname].keys())
+        try:
+            all_nodes.remove((None, None))
+        except ValueError:
+            pass
+        for found_step, found_index in sorted(all_nodes):
+            if found_step == step:
+                index = found_index
+        if index is None:
+            index = '0'
+    if step and index:
+        if (step, index) in all_manifests[chip.design][jobname]:
+            return all_manifests[chip.design][jobname][(step, index)]
+        else:
+            chip.logger.error(f'{step}{index} is not a valid node.')
+            return None
+
     if (None, None) in all_manifests[chip.design][jobname]:
-        manifest = all_manifests[chip.design][jobname][None, None]
-    else:
-        # pick newest manifest
-        manifest = list(sorted(all_manifests[chip.design][jobname].values(),
-                               key=lambda file: os.stat(file).st_ctime))[-1]
+        return all_manifests[chip.design][jobname][None, None]
 
-    return manifest
+    # pick newest manifest
+    return list(sorted(all_manifests[chip.design][jobname].values(),
+                       key=lambda file: os.stat(file).st_ctime))[-1]

siliconcompiler/apps/sc_dashboard.py

@@ -63,6 +63,8 @@ To include another chip object to compare to:
         if manifest:
             chip.logger.info(f'Loading manifest: {manifest}')
             chip.read_manifest(manifest)
+    else:
+        manifest = chip.get('option', 'cfg')
 
     # Error checking
     design = chip.get('design')
@@ -70,6 +72,10 @@ To include another chip object to compare to:
         chip.logger.error('Design not loaded')
         return 1
 
+    if not manifest:
+        chip.logger.error('Unable to determine job manifest')
+        return 2
+
     graph_chips = []
     if switches['graph_cfg']:
         for i, name_and_file_path in enumerate(switches['graph_cfg']):
@@ -86,7 +92,7 @@ To include another chip object to compare to:
                 raise ValueError(('graph_cfg accepts a max of 2 values, you supplied'
                                   f' {args} in "-graph_cfg {name_and_file_path}"'))
             if not os.path.isfile(file_path):
-                raise ValueError(f'not a valid file path : {file_path}')
+                raise ValueError(f'not a valid file path: {file_path}')
             graph_chip = siliconcompiler.core.Chip(design='')
             graph_chip.read_manifest(file_path)
             graph_chips.append({

siliconcompiler/apps/sc_install.py

@@ -139,8 +139,8 @@ def _get_tools_list():
 
 def _recommended_tool_groups(tools):
     groups = {
-        "asic": {"surelog", "sv2v", "yosys", "openroad", "klayout"},
-        "fpga": {"surelog", "sv2v", "yosys", "vpr"},
+        "asic": {"sv2v", "yosys", "openroad", "klayout"},
+        "fpga": {"sv2v", "yosys", "vpr"},
         "digital-simulation": {"verilator", "icarus", "gtkwave"},
         "analog-simulation": {"xyce"}
     }
@@ -162,7 +162,13 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript
 
 def main():
     progname = "sc-install"
-    description = """
+
+    tools = _get_tools_list()
+    group_desc = "\n".join(
+        [f"    {grp}: {', '.join(grp_tools)}"
+         for grp, grp_tools in _recommended_tool_groups(tools).items()])
+
+    description = f"""
 -----------------------------------------------------------
 SC app install supported tools.
 
@@ -187,14 +193,16 @@ To show the install script:
 To system debugging information (this should only be used to debug):
     sc-install -debug_machine
 -----------------------------------------------------------
+Tool groups:
+{group_desc}
+-----------------------------------------------------------
 """
+
     parser = argparse.ArgumentParser(
         prog=progname,
         description=description,
         formatter_class=HelpFormatter)
 
-    tools = _get_tools_list()
-
     if _get_os_name() is None:
         print("Unsupported operating system", file=sys.stderr)
         print_machine_info()

siliconcompiler/apps/sc_remote.py

@@ -97,8 +97,9 @@ To delete a job, use:
         chip.logger.error(f'Error: {", ".join(["-"+e for e in exclusive])} are mutually exclusive')
         return 1
     chip_cfg = chip.get('option', 'cfg')
-    if chip_cfg and not any([args[arg] for arg in cfg_only]):
+    if not chip_cfg and any([args[arg] for arg in cfg_only]):
         chip.logger.error(f'Error: -cfg is required for {", ".join(["-"+e for e in cfg_only])}')
+        return 2
     if any([args[arg] for arg in cfg_only]) and args['server']:
         chip.logger.error('Error: -server cannot be specified with '
                           f'{", ".join(["-"+e for e in cfg_only])}')

siliconcompiler/apps/sc_show.py

@@ -111,6 +111,8 @@ def main():
         if manifest:
             chip.logger.info(f'Loading manifest: {manifest}')
             chip.read_manifest(manifest)
+    else:
+        manifest = chip.get('option', 'cfg')
 
     # Error checking
     design = chip.get('design')
@@ -120,6 +122,10 @@ def main():
                           '-cfg, -design, and/or inputs.')
         return 1
 
+    if not manifest:
+        chip.logger.error('Unable to determine job manifest')
+        return 2
+
     # Read in file
     if filename:
         chip.logger.info(f"Displaying {filename}")

siliconcompiler/core.py

@@ -203,13 +203,11 @@ class Chip:
 
     def _add_file_logger(self, filename):
         # Add a file handler for logging
-        logformat = self.logger.handlers[0].formatter
-
         file_handler = logging.FileHandler(filename)
-        file_handler.setFormatter(logformat)
-
         self.logger.addHandler(file_handler)
 
+        self._init_logger_formats()
+
         return file_handler
 
     ###########################################################################
@@ -227,13 +225,35 @@ class Chip:
         else:
             in_run = False
 
+        # Save in run flag
+        self.logger._in_run = in_run
+        self.logger._in_step = step
+        self.logger._in_index = index
+
+        self.logger.setLevel(schema_utils.translate_loglevel(loglevel))
+
+        if not self.logger.hasHandlers():
+            stream_handler = logging.StreamHandler(stream=sys.stdout)
+            # Save console handler
+            self.logger._console = stream_handler
+            self.logger.addHandler(stream_handler)
+
+            self.logger._support_color = ColorStreamFormatter.supports_color(stream_handler)
+
+        self._init_logger_formats(loglevel=loglevel)
+
+    def _init_logger_formats(self, loglevel=None):
+        if not loglevel:
+            self.schema.get('option', 'loglevel',
+                            step=self.logger._in_step, index=self.logger._in_index)
+
         level_format = '%(levelname)-7s'
         log_format = [level_format]
         if loglevel == 'debug':
             log_format.append('%(funcName)-10s')
             log_format.append('%(lineno)-4s')
 
-        if in_run:
+        if self.logger._in_run:
             max_column_width = 20
             # Figure out how wide to make step and index fields
             max_step_len = 1
@@ -249,6 +269,9 @@ class Chip:
 
             jobname = self.get('option', 'jobname')
 
+            step = self.logger._in_step
+            index = self.logger._in_index
+
             if step is None:
                 step = '-' * max(max_step_len // 4, 1)
             if index is None:
@@ -266,19 +289,13 @@ class Chip:
         log_format.append('%(message)s')
         stream_logformat = log_formatprefix + ' | '.join(log_format[1:])
 
-        if not self.logger.hasHandlers():
-            stream_handler = logging.StreamHandler(stream=sys.stdout)
-            self.logger.addHandler(stream_handler)
-
-        for handler in self.logger.handlers:
-            if ColorStreamFormatter.supports_color(handler):
+        for handler in self.logger.handlers.copy():
+            if handler == self.logger._console and self.logger._support_color:
                 formatter = ColorStreamFormatter(log_formatprefix, level_format, stream_logformat)
             else:
                 formatter = LoggerFormatter(log_formatprefix, level_format, stream_logformat)
             handler.setFormatter(formatter)
 
-        self.logger.setLevel(schema_utils.translate_loglevel(loglevel))
-
     ###########################################################################
     def _init_codecs(self):
         # Custom error handlers used to provide warnings when invalid characters
@@ -3239,9 +3256,10 @@ class Chip:
             if sc_step and sc_index:
                 search_nodes.append((sc_step, sc_index))
             elif sc_step:
-                for check_step, check_index in nodes_to_execute(self, flow):
-                    if sc_step == check_step:
-                        search_nodes.append((check_step, check_index))
+                if flow is not None:
+                    for check_step, check_index in nodes_to_execute(self, flow):
+                        if sc_step == check_step:
+                            search_nodes.append((check_step, check_index))
             else:
                 if flow is not None:
                     for nodes in _get_flowgraph_execution_order(self,

siliconcompiler/fpgas/lattice_ice40.py

@@ -1,5 +1,4 @@
-import siliconcompiler
-from siliconcompiler.utils import register_sc_data_source
+from siliconcompiler import Chip, FPGA
 
 
 ####################################################
@@ -13,22 +12,13 @@ def setup():
     yosys + nextpnr
     '''
 
-    vendor = 'lattice'
-
-    lut_size = '4'
-
     all_fpgas = []
 
-    all_part_names = [
-        "ice40up5k-sg48",
-    ]
-
-    for part_name in all_part_names:
-        fpga = siliconcompiler.FPGA(part_name, package='siliconcompiler_data')
-        register_sc_data_source(fpga)
+    for part_name in ("ice40up5k-sg48",):
+        fpga = FPGA(part_name)
 
-        fpga.set('fpga', part_name, 'vendor', vendor)
-        fpga.set('fpga', part_name, 'lutsize', lut_size)
+        fpga.set('fpga', part_name, 'vendor', 'lattice')
+        fpga.set('fpga', part_name, 'lutsize', 4)
 
         all_fpgas.append(fpga)
 
@@ -37,5 +27,5 @@ def setup():
 
 #########################
 if __name__ == "__main__":
-    for fpga in setup(siliconcompiler.Chip('<fpga>')):
+    for fpga in setup(Chip('<fpga>')):
         fpga.write_manifest(f'{fpga.design}.json')

siliconcompiler/package/__init__.py

@@ -10,9 +10,6 @@ import functools
 import time
 from pathlib import Path
 
-from github import Github
-import github.Auth
-
 from siliconcompiler.utils import get_plugins
 
 
@@ -219,58 +216,6 @@ def register_python_data_source(chip,
                          ref=ref)
 
 
-def register_private_github_data_source(chip,
-                                        package_name,
-                                        repository,
-                                        release,
-                                        artifact):
-    gh = Github(auth=github.Auth.Token(__get_github_auth_token(package_name)))
-    repo = gh.get_repo(repository)
-
-    if not release:
-        release = repo.get_latest_release().tag_name
-
-    url = None
-    for repo_release in repo.get_releases():
-        if repo_release.tag_name == release:
-            for asset in repo_release.assets:
-                if asset.name == artifact:
-                    url = asset.url
-
-    if not url:
-        raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
-
-    chip.register_source(
-        package_name,
-        path=url,
-        ref=release)
-
-
-def __get_github_auth_token(package_name):
-    token_name = package_name.upper()
-    for tok in ('#', '$', '&', '-', '=', '!', '/'):
-        token_name = token_name.replace(tok, '')
-
-    search_env = (
-        f'GITHUB_{token_name}_TOKEN',
-        'GITHUB_TOKEN',
-        'GIT_TOKEN'
-    )
-
-    token = None
-    for env in search_env:
-        token = os.environ.get(env, None)
-
-        if token:
-            break
-
-    if not token:
-        raise ValueError('Unable to determine authorization token for GitHub, '
-                         f'please set one of the following environmental variables: {search_env}')
-
-    return token
-
-
 @functools.lru_cache(maxsize=1)
 def __get_python_module_mapping():
     mapping = {}
@@ -298,3 +243,14 @@ def __get_python_module_mapping():
                 mapping.setdefault(module, []).append(dist_name)
 
     return mapping
+
+
+def register_private_github_data_source(chip,
+                                        package_name,
+                                        repository,
+                                        release,
+                                        artifact):
+    chip.register_source(
+        package_name,
+        path=f"github+private://{repository}/{release}/{artifact}",
+        ref=release)

siliconcompiler/package/github.py

@@ -0,0 +1,124 @@
+import os
+from fasteners import InterProcessLock
+from github import Github, Auth
+from github.GithubException import UnknownObjectException
+from urllib.parse import urlparse
+from siliconcompiler.package import get_download_cache_path
+from siliconcompiler.package import aquire_data_lock, release_data_lock
+from siliconcompiler.package.https import _http_resolver
+
+
+def get_resolver(url):
+    if url.scheme in ("github",):
+        return github_any_resolver
+    if url.scheme in ("github+private",):
+        return github_private_resolver
+    return None
+
+
+def github_any_resolver(chip, package, path, ref, url, fetch):
+    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
+
+    if not fetch:
+        return data_path, False
+
+    # Acquire lock
+    data_lock = InterProcessLock(data_path_lock)
+    aquire_data_lock(data_path, data_lock)
+
+    if os.path.exists(data_path):
+        release_data_lock(data_lock)
+        return data_path, False
+
+    try:
+        return _github_resolver(chip, package, path, ref, url, data_lock)
+    except UnknownObjectException:
+        return github_private_resolver(chip, package, path, ref, url, fetch, data_lock=data_lock)
+
+
+def github_private_resolver(chip, package, path, ref, url, fetch, data_lock=None):
+    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
+
+    if not fetch:
+        return data_path, False
+
+    if not data_lock:
+        # Acquire lock
+        data_lock = InterProcessLock(data_path_lock)
+        aquire_data_lock(data_path, data_lock)
+
+    if os.path.exists(data_path):
+        release_data_lock(data_lock)
+        return data_path, False
+
+    gh = Github(auth=Auth.Token(__get_github_auth_token(package)))
+
+    return _github_resolver(chip, package, path, ref, url, data_lock, gh=gh)
+
+
+def _github_resolver(chip, package, path, ref, url, data_lock, gh=None):
+    if not gh:
+        gh = Github()
+
+    url_parts = (url.netloc, *url.path.split("/")[1:])
+
+    if len(url_parts) != 4:
+        raise ValueError(
+            f"{path} is not in the proper form: <owner>/<repository>/<version>/<artifact>")
+
+    repository = "/".join(url_parts[0:2])
+    release = url_parts[2]
+    artifact = url_parts[3]
+
+    release_url = __get_release_url(gh, repository, release, artifact)
+
+    return _http_resolver(chip, package, release_url, ref, urlparse(release_url), data_lock)
+
+
+def __get_release_url(gh, repository, release, artifact):
+    if artifact == f"{release}.zip":
+        return f"https://github.com/{repository}/archive/refs/tags/{release}.zip"
+    if artifact == f"{release}.tar.gz":
+        return f"https://github.com/{repository}/archive/refs/tags/{release}.tar.gz"
+
+    repo = gh.get_repo(repository)
+
+    if not release:
+        release = repo.get_latest_release().tag_name
+
+    url = None
+    for repo_release in repo.get_releases():
+        if repo_release.tag_name == release:
+            for asset in repo_release.assets:
+                if asset.name == artifact:
+                    url = asset.url
+
+    if not url:
+        raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
+
+    return url
+
+
+def __get_github_auth_token(package_name):
+    token_name = package_name.upper()
+    for tok in ('#', '$', '&', '-', '=', '!', '/'):
+        token_name = token_name.replace(tok, '')
+
+    search_env = (
+        f'GITHUB_{token_name}_TOKEN',
+        'GITHUB_TOKEN',
+        'GIT_TOKEN'
+    )
+
+    token = None
+    for env in search_env:
+        token = os.environ.get(env, None)
+
+        if token:
+            break
+
+    if not token:
+        raise ValueError('Unable to determine authorization token for GitHub, '
+                         f'please set one of the following environmental variables: {search_env}')
+
+    return token

siliconcompiler/package/https.py

@@ -35,6 +35,12 @@ def http_resolver(chip, package, path, ref, url, fetch):
         release_data_lock(data_lock)
         return data_path, False
 
+    return _http_resolver(chip, package, path, ref, url, data_lock)
+
+
+def _http_resolver(chip, package, path, ref, url, data_lock):
+    data_path, _ = get_download_cache_path(chip, package, ref)
+
     extract_from_url(chip, package, path, ref, url, data_path)
 
     release_data_lock(data_lock)

siliconcompiler/report/dashboard/components/__init__.py

@@ -9,6 +9,7 @@ import streamlit_antd_components as sac
 
 from PIL import Image
 
+import siliconcompiler
 from siliconcompiler import __version__ as sc_version
 from siliconcompiler import utils
 from siliconcompiler.report import report
@@ -31,7 +32,7 @@ SC_MENU = {
     "Report a Bug":
     '''https://github.com/siliconcompiler/siliconcompiler/issues''',
     "About": "\n\n".join(SC_ABOUT)}
-SC_DATA_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'data'))
+SC_DATA_ROOT = os.path.abspath(os.path.join(os.path.dirname(siliconcompiler.__file__), 'data'))
 SC_LOGO_PATH = os.path.join(SC_DATA_ROOT, 'logo.png')
 SC_FONT_PATH = os.path.join(SC_DATA_ROOT, 'RobotoMono', 'RobotoMono-Regular.ttf')
 

siliconcompiler/report/dashboard/components/flowgraph.py

@@ -34,6 +34,9 @@ def get_nodes_and_edges(chip):
     nodes = []
     edges = []
 
+    if not chip.get('option', 'flow'):
+        return nodes, edges
+
     default_node_border_width = 1
     successful_path_node_width = 3
     default_edge_width = 3

siliconcompiler/report/dashboard/utils/__init__.py

@@ -24,8 +24,9 @@ def make_node_to_step_index_map(chip, metric_dataframe):
             nodes of the selected chip
     '''
     node_to_step_index_map = {}
-    for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
-        node_to_step_index_map[f'{step}{index}'] = (step, index)
+    if chip.get('option', 'flow'):
+        for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
+            node_to_step_index_map[f'{step}{index}'] = (step, index)
 
     # concatenate step and index
     metric_dataframe.columns = metric_dataframe.columns.map(lambda x: f'{x[0]}{x[1]}')
@@ -54,6 +55,8 @@ def make_metric_to_metric_unit_map(metric_dataframe):
 
 
 def is_running(chip):
+    if not chip.get('option', 'flow'):
+        return False
     for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
         state = chip.get('record', 'status', step=step, index=index)
         if not NodeStatus.is_done(state):

siliconcompiler/report/utils.py

@@ -40,6 +40,9 @@ def _find_summary_metrics(chip, metrics_map):
 def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
     if not flow:
         flow = chip.get('option', 'flow')
+    if not flow:
+        return [], {}, {}, {}, [], {}
+
     if not flowgraph_nodes:
         flowgraph_nodes = nodes_to_execute(chip)
         # only report tool based steps functions