siliconcompiler 0.28.2__py3-none-any.whl → 0.28.4__py3-none-any.whl

siliconcompiler/_common.py

@@ -33,6 +33,18 @@ class NodeStatus():
             NodeStatus.PENDING,
         )
 
+    def is_success(status):
+        return status in (
+            NodeStatus.SUCCESS,
+            NodeStatus.SKIPPED
+        )
+
+    def is_error(status):
+        return status in (
+            NodeStatus.ERROR,
+            NodeStatus.TIMEOUT
+        )
+
 
 ###############################################################################
 # Package Customization classes

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.28.2'
+version = '0.28.4'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_dashboard.py

@@ -82,9 +82,13 @@ To include another chip object to compare to:
                 raise ValueError(f'not a valid file path : {file_path}')
             graph_chip = siliconcompiler.core.Chip(design='')
             graph_chip.read_manifest(file_path)
-            graph_chips.append({'chip': graph_chip, 'name': name})
+            graph_chips.append({
+                'chip': graph_chip,
+                'name': name,
+                'cfg_path': os.path.abspath(file_path)
+            })
 
-    chip._dashboard(wait=True, port=switches['port'], graph_chips=graph_chips)
+    chip.dashboard(wait=True, port=switches['port'], graph_chips=graph_chips)
 
     return 0
 

siliconcompiler/apps/sc_install.py

@@ -70,17 +70,46 @@ def show_tool(tool, script):
     print_header("end")
 
 
+def _get_os_name():
+    machine_info = _get_machine_info()
+    system = machine_info.get('system', "").lower()
+    distro = machine_info.get('distro', "").lower()
+    osversion = machine_info.get('osversion', "").lower()
+    if system == 'linux':
+        if distro == 'ubuntu':
+            version, _ = osversion.split('.')
+            return f"{distro}{version}"
+        elif distro == 'rocky':
+            version, _ = osversion.split('.')
+            return f"rhel{version}"
+        elif distro == 'rhel':
+            version, _ = osversion.split('.')
+            return f"rhel{version}"
+    return None
+
+
+def print_machine_info():
+    machine_info = _get_machine_info()
+    mapped_os = _get_os_name()
+
+    print("System:   ", machine_info.get('system', None))
+    print("Distro:   ", machine_info.get('distro', None))
+    print("Version:  ", machine_info.get('osversion', None))
+    print("Mapped OS:", mapped_os)
+    print("Scripts:  ", _get_tool_script_dir())
+
+
+def _get_tool_script_dir():
+    return Path(siliconcompiler.__file__).parent / "toolscripts"
+
+
 def _get_tools_list():
-    tools_root = Path(siliconcompiler.__file__).parent / "toolscripts"
+    tools_root = _get_tool_script_dir()
 
-    machine_info = _get_machine_info()
     script_dir = None
-    if machine_info['system'].lower() == 'linux':
-        if machine_info['distro'].lower() == 'ubuntu':
-            version, _ = machine_info['osversion'].split('.')
-            script_dir = f"{machine_info['distro'].lower()}{version}"
-    if script_dir:
-        script_dir = tools_root / script_dir
+    os_dir = _get_os_name()
+    if os_dir:
+        script_dir = tools_root / os_dir
         if not script_dir.exists():
             script_dir = None
 
@@ -93,14 +122,20 @@ def _get_tools_list():
     return tools
 
 
-def _recommended_tool_groups():
-    return {
+def _recommended_tool_groups(tools):
+    groups = {
         "asic": {"surelog", "sv2v", "yosys", "openroad", "klayout"},
         "fpga": {"surelog", "sv2v", "yosys", "vpr"},
         "digital-simulation": {"verilator", "icarus"},
         "analog-simulation": {"xyce"}
     }
 
+    filter_groups = {}
+    for group, group_tools in groups.items():
+        if all([tool in tools for tool in group_tools]):
+            filter_groups[group] = group_tools
+    return filter_groups
+
 
 def main():
     progname = "sc-install"
@@ -125,6 +160,9 @@ To build tools in a different location:
 
 To show the install script:
     sc-install -show openroad
+
+To system debugging information (this should only be used to debug):
+    sc-install -debug_machine
 -----------------------------------------------------------
 """
     parser = argparse.ArgumentParser(
@@ -141,11 +179,12 @@ To show the install script:
         choices=tool_choices,
         help="tool to install")
 
+    tool_groups = _recommended_tool_groups(tools)
     parser.add_argument(
         "-group",
         nargs="+",
-        choices=_recommended_tool_groups().keys(),
-        help="tool group to install")
+        choices=tool_groups.keys(),
+        help=f"tool group to install{' - not supported' if not tool_groups else ''}")
 
     parser.add_argument(
         "-prefix",
@@ -164,15 +203,24 @@ To show the install script:
         action="store_true",
         help="Show the install script and exit")
 
+    parser.add_argument(
+        "-debug_machine",
+        action="store_true",
+        help="Show information about this machine and exit")
+
     args = parser.parse_args()
 
+    if args.debug_machine:
+        print_machine_info()
+        return 0
+
     if not args.tool:
         args.tool = []
 
     args.tool = list(args.tool)
     if args.group:
         for group in args.group:
-            args.tool.extend(_recommended_tool_groups()[group])
+            args.tool.extend(tool_groups[group])
 
     tools_handled = set()
     for tool in args.tool:

siliconcompiler/apps/sc_remote.py

@@ -185,7 +185,7 @@ To delete a job, use:
     # If only a manifest is specified, make a 'check_progress/' request and report results:
     elif chip_cfg:
         try:
-            check_progress(chip)
+            check_progress(chip, [], {})
         except SiliconCompilerError as e:
             chip.logger.error(f'{e}')
             return 1

siliconcompiler/core.py

@@ -88,6 +88,9 @@ class Chip:
         # Cache of file hashes
         self.__hashes = {}
 
+        # Dashboard
+        self._dash = None
+
         # Showtools
         self._showtools = {}
         for plugin in utils.get_plugins('show'):
@@ -758,7 +761,7 @@ class Chip:
         return fullstr
 
     ###########################################################################
-    def valid(self, *keypath, default_valid=False, job=None):
+    def valid(self, *keypath, default_valid=False, job=None, check_complete=False):
         """
         Checks validity of a keypath.
 
@@ -770,6 +773,7 @@ class Chip:
             keypaths as a wildcard. Defaults to False.
             job (str): Jobname to use for dictionary access in place of the
                 current active jobname.
+            check_complete (bool): Require the keypath be a complete path.
 
         Returns:
             Boolean indicating validity of keypath.
@@ -782,7 +786,10 @@ class Chip:
             >>> check = chip.valid('metric', 'foo', '0', 'tasktime', default_valid=True)
             Returns True, even if "foo" and "0" aren't in current configuration.
         """
-        return self.schema.valid(*keypath, default_valid=default_valid, job=job)
+        return self.schema.valid(*keypath,
+                                 default_valid=default_valid,
+                                 job=job,
+                                 check_complete=check_complete)
 
     ###########################################################################
     def get(self, *keypath, field='value', job=None, step=None, index=None):
@@ -1123,6 +1130,9 @@ class Chip:
                     quiet=quiet)
             return
 
+        if filename is None:
+            raise ValueError(f"{category} cannot process None")
+
         # Normalize value to string in case we receive a pathlib.Path
         filename = str(filename)
 
@@ -1147,8 +1157,9 @@ class Chip:
             use_filetype = filetype
 
         if not use_fileset or not use_filetype:
-            self.logger.error(f'Unable to infer {category} fileset and/or filetype for '
-                              f'{filename} based on file extension.')
+            raise SiliconCompilerError(
+                f'Unable to infer {category} fileset and/or filetype for '
+                f'{filename} based on file extension.')
         elif not quiet:
             if not fileset and not filetype:
                 self.logger.info(f'{filename} inferred as {use_fileset}/{use_filetype}')
@@ -1352,7 +1363,7 @@ class Chip:
             basename = str(pathlib.PurePosixPath(*path_paths[0:n]))
             endname = str(pathlib.PurePosixPath(*path_paths[n:]))
 
-            import_name = self.__get_imported_filename(basename, package)
+            import_name = utils.get_hashed_filename(basename, package=package)
             if import_name not in collected_files:
                 continue
 
@@ -2040,6 +2051,30 @@ class Chip:
             dot.graph_attr['ranksep'] = '0.75'
         dot.attr(bgcolor=background)
 
+        subgraphs = {
+            "graphs": {
+                "sc-inputs": {
+                    "graphs": {},
+                    "nodes": []
+                }
+            },
+            "nodes": []
+        }
+        for node, info in nodes.items():
+            if info['is_input']:
+                subgraph_temp = subgraphs["graphs"]["sc-inputs"]
+            else:
+                subgraph_temp = subgraphs
+
+            for key in node.split(".")[0:-1]:
+                if key not in subgraph_temp["graphs"]:
+                    subgraph_temp["graphs"][key] = {
+                        "graphs": {},
+                        "nodes": []
+                    }
+                subgraph_temp = subgraph_temp["graphs"][key]
+            subgraph_temp["nodes"].append(node)
+
         with dot.subgraph(name='inputs') as input_graph:
             input_graph.graph_attr['cluster'] = 'true'
             input_graph.graph_attr['color'] = background
@@ -2051,34 +2086,72 @@ class Chip:
                     fontcolor=fontcolor, fontsize=fontsize, ordering="in",
                     penwidth=penwidth, fillcolor=fillcolor, shape="box")
 
-        with dot.subgraph(name='input_nodes') as input_graph_nodes:
-            input_graph_nodes.graph_attr['cluster'] = 'true'
-            input_graph_nodes.graph_attr['color'] = background
+        def make_node(graph, node, prefix):
+            info = nodes[node]
 
-            # add nodes
             shape = "oval" if not show_io else "Mrecord"
-            for node, info in nodes.items():
-                task_label = f"\\n ({info['task']})" if info['task'] is not None else ""
-                if show_io:
-                    input_labels = [f"<{ikey}> {ifile}" for ifile, ikey in info['inputs'].items()]
-                    output_labels = [f"<{okey}> {ofile}" for ofile, okey in info['outputs'].items()]
-                    center_text = f"\\n {node} {task_label} \\n\\n"
-                    labelname = "{"
-                    if input_labels:
-                        labelname += f"{{ {' | '.join(input_labels)} }} |"
-                    labelname += center_text
-                    if output_labels:
-                        labelname += f"| {{ {' | '.join(output_labels)} }}"
-                    labelname += "}"
+            task_label = f"\\n ({info['task']})" if info['task'] is not None else ""
+            if show_io:
+                input_labels = [f"<{ikey}> {ifile}" for ifile, ikey in info['inputs'].items()]
+                output_labels = [f"<{okey}> {ofile}" for ofile, okey in info['outputs'].items()]
+                center_text = f"\\n {node.replace(prefix, '')} {task_label} \\n\\n"
+                labelname = "{"
+                if input_labels:
+                    labelname += f"{{ {' | '.join(input_labels)} }} |"
+                labelname += center_text
+                if output_labels:
+                    labelname += f"| {{ {' | '.join(output_labels)} }}"
+                labelname += "}"
+            else:
+                labelname = f"{node.replace(prefix, '')}{task_label}"
+
+            graph.node(node, label=labelname, bordercolor=fontcolor, style='filled',
+                       fontcolor=fontcolor, fontsize=fontsize, ordering="in",
+                       penwidth=penwidth, fillcolor=fillcolor, shape=shape)
+
+        graph_idx = 0
+
+        def get_node_count(graph_info):
+            nodes = len(graph_info["nodes"])
+
+            for subgraph in graph_info["graphs"]:
+                nodes += get_node_count(graph_info["graphs"][subgraph])
+
+            return nodes
+
+        def build_graph(graph_info, parent, prefix):
+            nonlocal graph_idx
+
+            for subgraph in graph_info["graphs"]:
+                if get_node_count(graph_info["graphs"][subgraph]) > 1:
+                    graph = graphviz.Digraph(name=f"cluster_{graph_idx}")
+                    graph_idx += 1
+
+                    graph.graph_attr['rankdir'] = rankdir
+                    graph.attr(bgcolor=background)
+
+                    if subgraph == "sc-inputs":
+                        graph.attr(style='invis')
+                    else:
+                        graph.attr(color=fontcolor)
+                        graph.attr(style='rounded')
+                        graph.attr(shape='oval')
+                        graph.attr(label=subgraph)
+                        graph.attr(labeljust='l')
+                        graph.attr(fontcolor=fontcolor)
+                        graph.attr(fontsize=str(int(fontsize) + 2))
                 else:
-                    labelname = f"{node}{task_label}"
+                    graph = parent
+
+                build_graph(graph_info["graphs"][subgraph], graph, f"{prefix}{subgraph}.")
+
+                if graph is not parent:
+                    parent.subgraph(graph)
 
-                dst = dot
-                if info['is_input']:
-                    dst = input_graph_nodes
-                dst.node(node, label=labelname, bordercolor=fontcolor, style='filled',
-                         fontcolor=fontcolor, fontsize=fontsize, ordering="in",
-                         penwidth=penwidth, fillcolor=fillcolor, shape=shape)
+            for subnode in graph_info["nodes"]:
+                make_node(parent, subnode, prefix)
+
+        build_graph(subgraphs, dot, "")
 
         for edge0, edge1, weight in edges:
             dot.edge(f'{edge0}{out_label_suffix}', f'{edge1}{in_label_suffix}', weight=str(weight))
@@ -2383,7 +2456,7 @@ class Chip:
 
             abspath = dirs[(package, path)]
             if abspath:
-                filename = self.__get_imported_filename(posix_path, package)
+                filename = utils.get_hashed_filename(posix_path, package=package)
                 dst_path = os.path.join(directory, filename)
                 if os.path.exists(dst_path):
                     continue
@@ -2455,7 +2528,7 @@ class Chip:
 
             abspath = files[(package, path)]
             if abspath:
-                filename = self.__get_imported_filename(posix_path, package)
+                filename = utils.get_hashed_filename(posix_path, package=package)
                 dst_path = os.path.join(directory, filename)
                 if verbose:
                     self.logger.info(f"Copying {abspath} to '{directory}' directory")
@@ -2691,12 +2764,12 @@ class Chip:
         return hashlist
 
     ###########################################################################
-    def _dashboard(self, wait=True, port=None, graph_chips=None):
+    def dashboard(self, wait=True, port=None, graph_chips=None):
         '''
         Open a session of the dashboard.
 
         The dashboard can be viewed in any webbrowser and can be accessed via:
-        http://localhost:8501/
+        http://localhost:<port>/
 
         Args:
             wait (bool): If True, this call will wait in this method
@@ -2707,21 +2780,28 @@ class Chip:
                 {'chip': chip object, 'name': chip name}
 
         Examples:
-            >>> chip._dashboard()
+            >>> chip.dashboard()
             Opens a sesison of the dashboard.
         '''
-        dash = Dashboard(self, port=port, graph_chips=graph_chips)
-        dash.open_dashboard()
+        if self._dash:
+            # Remove previous dashboard
+            self._dash.stop()
+            self._dash = None
+
+        self._dash = Dashboard(self, port=port, graph_chips=graph_chips)
+        self._dash.open_dashboard()
+
         if wait:
             try:
-                dash.wait()
+                self._dash.wait()
             except KeyboardInterrupt:
-                dash._sleep()
+                self._dash._sleep()
             finally:
-                dash.stop()
+                self._dash.stop()
+                self._dash = None
             return None
 
-        return dash
+        return self._dash
 
     ###########################################################################
     def summary(self, show_all_indices=False, generate_image=True, generate_html=True):
@@ -2761,8 +2841,8 @@ class Chip:
         work_dir = self.getworkdir()
         if os.path.isdir(work_dir):
             # Mark file paths where the reports can be found if they were generated.
-            results_html = os.path.join(work_dir, 'report.html')
             results_img = os.path.join(work_dir, f'{self.design}.png')
+            results_html = os.path.join(work_dir, 'report.html')
 
             if generate_image:
                 _generate_summary_image(self, results_img)
@@ -2770,13 +2850,18 @@ class Chip:
             if generate_html:
                 _generate_html_report(self, flow, nodes_to_execute, results_html)
 
+            # dashboard does not generate any data
+            self.logger.info(f'Dashboard at "sc-dashboard -cfg {work_dir}/{self.design}.pkg.json"')
+
             # Try to open the results and layout only if '-nodisplay' is not set.
-            # Priority: PNG, PDF, HTML.
-            if (not self.get('option', 'nodisplay')):
+            # Priority: PNG > HTML > dashboard.
+            if not self.get('option', 'nodisplay'):
                 if os.path.isfile(results_img):
                     _open_summary_image(results_img)
                 elif os.path.isfile(results_html):
                     _open_html_report(self, results_html)
+                else:
+                    self._dashboard(wait=False)
 
     ###########################################################################
     def clock(self, pin, period, jitter=0, mode='global'):
@@ -3120,6 +3205,7 @@ class Chip:
         self.set('option', 'nodisplay', False, clobber=True)
         self.set('option', 'continue', True, clobber=True)
         self.set('option', 'quiet', False, clobber=True)
+        self.set('option', 'clean', True, clobber=True)
         self.set('arg', 'step', None, clobber=True)
         self.set('arg', 'index', None, clobber=True)
         self.unset('option', 'to')
@@ -3214,31 +3300,6 @@ class Chip:
 
         return os.path.join(*dirlist)
 
-    #######################################
-    def __get_imported_filename(self, pathstr, package=None):
-        ''' Utility to map collected file to an unambiguous name based on its path.
-
-        The mapping looks like:
-        path/to/file.ext => file_<md5('path/to/file.ext')>.ext
-        '''
-        path = pathlib.PurePosixPath(pathstr)
-        ext = ''.join(path.suffixes)
-
-        # strip off all file suffixes to get just the bare name
-        barepath = path
-        while barepath.suffix:
-            barepath = pathlib.PurePosixPath(barepath.stem)
-        filename = str(barepath.parts[-1])
-
-        if not package:
-            package = ''
-        else:
-            package = f'{package}:'
-        path_to_hash = f'{package}{str(path)}'
-        pathhash = hashlib.sha1(path_to_hash.encode('utf-8')).hexdigest()
-
-        return f'{filename}_{pathhash}{ext}'
-
     def error(self, msg):
         '''
         Raises error.
@@ -3273,6 +3334,9 @@ class Chip:
         # Modules are not serializable, so save without cache
         attributes['_showtools'] = {}
 
+        # Dashboard is not serializable
+        attributes['_dash'] = None
+
         # We have to remove the chip's logger before serializing the object
         # since the logger object is not serializable.
         del attributes['logger']

siliconcompiler/fpgas/vpr_example.py

@@ -43,6 +43,14 @@ def setup():
 
         fpga.set('fpga', part_name, 'vendor', vendor)
 
+        # Part name is specified per architecture file.  Device code specifies
+        # which <fixed_layout> name to use when running VPR.  These examples
+        # use the following names:
+        if (part_name == 'example_arch_X005Y005'):
+            fpga.set('fpga', part_name, 'var', 'vpr_device_code', 'fpga_beta')
+        else:
+            fpga.set('fpga', part_name, 'var', 'vpr_device_code', part_name)
+
         fpga.set('fpga', part_name, 'lutsize', lut_size)
 
         arch_root = os.path.join(flow_root, 'arch', part_name)

siliconcompiler/package.py

@@ -88,7 +88,8 @@ def _path(chip, package, download_handler):
             chip._packages[package] = data_path
         return data_path
 
-    raise SiliconCompilerError(f'Extracting {package} data to {data_path} failed')
+    raise SiliconCompilerError(f'Extracting {package} data to {data_path} failed',
+                               chip=chip)
 
 
 def path(chip, package):
@@ -188,7 +189,7 @@ def clone_synchronized(chip, package, data, data_path):
             elif url.scheme in ['git', 'git+https']:
                 chip.logger.error('Failed to authenticate. Please use a token or ssh.')
         else:
-            raise e
+            chip.logger.error(str(e))
 
 
 def clone_from_git(chip, package, data, repo_path):

siliconcompiler/remote/client.py

@@ -178,7 +178,7 @@ def _log_truncated_stats(chip, status, nodes_with_status, nodes_to_print):
 
 
 ###################################
-def _process_progress_info(chip, progress_info, nodes_to_print=3):
+def _process_progress_info(chip, progress_info, recorded_nodes, all_nodes, nodes_to_print=3):
     '''
     Helper method to log information about a remote run's progress,
     based on information returned from a 'check_progress/' call.
@@ -201,6 +201,11 @@ def _process_progress_info(chip, progress_info, nodes_to_print=3):
                 # collect completed
                 completed.append(node)
 
+            if node in all_nodes:
+                step, index = all_nodes[node]
+                if (step, index) not in recorded_nodes:
+                    chip.set('record', 'status', status, step=step, index=index)
+
         nodes_to_log = {key: nodes_to_log[key] for key in sorted(nodes_to_log.keys())}
 
         # Log information about the job's progress.
@@ -360,11 +365,32 @@ def __remote_run_loop(chip, check_interval):
     completed = []
     result_procs = []
 
+    recorded = []
+
     for step, index in nodes_to_execute(chip):
         if SCNodeStatus.is_done(chip.get('record', 'status', step=step, index=index)):
             continue
         all_nodes[f'{step}{index}'] = (step, index)
 
+    def import_manifests():
+        changed = False
+        for step, index in all_nodes.values():
+            if (step, index) in recorded:
+                continue
+
+            manifest = os.path.join(chip.getworkdir(step=step, index=index),
+                                    'outputs',
+                                    f'{chip.design}.pkg.json')
+            if os.path.exists(manifest):
+                try:
+                    chip.schema.read_journal(manifest)
+                    recorded.append((step, index))
+                    changed = True
+                except:  # noqa E722
+                    # Import may fail if file is still getting written
+                    pass
+        return changed
+
     def schedule_download(node):
         node_proc = multiprocessor.Process(target=fetch_results,
                                            args=(chip, node))
@@ -376,7 +402,12 @@ def __remote_run_loop(chip, check_interval):
 
     while is_busy:
         time.sleep(check_interval)
-        new_completed, is_busy = check_progress(chip)
+        import_manifests()
+        new_completed, is_busy = check_progress(chip, recorded, all_nodes)
+
+        if chip._dash:
+            chip._dash.update_manifest()
+
         nodes_to_fetch = []
         for node in new_completed:
             if node not in completed:
@@ -400,26 +431,26 @@ def __remote_run_loop(chip, check_interval):
         proc.join()
 
     # Read in node manifests
-    for step, index in all_nodes.values():
-        manifest = os.path.join(chip.getworkdir(step=step, index=index),
-                                'outputs',
-                                f'{chip.design}.pkg.json')
-        if os.path.exists(manifest):
-            chip.schema.read_journal(manifest)
+    import_manifests()
 
     # Un-set the 'remote' option to avoid from/to-based summary/show errors
     chip.unset('option', 'remote')
 
+    if chip._dash:
+        chip._dash.update_manifest()
+
 
 ###################################
-def check_progress(chip):
+def check_progress(chip, recorded_nodes, all_nodes):
     try:
         is_busy_info = is_job_busy(chip)
         is_busy = is_busy_info['busy']
         completed = []
         if is_busy:
             completed = _process_progress_info(chip,
-                                               is_busy_info)
+                                               is_busy_info,
+                                               recorded_nodes,
+                                               all_nodes)
         return completed, is_busy
     except Exception as e:
         # Sometimes an exception is raised if the request library cannot

siliconcompiler/report/__init__.py

@@ -1,7 +1,7 @@
 from .summary_image import _generate_summary_image, _open_summary_image
 from .html_report import _generate_html_report, _open_html_report
 from .summary_table import _show_summary_table
-from .streamlit_report import Dashboard
+from .dashboard import Dashboard
 
 __all__ = [
     "_generate_summary_image",