siliconcompiler 0.28.2__py3-none-any.whl → 0.28.3__py3-none-any.whl

siliconcompiler/_common.py

@@ -33,6 +33,18 @@ class NodeStatus():
             NodeStatus.PENDING,
         )
 
+    def is_success(status):
+        return status in (
+            NodeStatus.SUCCESS,
+            NodeStatus.SKIPPED
+        )
+
+    def is_error(status):
+        return status in (
+            NodeStatus.ERROR,
+            NodeStatus.TIMEOUT
+        )
+
 
 ###############################################################################
 # Package Customization classes

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.28.2'
+version = '0.28.3'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_dashboard.py

@@ -82,7 +82,11 @@ To include another chip object to compare to:
                 raise ValueError(f'not a valid file path : {file_path}')
             graph_chip = siliconcompiler.core.Chip(design='')
             graph_chip.read_manifest(file_path)
-            graph_chips.append({'chip': graph_chip, 'name': name})
+            graph_chips.append({
+                'chip': graph_chip,
+                'name': name,
+                'cfg_path': os.path.abspath(file_path)
+            })
 
     chip._dashboard(wait=True, port=switches['port'], graph_chips=graph_chips)
 

siliconcompiler/apps/sc_install.py

@@ -70,17 +70,46 @@ def show_tool(tool, script):
     print_header("end")
 
 
+def _get_os_name():
+    machine_info = _get_machine_info()
+    system = machine_info.get('system', "").lower()
+    distro = machine_info.get('distro', "").lower()
+    osversion = machine_info.get('osversion', "").lower()
+    if system == 'linux':
+        if distro == 'ubuntu':
+            version, _ = osversion.split('.')
+            return f"{distro}{version}"
+        elif distro == 'rocky':
+            version, _ = osversion.split('.')
+            return f"rhel{version}"
+        elif distro == 'rhel':
+            version, _ = osversion.split('.')
+            return f"rhel{version}"
+    return None
+
+
+def print_machine_info():
+    machine_info = _get_machine_info()
+    mapped_os = _get_os_name()
+
+    print("System:   ", machine_info.get('system', None))
+    print("Distro:   ", machine_info.get('distro', None))
+    print("Version:  ", machine_info.get('osversion', None))
+    print("Mapped OS:", mapped_os)
+    print("Scripts:  ", _get_tool_script_dir())
+
+
+def _get_tool_script_dir():
+    return Path(siliconcompiler.__file__).parent / "toolscripts"
+
+
 def _get_tools_list():
-    tools_root = Path(siliconcompiler.__file__).parent / "toolscripts"
+    tools_root = _get_tool_script_dir()
 
-    machine_info = _get_machine_info()
     script_dir = None
-    if machine_info['system'].lower() == 'linux':
-        if machine_info['distro'].lower() == 'ubuntu':
-            version, _ = machine_info['osversion'].split('.')
-            script_dir = f"{machine_info['distro'].lower()}{version}"
-    if script_dir:
-        script_dir = tools_root / script_dir
+    os_dir = _get_os_name()
+    if os_dir:
+        script_dir = tools_root / os_dir
         if not script_dir.exists():
             script_dir = None
 
@@ -93,14 +122,20 @@ def _get_tools_list():
     return tools
 
 
-def _recommended_tool_groups():
-    return {
+def _recommended_tool_groups(tools):
+    groups = {
         "asic": {"surelog", "sv2v", "yosys", "openroad", "klayout"},
         "fpga": {"surelog", "sv2v", "yosys", "vpr"},
         "digital-simulation": {"verilator", "icarus"},
         "analog-simulation": {"xyce"}
     }
 
+    filter_groups = {}
+    for group, group_tools in groups.items():
+        if all([tool in tools for tool in group_tools]):
+            filter_groups[group] = group_tools
+    return filter_groups
+
 
 def main():
     progname = "sc-install"
@@ -125,6 +160,9 @@ To build tools in a different location:
 
 To show the install script:
     sc-install -show openroad
+
+To system debugging information (this should only be used to debug):
+    sc-install -debug_machine
 -----------------------------------------------------------
 """
     parser = argparse.ArgumentParser(
@@ -141,11 +179,12 @@ To show the install script:
         choices=tool_choices,
         help="tool to install")
 
+    tool_groups = _recommended_tool_groups(tools)
     parser.add_argument(
         "-group",
         nargs="+",
-        choices=_recommended_tool_groups().keys(),
-        help="tool group to install")
+        choices=tool_groups.keys(),
+        help=f"tool group to install{' - not supported' if not tool_groups else ''}")
 
     parser.add_argument(
         "-prefix",
@@ -164,15 +203,24 @@ To show the install script:
         action="store_true",
         help="Show the install script and exit")
 
+    parser.add_argument(
+        "-debug_machine",
+        action="store_true",
+        help="Show information about this machine and exit")
+
     args = parser.parse_args()
 
+    if args.debug_machine:
+        print_machine_info()
+        return 0
+
     if not args.tool:
         args.tool = []
 
     args.tool = list(args.tool)
     if args.group:
         for group in args.group:
-            args.tool.extend(_recommended_tool_groups()[group])
+            args.tool.extend(tool_groups[group])
 
     tools_handled = set()
     for tool in args.tool:

siliconcompiler/apps/sc_remote.py

@@ -185,7 +185,7 @@ To delete a job, use:
     # If only a manifest is specified, make a 'check_progress/' request and report results:
     elif chip_cfg:
         try:
-            check_progress(chip)
+            check_progress(chip, [], {})
         except SiliconCompilerError as e:
             chip.logger.error(f'{e}')
             return 1

siliconcompiler/core.py

@@ -88,6 +88,9 @@ class Chip:
         # Cache of file hashes
         self.__hashes = {}
 
+        # Dashboard
+        self._dash = None
+
         # Showtools
         self._showtools = {}
         for plugin in utils.get_plugins('show'):
@@ -758,7 +761,7 @@ class Chip:
         return fullstr
 
     ###########################################################################
-    def valid(self, *keypath, default_valid=False, job=None):
+    def valid(self, *keypath, default_valid=False, job=None, check_complete=False):
         """
         Checks validity of a keypath.
 
@@ -770,6 +773,7 @@ class Chip:
             keypaths as a wildcard. Defaults to False.
             job (str): Jobname to use for dictionary access in place of the
                 current active jobname.
+            check_complete (bool): Require the keypath be a complete path.
 
         Returns:
             Boolean indicating validity of keypath.
@@ -782,7 +786,10 @@ class Chip:
             >>> check = chip.valid('metric', 'foo', '0', 'tasktime', default_valid=True)
             Returns True, even if "foo" and "0" aren't in current configuration.
         """
-        return self.schema.valid(*keypath, default_valid=default_valid, job=job)
+        return self.schema.valid(*keypath,
+                                 default_valid=default_valid,
+                                 job=job,
+                                 check_complete=check_complete)
 
     ###########################################################################
     def get(self, *keypath, field='value', job=None, step=None, index=None):
@@ -1123,6 +1130,9 @@ class Chip:
                     quiet=quiet)
             return
 
+        if filename is None:
+            raise ValueError(f"{category} cannot process None")
+
         # Normalize value to string in case we receive a pathlib.Path
         filename = str(filename)
 
@@ -1147,8 +1157,9 @@ class Chip:
             use_filetype = filetype
 
         if not use_fileset or not use_filetype:
-            self.logger.error(f'Unable to infer {category} fileset and/or filetype for '
-                              f'{filename} based on file extension.')
+            raise SiliconCompilerError(
+                f'Unable to infer {category} fileset and/or filetype for '
+                f'{filename} based on file extension.')
         elif not quiet:
             if not fileset and not filetype:
                 self.logger.info(f'{filename} inferred as {use_fileset}/{use_filetype}')
@@ -2710,18 +2721,25 @@ class Chip:
             >>> chip._dashboard()
             Opens a sesison of the dashboard.
         '''
-        dash = Dashboard(self, port=port, graph_chips=graph_chips)
-        dash.open_dashboard()
+        if self._dash:
+            # Remove previous dashboard
+            self._dash.stop()
+            self._dash = None
+
+        self._dash = Dashboard(self, port=port, graph_chips=graph_chips)
+        self._dash.open_dashboard()
+
         if wait:
             try:
-                dash.wait()
+                self._dash.wait()
             except KeyboardInterrupt:
-                dash._sleep()
+                self._dash._sleep()
             finally:
-                dash.stop()
+                self._dash.stop()
+                self._dash = None
             return None
 
-        return dash
+        return self._dash
 
     ###########################################################################
     def summary(self, show_all_indices=False, generate_image=True, generate_html=True):
@@ -2761,8 +2779,8 @@ class Chip:
         work_dir = self.getworkdir()
         if os.path.isdir(work_dir):
             # Mark file paths where the reports can be found if they were generated.
-            results_html = os.path.join(work_dir, 'report.html')
             results_img = os.path.join(work_dir, f'{self.design}.png')
+            results_html = os.path.join(work_dir, 'report.html')
 
             if generate_image:
                 _generate_summary_image(self, results_img)
@@ -2770,13 +2788,18 @@ class Chip:
             if generate_html:
                 _generate_html_report(self, flow, nodes_to_execute, results_html)
 
+            # dashboard does not generate any data
+            self.logger.info(f'Dashboard at "sc-dashboard -cfg {work_dir}/{self.design}.pkg.json"')
+
             # Try to open the results and layout only if '-nodisplay' is not set.
-            # Priority: PNG, PDF, HTML.
-            if (not self.get('option', 'nodisplay')):
+            # Priority: PNG > HTML > dashboard.
+            if not self.get('option', 'nodisplay'):
                 if os.path.isfile(results_img):
                     _open_summary_image(results_img)
                 elif os.path.isfile(results_html):
                     _open_html_report(self, results_html)
+                else:
+                    self._dashboard(wait=False)
 
     ###########################################################################
     def clock(self, pin, period, jitter=0, mode='global'):
@@ -3273,6 +3296,9 @@ class Chip:
         # Modules are not serializable, so save without cache
         attributes['_showtools'] = {}
 
+        # Dashboard is not serializable
+        attributes['_dash'] = None
+
         # We have to remove the chip's logger before serializing the object
         # since the logger object is not serializable.
         del attributes['logger']

siliconcompiler/remote/client.py

@@ -178,7 +178,7 @@ def _log_truncated_stats(chip, status, nodes_with_status, nodes_to_print):
 
 
 ###################################
-def _process_progress_info(chip, progress_info, nodes_to_print=3):
+def _process_progress_info(chip, progress_info, recorded_nodes, all_nodes, nodes_to_print=3):
     '''
     Helper method to log information about a remote run's progress,
     based on information returned from a 'check_progress/' call.
@@ -201,6 +201,11 @@ def _process_progress_info(chip, progress_info, nodes_to_print=3):
                 # collect completed
                 completed.append(node)
 
+            if node in all_nodes:
+                step, index = all_nodes[node]
+                if (step, index) not in recorded_nodes:
+                    chip.set('record', 'status', status, step=step, index=index)
+
         nodes_to_log = {key: nodes_to_log[key] for key in sorted(nodes_to_log.keys())}
 
         # Log information about the job's progress.
@@ -360,11 +365,32 @@ def __remote_run_loop(chip, check_interval):
     completed = []
     result_procs = []
 
+    recorded = []
+
     for step, index in nodes_to_execute(chip):
         if SCNodeStatus.is_done(chip.get('record', 'status', step=step, index=index)):
             continue
         all_nodes[f'{step}{index}'] = (step, index)
 
+    def import_manifests():
+        changed = False
+        for step, index in all_nodes.values():
+            if (step, index) in recorded:
+                continue
+
+            manifest = os.path.join(chip.getworkdir(step=step, index=index),
+                                    'outputs',
+                                    f'{chip.design}.pkg.json')
+            if os.path.exists(manifest):
+                try:
+                    chip.schema.read_journal(manifest)
+                    recorded.append((step, index))
+                    changed = True
+                except:  # noqa E722
+                    # Import may fail if file is still getting written
+                    pass
+        return changed
+
     def schedule_download(node):
         node_proc = multiprocessor.Process(target=fetch_results,
                                            args=(chip, node))
@@ -376,7 +402,12 @@ def __remote_run_loop(chip, check_interval):
 
     while is_busy:
         time.sleep(check_interval)
-        new_completed, is_busy = check_progress(chip)
+        import_manifests()
+        new_completed, is_busy = check_progress(chip, recorded, all_nodes)
+
+        if chip._dash:
+            chip._dash.update_manifest()
+
         nodes_to_fetch = []
         for node in new_completed:
             if node not in completed:
@@ -400,26 +431,26 @@ def __remote_run_loop(chip, check_interval):
         proc.join()
 
     # Read in node manifests
-    for step, index in all_nodes.values():
-        manifest = os.path.join(chip.getworkdir(step=step, index=index),
-                                'outputs',
-                                f'{chip.design}.pkg.json')
-        if os.path.exists(manifest):
-            chip.schema.read_journal(manifest)
+    import_manifests()
 
     # Un-set the 'remote' option to avoid from/to-based summary/show errors
     chip.unset('option', 'remote')
 
+    if chip._dash:
+        chip._dash.update_manifest()
+
 
 ###################################
-def check_progress(chip):
+def check_progress(chip, recorded_nodes, all_nodes):
     try:
         is_busy_info = is_job_busy(chip)
         is_busy = is_busy_info['busy']
         completed = []
         if is_busy:
             completed = _process_progress_info(chip,
-                                               is_busy_info)
+                                               is_busy_info,
+                                               recorded_nodes,
+                                               all_nodes)
         return completed, is_busy
     except Exception as e:
         # Sometimes an exception is raised if the request library cannot

siliconcompiler/report/__init__.py

@@ -1,7 +1,7 @@
 from .summary_image import _generate_summary_image, _open_summary_image
 from .html_report import _generate_html_report, _open_html_report
 from .summary_table import _show_summary_table
-from .streamlit_report import Dashboard
+from .dashboard import Dashboard
 
 __all__ = [
     "_generate_summary_image",

siliconcompiler/report/{streamlit_report.py → dashboard/__init__.py}

@@ -10,11 +10,20 @@ import multiprocessing
 import subprocess
 import atexit
 import shutil
+import fasteners
+import signal
+
+from siliconcompiler.report.dashboard import utils
 
 
 class Dashboard():
     __port = 8501
 
+    @staticmethod
+    def __signal_handler(signal, frame):
+        # used to avoid issues during shutdown
+        pass
+
     def __init__(self, chip, port=None, graph_chips=None):
         if not port:
             port = Dashboard.__port
@@ -24,16 +33,18 @@ class Dashboard():
         self.__directory = tempfile.mkdtemp(prefix='sc_dashboard_',
                                             suffix=f'_{self.__chip.design}')
         self.__manifest = os.path.join(self.__directory, 'manifest.json')
+        self.__manifest_lock = os.path.join(self.__directory, 'manifest.lock')
         self.__port = port
         dirname = os.path.dirname(__file__)
-        self.__streamlit_file = os.path.join(dirname, 'streamlit_viewer.py')
+        self.__streamlit_file = os.path.join(dirname, 'viewer.py')
 
         self.__streamlit_args = [
             ("browser.gatherUsageStats", False),
             ("browser.serverPort", self.__port),
             ("logger.level", 'error'),
             ("runner.fastReruns", True),
-            ("server.port", self.__port)
+            ("server.port", self.__port),
+            ("client.toolbarMode", "viewer")
         ]
 
         # pass in a json object called __graph_chips
@@ -42,20 +53,33 @@ class Dashboard():
 
         # use of list is to preserve order
         self.__graph_chips = []
-        self.__graph_chips_names = []
+        graph_chips_config = []
         if graph_chips:
             for chip_object_and_name in graph_chips:
                 chip_file_path = \
                     os.path.join(self.__directory,
                                  f"{chip_object_and_name['name']}.json")
-                self.__graph_chips.append({'chip': chip_object_and_name['chip'],
-                                           'name': chip_file_path})
-                self.__graph_chips_names.append(chip_file_path)
-
-        self.__config = {"manifest": self.__manifest,
-                         "graph_chips": self.__graph_chips_names}
+                self.__graph_chips.append({
+                    'chip': chip_object_and_name['chip'],
+                    'name': chip_file_path
+                })
+                graph_chips_config.append({
+                    "path": chip_file_path,
+                    "cwd": utils.get_chip_cwd(
+                        chip_object_and_name['chip'],
+                        chip_object_and_name['cfg_path'])
+                })
+
+        self.__config = {
+            "manifest": self.__manifest,
+            "lock": self.__manifest_lock,
+            "graph_chips": graph_chips_config
+        }
 
         self.__sleep_time = 0.5
+        self.__signal_handler = None
+
+        self.__lock = fasteners.InterProcessLock(self.__manifest_lock)
 
         atexit.register(self.__cleanup)
 
@@ -70,10 +94,19 @@ class Dashboard():
         self.__dashboard = multiprocessing.Process(
             target=self._run_streamlit_bootstrap)
 
+        self.__signal_handler = signal.signal(signal.SIGINT, Dashboard.__signal_handler)
+
         self.__dashboard.start()
 
     def update_manifest(self):
-        self.__chip.write_manifest(self.__manifest)
+        if not self.__manifest:
+            return
+
+        new_file = f"{self.__manifest}.new.json"
+        self.__chip.write_manifest(new_file)
+
+        with self.__lock:
+            shutil.move(new_file, self.__manifest)
 
     def update_graph_manifests(self):
         for chip_object_and_name in self.__graph_chips:
@@ -103,8 +136,12 @@ class Dashboard():
             self.__dashboard.terminate()
             self._sleep()
 
+        if self.__signal_handler:
+            signal.signal(signal.SIGINT, self.__signal_handler)
+
         self.__dashboard = None
         self.__manifest = None
+        self.__signal_handler = None
 
     def wait(self):
         self.__dashboard.join()