siliconcompiler 0.28.1__py3-none-any.whl → 0.28.3__py3-none-any.whl

siliconcompiler/_common.py

@@ -33,6 +33,18 @@ class NodeStatus():
             NodeStatus.PENDING,
         )
 
+    def is_success(status):
+        return status in (
+            NodeStatus.SUCCESS,
+            NodeStatus.SKIPPED
+        )
+
+    def is_error(status):
+        return status in (
+            NodeStatus.ERROR,
+            NodeStatus.TIMEOUT
+        )
+
 
 ###############################################################################
 # Package Customization classes

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.28.1'
+version = '0.28.3'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_dashboard.py

@@ -82,7 +82,11 @@ To include another chip object to compare to:
                 raise ValueError(f'not a valid file path : {file_path}')
             graph_chip = siliconcompiler.core.Chip(design='')
             graph_chip.read_manifest(file_path)
-            graph_chips.append({'chip': graph_chip, 'name': name})
+            graph_chips.append({
+                'chip': graph_chip,
+                'name': name,
+                'cfg_path': os.path.abspath(file_path)
+            })
 
     chip._dashboard(wait=True, port=switches['port'], graph_chips=graph_chips)
 

siliconcompiler/apps/sc_install.py

@@ -0,0 +1,240 @@
+# Copyright 2024 Silicon Compiler Authors. All Rights Reserved.
+
+import argparse
+import glob
+import subprocess
+import sys
+import shutil
+import re
+import os.path
+from collections.abc import Container
+from pathlib import Path
+import siliconcompiler
+from siliconcompiler.scheduler import _get_machine_info
+
+
+class ChoiceOptional(Container):
+    def __init__(self, choices):
+        super().__init__()
+
+        self.__choices = set(choices)
+
+    def __contains__(self, item):
+        if not item:
+            # allow empty value
+            return True
+        return item in self.__choices
+
+    def __iter__(self):
+        return self.__choices.__iter__()
+
+    def get_items(self, choices):
+        items = set(choices)
+        return sorted(list(items))
+
+
+def install_tool(tool, script, build_dir, prefix):
+    # Ensure build dir is available
+    build_dir = Path(build_dir) / tool
+    shutil.rmtree(str(build_dir), ignore_errors=True)
+    build_dir.mkdir(parents=True, exist_ok=True)
+
+    # setup environment
+    env = os.environ.copy()
+    env["PREFIX"] = prefix
+
+    # run
+    ret = subprocess.call(script, env=env, cwd=build_dir)
+    if ret != 0:
+        print(f"Error occurred while building/installing {tool}")
+        return False
+    return True
+
+
+def show_tool(tool, script):
+    def print_header(head):
+        border_len = max(80, len(script) + 2)
+        border = border_len*"#"
+        print(border)
+        print(f"# {tool} script / {head}")
+        if head == "start":
+            print(f"# {script}")
+        print(border)
+
+    print_header("start")
+
+    with open(script) as f:
+        for line in f:
+            print(line.rstrip())
+
+    print_header("end")
+
+
+def _get_os_name():
+    machine_info = _get_machine_info()
+    system = machine_info.get('system', "").lower()
+    distro = machine_info.get('distro', "").lower()
+    osversion = machine_info.get('osversion', "").lower()
+    if system == 'linux':
+        if distro == 'ubuntu':
+            version, _ = osversion.split('.')
+            return f"{distro}{version}"
+        elif distro == 'rocky':
+            version, _ = osversion.split('.')
+            return f"rhel{version}"
+        elif distro == 'rhel':
+            version, _ = osversion.split('.')
+            return f"rhel{version}"
+    return None
+
+
+def print_machine_info():
+    machine_info = _get_machine_info()
+    mapped_os = _get_os_name()
+
+    print("System:   ", machine_info.get('system', None))
+    print("Distro:   ", machine_info.get('distro', None))
+    print("Version:  ", machine_info.get('osversion', None))
+    print("Mapped OS:", mapped_os)
+    print("Scripts:  ", _get_tool_script_dir())
+
+
+def _get_tool_script_dir():
+    return Path(siliconcompiler.__file__).parent / "toolscripts"
+
+
+def _get_tools_list():
+    tools_root = _get_tool_script_dir()
+
+    script_dir = None
+    os_dir = _get_os_name()
+    if os_dir:
+        script_dir = tools_root / os_dir
+        if not script_dir.exists():
+            script_dir = None
+
+    tools = {}
+    if script_dir:
+        for script in glob.glob(str(script_dir / "install-*.sh")):
+            tool = re.match(r"install-(.*)\.sh", os.path.basename(script).lower())
+            tools[tool.group(1)] = script
+
+    return tools
+
+
+def _recommended_tool_groups(tools):
+    groups = {
+        "asic": {"surelog", "sv2v", "yosys", "openroad", "klayout"},
+        "fpga": {"surelog", "sv2v", "yosys", "vpr"},
+        "digital-simulation": {"verilator", "icarus"},
+        "analog-simulation": {"xyce"}
+    }
+
+    filter_groups = {}
+    for group, group_tools in groups.items():
+        if all([tool in tools for tool in group_tools]):
+            filter_groups[group] = group_tools
+    return filter_groups
+
+
+def main():
+    progname = "sc-install"
+    description = """
+-----------------------------------------------------------
+SC app install supported tools.
+
+To install a single tool:
+    sc-install openroad
+
+To install multiple tools:
+    sc-install openroad yosys klayout
+
+To install a group of tools:
+    sc-install -group asic
+
+To install tools in a different location:
+    sc-install -prefix /usr/local yosys
+
+To build tools in a different location:
+    sc-install -build_dir /tmp yosys
+
+To show the install script:
+    sc-install -show openroad
+
+To system debugging information (this should only be used to debug):
+    sc-install -debug_machine
+-----------------------------------------------------------
+"""
+    parser = argparse.ArgumentParser(
+        prog=progname,
+        description=description,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    tools = _get_tools_list()
+
+    tool_choices = ChoiceOptional(tools.keys())
+    parser.add_argument(
+        "tool",
+        nargs="*",
+        choices=tool_choices,
+        help="tool to install")
+
+    tool_groups = _recommended_tool_groups(tools)
+    parser.add_argument(
+        "-group",
+        nargs="+",
+        choices=tool_groups.keys(),
+        help=f"tool group to install{' - not supported' if not tool_groups else ''}")
+
+    parser.add_argument(
+        "-prefix",
+        default=Path.home() / ".local",
+        help="Prefix to use when installing tool",
+        metavar="<path>")
+
+    parser.add_argument(
+        "-build_dir",
+        default=Path.home() / ".sc" / "tool_build",
+        help="Directory to build the tool in",
+        metavar="<path>")
+
+    parser.add_argument(
+        "-show",
+        action="store_true",
+        help="Show the install script and exit")
+
+    parser.add_argument(
+        "-debug_machine",
+        action="store_true",
+        help="Show information about this machine and exit")
+
+    args = parser.parse_args()
+
+    if args.debug_machine:
+        print_machine_info()
+        return 0
+
+    if not args.tool:
+        args.tool = []
+
+    args.tool = list(args.tool)
+    if args.group:
+        for group in args.group:
+            args.tool.extend(tool_groups[group])
+
+    tools_handled = set()
+    for tool in args.tool:
+        if tool in tools_handled:
+            continue
+        tools_handled.add(tool)
+        if args.show:
+            show_tool(tool, tools[tool])
+        else:
+            if not install_tool(tool, tools[tool], args.build_dir, args.prefix):
+                return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

siliconcompiler/apps/sc_remote.py

@@ -185,7 +185,7 @@ To delete a job, use:
     # If only a manifest is specified, make a 'check_progress/' request and report results:
     elif chip_cfg:
         try:
-            check_progress(chip)
+            check_progress(chip, [], {})
         except SiliconCompilerError as e:
             chip.logger.error(f'{e}')
             return 1

siliconcompiler/core.py

@@ -88,6 +88,9 @@ class Chip:
         # Cache of file hashes
         self.__hashes = {}
 
+        # Dashboard
+        self._dash = None
+
         # Showtools
         self._showtools = {}
         for plugin in utils.get_plugins('show'):
@@ -758,7 +761,7 @@ class Chip:
         return fullstr
 
     ###########################################################################
-    def valid(self, *keypath, default_valid=False, job=None):
+    def valid(self, *keypath, default_valid=False, job=None, check_complete=False):
         """
         Checks validity of a keypath.
 
@@ -770,6 +773,7 @@ class Chip:
             keypaths as a wildcard. Defaults to False.
             job (str): Jobname to use for dictionary access in place of the
                 current active jobname.
+            check_complete (bool): Require the keypath be a complete path.
 
         Returns:
             Boolean indicating validity of keypath.
@@ -782,7 +786,10 @@ class Chip:
             >>> check = chip.valid('metric', 'foo', '0', 'tasktime', default_valid=True)
             Returns True, even if "foo" and "0" aren't in current configuration.
         """
-        return self.schema.valid(*keypath, default_valid=default_valid, job=job)
+        return self.schema.valid(*keypath,
+                                 default_valid=default_valid,
+                                 job=job,
+                                 check_complete=check_complete)
 
     ###########################################################################
     def get(self, *keypath, field='value', job=None, step=None, index=None):
@@ -1108,6 +1115,24 @@ class Chip:
         Performs a lookup in the io map for the fileset and filetype
         and will use those if they are not provided in the arguments
         '''
+        # Handle list inputs
+        if isinstance(filename, (list, tuple)):
+            for file in filename:
+                self._add_input_output(
+                    category,
+                    file,
+                    fileset=fileset,
+                    filetype=filetype,
+                    iomap=iomap,
+                    step=step,
+                    index=index,
+                    package=package,
+                    quiet=quiet)
+            return
+
+        if filename is None:
+            raise ValueError(f"{category} cannot process None")
+
         # Normalize value to string in case we receive a pathlib.Path
         filename = str(filename)
 
@@ -1132,8 +1157,9 @@ class Chip:
             use_filetype = filetype
 
         if not use_fileset or not use_filetype:
-            self.logger.error(f'Unable to infer {category} fileset and/or filetype for '
-                              f'{filename} based on file extension.')
+            raise SiliconCompilerError(
+                f'Unable to infer {category} fileset and/or filetype for '
+                f'{filename} based on file extension.')
         elif not quiet:
             if not fileset and not filetype:
                 self.logger.info(f'{filename} inferred as {use_fileset}/{use_filetype}')
@@ -2695,18 +2721,25 @@ class Chip:
             >>> chip._dashboard()
             Opens a sesison of the dashboard.
         '''
-        dash = Dashboard(self, port=port, graph_chips=graph_chips)
-        dash.open_dashboard()
+        if self._dash:
+            # Remove previous dashboard
+            self._dash.stop()
+            self._dash = None
+
+        self._dash = Dashboard(self, port=port, graph_chips=graph_chips)
+        self._dash.open_dashboard()
+
         if wait:
             try:
-                dash.wait()
+                self._dash.wait()
             except KeyboardInterrupt:
-                dash._sleep()
+                self._dash._sleep()
             finally:
-                dash.stop()
+                self._dash.stop()
+                self._dash = None
             return None
 
-        return dash
+        return self._dash
 
     ###########################################################################
     def summary(self, show_all_indices=False, generate_image=True, generate_html=True):
@@ -2746,8 +2779,8 @@ class Chip:
         work_dir = self.getworkdir()
         if os.path.isdir(work_dir):
             # Mark file paths where the reports can be found if they were generated.
-            results_html = os.path.join(work_dir, 'report.html')
             results_img = os.path.join(work_dir, f'{self.design}.png')
+            results_html = os.path.join(work_dir, 'report.html')
 
             if generate_image:
                 _generate_summary_image(self, results_img)
@@ -2755,13 +2788,18 @@ class Chip:
             if generate_html:
                 _generate_html_report(self, flow, nodes_to_execute, results_html)
 
+            # dashboard does not generate any data
+            self.logger.info(f'Dashboard at "sc-dashboard -cfg {work_dir}/{self.design}.pkg.json"')
+
             # Try to open the results and layout only if '-nodisplay' is not set.
-            # Priority: PNG, PDF, HTML.
-            if (not self.get('option', 'nodisplay')):
+            # Priority: PNG > HTML > dashboard.
+            if not self.get('option', 'nodisplay'):
                 if os.path.isfile(results_img):
                     _open_summary_image(results_img)
                 elif os.path.isfile(results_html):
                     _open_html_report(self, results_html)
+                else:
+                    self._dashboard(wait=False)
 
     ###########################################################################
     def clock(self, pin, period, jitter=0, mode='global'):
@@ -3258,6 +3296,9 @@ class Chip:
         # Modules are not serializable, so save without cache
         attributes['_showtools'] = {}
 
+        # Dashboard is not serializable
+        attributes['_dash'] = None
+
         # We have to remove the chip's logger before serializing the object
         # since the logger object is not serializable.
         del attributes['logger']

siliconcompiler/remote/client.py

@@ -178,7 +178,7 @@ def _log_truncated_stats(chip, status, nodes_with_status, nodes_to_print):
 
 
 ###################################
-def _process_progress_info(chip, progress_info, nodes_to_print=3):
+def _process_progress_info(chip, progress_info, recorded_nodes, all_nodes, nodes_to_print=3):
     '''
     Helper method to log information about a remote run's progress,
     based on information returned from a 'check_progress/' call.
@@ -201,6 +201,11 @@ def _process_progress_info(chip, progress_info, nodes_to_print=3):
                 # collect completed
                 completed.append(node)
 
+            if node in all_nodes:
+                step, index = all_nodes[node]
+                if (step, index) not in recorded_nodes:
+                    chip.set('record', 'status', status, step=step, index=index)
+
         nodes_to_log = {key: nodes_to_log[key] for key in sorted(nodes_to_log.keys())}
 
         # Log information about the job's progress.
@@ -360,11 +365,32 @@ def __remote_run_loop(chip, check_interval):
     completed = []
     result_procs = []
 
+    recorded = []
+
     for step, index in nodes_to_execute(chip):
         if SCNodeStatus.is_done(chip.get('record', 'status', step=step, index=index)):
             continue
         all_nodes[f'{step}{index}'] = (step, index)
 
+    def import_manifests():
+        changed = False
+        for step, index in all_nodes.values():
+            if (step, index) in recorded:
+                continue
+
+            manifest = os.path.join(chip.getworkdir(step=step, index=index),
+                                    'outputs',
+                                    f'{chip.design}.pkg.json')
+            if os.path.exists(manifest):
+                try:
+                    chip.schema.read_journal(manifest)
+                    recorded.append((step, index))
+                    changed = True
+                except:  # noqa E722
+                    # Import may fail if file is still getting written
+                    pass
+        return changed
+
     def schedule_download(node):
         node_proc = multiprocessor.Process(target=fetch_results,
                                            args=(chip, node))
@@ -376,7 +402,12 @@ def __remote_run_loop(chip, check_interval):
 
     while is_busy:
         time.sleep(check_interval)
-        new_completed, is_busy = check_progress(chip)
+        import_manifests()
+        new_completed, is_busy = check_progress(chip, recorded, all_nodes)
+
+        if chip._dash:
+            chip._dash.update_manifest()
+
         nodes_to_fetch = []
         for node in new_completed:
             if node not in completed:
@@ -400,26 +431,26 @@ def __remote_run_loop(chip, check_interval):
         proc.join()
 
     # Read in node manifests
-    for step, index in all_nodes.values():
-        manifest = os.path.join(chip.getworkdir(step=step, index=index),
-                                'outputs',
-                                f'{chip.design}.pkg.json')
-        if os.path.exists(manifest):
-            chip.schema.read_journal(manifest)
+    import_manifests()
 
     # Un-set the 'remote' option to avoid from/to-based summary/show errors
     chip.unset('option', 'remote')
 
+    if chip._dash:
+        chip._dash.update_manifest()
+
 
 ###################################
-def check_progress(chip):
+def check_progress(chip, recorded_nodes, all_nodes):
     try:
         is_busy_info = is_job_busy(chip)
         is_busy = is_busy_info['busy']
         completed = []
         if is_busy:
             completed = _process_progress_info(chip,
-                                               is_busy_info)
+                                               is_busy_info,
+                                               recorded_nodes,
+                                               all_nodes)
         return completed, is_busy
     except Exception as e:
         # Sometimes an exception is raised if the request library cannot

siliconcompiler/report/__init__.py

@@ -1,7 +1,7 @@
 from .summary_image import _generate_summary_image, _open_summary_image
 from .html_report import _generate_html_report, _open_html_report
 from .summary_table import _show_summary_table
-from .streamlit_report import Dashboard
+from .dashboard import Dashboard
 
 __all__ = [
     "_generate_summary_image",

siliconcompiler/report/{streamlit_report.py → dashboard/__init__.py}

@@ -10,11 +10,20 @@ import multiprocessing
 import subprocess
 import atexit
 import shutil
+import fasteners
+import signal
+
+from siliconcompiler.report.dashboard import utils
 
 
 class Dashboard():
     __port = 8501
 
+    @staticmethod
+    def __signal_handler(signal, frame):
+        # used to avoid issues during shutdown
+        pass
+
     def __init__(self, chip, port=None, graph_chips=None):
         if not port:
             port = Dashboard.__port
@@ -24,16 +33,18 @@ class Dashboard():
         self.__directory = tempfile.mkdtemp(prefix='sc_dashboard_',
                                             suffix=f'_{self.__chip.design}')
         self.__manifest = os.path.join(self.__directory, 'manifest.json')
+        self.__manifest_lock = os.path.join(self.__directory, 'manifest.lock')
         self.__port = port
         dirname = os.path.dirname(__file__)
-        self.__streamlit_file = os.path.join(dirname, 'streamlit_viewer.py')
+        self.__streamlit_file = os.path.join(dirname, 'viewer.py')
 
         self.__streamlit_args = [
             ("browser.gatherUsageStats", False),
             ("browser.serverPort", self.__port),
             ("logger.level", 'error'),
             ("runner.fastReruns", True),
-            ("server.port", self.__port)
+            ("server.port", self.__port),
+            ("client.toolbarMode", "viewer")
         ]
 
         # pass in a json object called __graph_chips
@@ -42,20 +53,33 @@ class Dashboard():
 
         # use of list is to preserve order
         self.__graph_chips = []
-        self.__graph_chips_names = []
+        graph_chips_config = []
         if graph_chips:
             for chip_object_and_name in graph_chips:
                 chip_file_path = \
                     os.path.join(self.__directory,
                                  f"{chip_object_and_name['name']}.json")
-                self.__graph_chips.append({'chip': chip_object_and_name['chip'],
-                                           'name': chip_file_path})
-                self.__graph_chips_names.append(chip_file_path)
-
-        self.__config = {"manifest": self.__manifest,
-                         "graph_chips": self.__graph_chips_names}
+                self.__graph_chips.append({
+                    'chip': chip_object_and_name['chip'],
+                    'name': chip_file_path
+                })
+                graph_chips_config.append({
+                    "path": chip_file_path,
+                    "cwd": utils.get_chip_cwd(
+                        chip_object_and_name['chip'],
+                        chip_object_and_name['cfg_path'])
+                })
+
+        self.__config = {
+            "manifest": self.__manifest,
+            "lock": self.__manifest_lock,
+            "graph_chips": graph_chips_config
+        }
 
         self.__sleep_time = 0.5
+        self.__signal_handler = None
+
+        self.__lock = fasteners.InterProcessLock(self.__manifest_lock)
 
         atexit.register(self.__cleanup)
 
@@ -70,10 +94,19 @@ class Dashboard():
         self.__dashboard = multiprocessing.Process(
             target=self._run_streamlit_bootstrap)
 
+        self.__signal_handler = signal.signal(signal.SIGINT, Dashboard.__signal_handler)
+
         self.__dashboard.start()
 
     def update_manifest(self):
-        self.__chip.write_manifest(self.__manifest)
+        if not self.__manifest:
+            return
+
+        new_file = f"{self.__manifest}.new.json"
+        self.__chip.write_manifest(new_file)
+
+        with self.__lock:
+            shutil.move(new_file, self.__manifest)
 
     def update_graph_manifests(self):
         for chip_object_and_name in self.__graph_chips:
@@ -103,8 +136,12 @@ class Dashboard():
             self.__dashboard.terminate()
             self._sleep()
 
+        if self.__signal_handler:
+            signal.signal(signal.SIGINT, self.__signal_handler)
+
         self.__dashboard = None
         self.__manifest = None
+        self.__signal_handler = None
 
     def wait(self):
         self.__dashboard.join()