siliconcompiler 0.32.1__py3-none-any.whl → 0.32.3__py3-none-any.whl

siliconcompiler/report/dashboard/web/__init__.py

@@ -0,0 +1,196 @@
+import os
+import time
+import tempfile
+import json
+
+import multiprocessing
+import subprocess
+import atexit
+import shutil
+import fasteners
+import signal
+import socketserver
+
+from siliconcompiler.report.dashboard import AbstractDashboard
+from siliconcompiler.report.dashboard.web import utils
+
+try:
+    from streamlit.web import bootstrap
+    from streamlit import config as _config
+except ModuleNotFoundError:
+    bootstrap = None
+    _config = None
+
+
+class WebDashboard(AbstractDashboard):
+    __port = 8501
+
+    @staticmethod
+    def __signal_handler(signal, frame):
+        # used to avoid issues during shutdown
+        pass
+
+    def __init__(self, chip, port=None, graph_chips=None):
+        super().__init__(chip)
+
+        if not bootstrap:
+            raise NotImplementedError('streamlit is not available')
+
+        if not port:
+            port = WebDashboard.get_next_port()
+        if not port:
+            port = WebDashboard.__port
+
+        self.__dashboard = None
+        self.__chip = chip
+        self.__directory = tempfile.mkdtemp(prefix='sc_dashboard_',
+                                            suffix=f'_{self.__chip.design}')
+        self.__manifest = os.path.join(self.__directory, 'manifest.json')
+        self.__manifest_lock = os.path.join(self.__directory, 'manifest.lock')
+        self.__port = port
+        dirname = os.path.dirname(__file__)
+        self.__streamlit_file = os.path.join(dirname, 'viewer.py')
+
+        self.__streamlit_args = [
+            ("browser.gatherUsageStats", False),
+            ("browser.serverPort", self.__port),
+            ("logger.level", 'error'),
+            ("runner.fastReruns", True),
+            ("server.port", self.__port),
+            ("client.toolbarMode", "viewer")
+        ]
+
+        # pass in a json object called __graph_chips
+        # the key is the chip_name and value is the filepath
+        # if another argument is passed
+
+        # use of list is to preserve order
+        self.__graph_chips = []
+        graph_chips_config = []
+        if graph_chips:
+            for chip_object_and_name in graph_chips:
+                chip_file_path = \
+                    os.path.join(self.__directory,
+                                 f"{chip_object_and_name['name']}.json")
+                self.__graph_chips.append({
+                    'chip': chip_object_and_name['chip'],
+                    'name': chip_file_path
+                })
+                graph_chips_config.append({
+                    "path": chip_file_path,
+                    "cwd": utils.get_chip_cwd(
+                        chip_object_and_name['chip'],
+                        chip_object_and_name['cfg_path'])
+                })
+
+        self.__config = {
+            "manifest": self.__manifest,
+            "lock": self.__manifest_lock,
+            "graph_chips": graph_chips_config
+        }
+
+        self.__sleep_time = 0.5
+        self.__signal_handler = None
+
+        self.__lock = fasteners.InterProcessLock(self.__manifest_lock)
+
+        atexit.register(self.__cleanup)
+
+    def open_dashboard(self):
+        with open(self.__get_config_file(), 'w') as f:
+            json.dump(self.__config, f, indent=4)
+
+        self.update_manifest()
+
+        self.update_graph_manifests()
+
+        self.__dashboard = multiprocessing.Process(
+            target=self._run_streamlit_bootstrap)
+
+        self.__signal_handler = signal.signal(signal.SIGINT, WebDashboard.__signal_handler)
+
+        self.__dashboard.start()
+
+    def update_manifest(self, payload=None):
+        if not self.__manifest:
+            return
+
+        new_file = f"{self.__manifest}.new.json"
+        self.__chip.write_manifest(new_file)
+
+        with self.__lock:
+            shutil.move(new_file, self.__manifest)
+
+    def update_graph_manifests(self):
+        for chip_object_and_name in self.__graph_chips:
+            chip = chip_object_and_name['chip']
+            file_path = chip_object_and_name['name']
+            chip.write_manifest(file_path)
+
+    def __get_config_file(self):
+        return os.path.join(self.__directory, 'config.json')
+
+    def is_running(self):
+        if self.__dashboard is None:
+            return False
+
+        if self.__dashboard.is_alive():
+            return True
+
+        self.__dashboard = None
+        self.__manifest = None
+        return False
+
+    def end_of_run(self):
+        pass
+
+    def stop(self):
+        if not self.is_running():
+            return
+
+        while self.__dashboard.is_alive():
+            self.__dashboard.terminate()
+            self._sleep()
+
+        if self.__signal_handler:
+            signal.signal(signal.SIGINT, self.__signal_handler)
+
+        self.__dashboard = None
+        self.__manifest = None
+        self.__signal_handler = None
+
+    def wait(self):
+        self.__dashboard.join()
+
+    def _sleep(self):
+        time.sleep(self.__sleep_time)
+
+    def _run_streamlit_bootstrap(self):
+        for config, val in self.__streamlit_args:
+            _config.set_option(config, val)
+
+        bootstrap.run(self.__streamlit_file,
+                      False,
+                      [self.__get_config_file()],
+                      flag_options={})
+
+    def __run_streamlit_subproc(self):
+        cmd = ['streamlit', 'run',
+               self.__streamlit_file, self.__get_config_file()]
+        for config, val in self.__streamlit_args:
+            cmd.append(f'--{config}')
+            cmd.append(val)
+
+        subprocess.Popen(cmd)
+
+    def __cleanup(self):
+        self.stop()
+
+        if os.path.exists(self.__directory):
+            shutil.rmtree(self.__directory)
+
+    @staticmethod
+    def get_next_port():
+        with socketserver.TCPServer(("localhost", 0), None) as s:
+            return s.server_address[1]
+        return None

siliconcompiler/report/dashboard/{components → web/components}/__init__.py

@@ -14,10 +14,10 @@ from siliconcompiler import __version__ as sc_version
 from siliconcompiler import utils
 from siliconcompiler.report import report
 
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import layouts
-from siliconcompiler.report.dashboard.utils import file_utils
-from siliconcompiler.report.dashboard.components import flowgraph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import layouts
+from siliconcompiler.report.dashboard.web.utils import file_utils
+from siliconcompiler.report.dashboard.web.components import flowgraph
 
 
 SC_ABOUT = [

siliconcompiler/report/dashboard/{components → web/components}/graph.py

@@ -5,7 +5,7 @@ import streamlit
 
 from siliconcompiler.report import report
 
-from siliconcompiler.report.dashboard import state
+from siliconcompiler.report.dashboard.web import state
 
 
 def _get_report_chips():

siliconcompiler/report/dashboard/{layouts → web/layouts}/__init__.py

@@ -1,6 +1,6 @@
-from siliconcompiler.report.dashboard.layouts import vertical_flowgraph
-from siliconcompiler.report.dashboard.layouts import vertical_flowgraph_sac_tabs
-from siliconcompiler.report.dashboard.layouts import vertical_flowgraph_node_tab
+from siliconcompiler.report.dashboard.web.layouts import vertical_flowgraph
+from siliconcompiler.report.dashboard.web.layouts import vertical_flowgraph_sac_tabs
+from siliconcompiler.report.dashboard.web.layouts import vertical_flowgraph_node_tab
 
 __LAYOUTS = {
     "vertical_flowgraph": vertical_flowgraph.layout,

siliconcompiler/report/dashboard/{layouts → web/layouts}/_common.py

@@ -1,6 +1,6 @@
 import streamlit_antd_components as sac
 
-from siliconcompiler.report.dashboard import state
+from siliconcompiler.report.dashboard.web import state
 
 
 def check_rerun():

siliconcompiler/report/dashboard/{layouts → web/layouts}/vertical_flowgraph.py

@@ -1,11 +1,11 @@
 import os
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard.components import graph
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
-from siliconcompiler.report.dashboard.layouts import _common
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web.components import graph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
+from siliconcompiler.report.dashboard.web.layouts import _common
 
 
 def layout():

siliconcompiler/report/dashboard/{layouts → web/layouts}/vertical_flowgraph_node_tab.py

@@ -1,12 +1,12 @@
 import os
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard.components import graph
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
-from siliconcompiler.report.dashboard.utils import file_utils
-from siliconcompiler.report.dashboard.layouts import _common
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web.components import graph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
+from siliconcompiler.report.dashboard.web.utils import file_utils
+from siliconcompiler.report.dashboard.web.layouts import _common
 import streamlit_antd_components as sac
 
 

siliconcompiler/report/dashboard/{layouts → web/layouts}/vertical_flowgraph_sac_tabs.py

@@ -1,12 +1,12 @@
 import os
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard.components import graph
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
-from siliconcompiler.report.dashboard.utils import file_utils
-from siliconcompiler.report.dashboard.layouts import _common
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web.components import graph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
+from siliconcompiler.report.dashboard.web.utils import file_utils
+from siliconcompiler.report.dashboard.web.layouts import _common
 import streamlit_antd_components as sac
 
 

siliconcompiler/report/dashboard/{viewer.py → web/viewer.py}

@@ -1,9 +1,9 @@
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard import layouts
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web import layouts
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
 
 import streamlit_autorefresh
 

siliconcompiler/scheduler/__init__.py

@@ -2,6 +2,7 @@ import contextlib
 import distro
 import getpass
 import multiprocessing
+import logging
 import os
 import platform
 import psutil
@@ -17,6 +18,7 @@ import packaging.specifiers
 from io import StringIO
 import traceback
 from datetime import datetime
+from logging.handlers import QueueHandler, QueueListener
 from siliconcompiler import sc_open
 from siliconcompiler import utils
 from siliconcompiler import _metadata
@@ -29,6 +31,7 @@ from siliconcompiler.flowgraph import _get_flowgraph_nodes, _get_flowgraph_execu
     _get_pruned_node_inputs, _get_flowgraph_entry_nodes, \
     _unreachable_steps_to_execute, _nodes_to_execute, \
     get_nodes_from, nodes_to_execute, _check_flowgraph
+from siliconcompiler.utils.logging import SCBlankLoggerFormatter
 from siliconcompiler.tools._common import input_file_node_name
 import lambdapdk
 from siliconcompiler.tools._common import get_tool_task, record_metric
@@ -85,6 +88,8 @@ def run(chip):
 
     # Re-init logger to include run info after setting up flowgraph.
     chip._init_logger(in_run=True)
+    if chip._dash and not chip._dash.is_running():
+        chip._dash.open_dashboard()
 
     # Check if flowgraph is complete and valid
     flow = chip.get('option', 'flow')
@@ -128,10 +133,6 @@ def _finalize_run(chip):
     filepath = os.path.join(chip.getworkdir(), f"{chip.design}.pkg.json")
     chip.write_manifest(filepath)
 
-    # Update dashboard
-    if chip._dash:
-        chip._dash.update_manifest()
-
     send_messages.send(chip, 'summary', None, None)
 
 
@@ -275,7 +276,12 @@ def _local_process(chip, flow):
     nodes_to_run = {}
     processes = {}
     local_processes = []
-    _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow)
+    log_queue = _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow)
+
+    # Handle logs across threads
+    log_listener = QueueListener(log_queue, chip.logger._console)
+    chip.logger._console.setFormatter(SCBlankLoggerFormatter())
+    log_listener.start()
 
     # Update dashboard before run begins
     if chip._dash:
@@ -285,6 +291,7 @@ def _local_process(chip, flow):
         _launch_nodes(chip, nodes_to_run, processes, local_processes)
     except KeyboardInterrupt:
         # exit immediately
+        log_listener.stop()
         sys.exit(0)
 
     if _get_callback('post_run'):
@@ -292,6 +299,10 @@ def _local_process(chip, flow):
 
     _check_nodes_status(chip, flow)
 
+    # Cleanup logger
+    log_listener.stop()
+    chip._init_logger_formats()
+
 
 def __is_posix():
     return sys.platform != 'win32'
@@ -421,7 +432,7 @@ def _check_version(chip, reported_version, tool, step, index):
 
 
 ###########################################################################
-def _runtask(chip, flow, step, index, exec_func, pipe=None, replay=False):
+def _runtask(chip, flow, step, index, exec_func, pipe=None, queue=None, replay=False):
     '''
     Private per node run method called by run().
 
@@ -437,6 +448,11 @@ def _runtask(chip, flow, step, index, exec_func, pipe=None, replay=False):
     chip._init_codecs()
 
     chip._init_logger(step, index, in_run=True)
+    if queue:
+        chip.logger.removeHandler(chip.logger._console)
+        chip.logger._console = QueueHandler(queue)
+        chip.logger.addHandler(chip.logger._console)
+        chip._init_logger_formats()
 
     chip.set('arg', 'step', step, clobber=True)
     chip.set('arg', 'index', index, clobber=True)
@@ -541,13 +557,12 @@ def _select_inputs(chip, step, index, trial=False):
                             '_select_inputs',
                             None)
     if select_inputs:
-        log_handlers = None
+        log_level = chip.logger.level
         if trial:
-            log_handlers = chip.logger.handlers.copy()
-            chip.logger.handlers.clear()
+            chip.logger.setLevel(logging.CRITICAL)
         sel_inputs = select_inputs(chip, step, index)
-        if log_handlers:
-            chip.logger.handlers = log_handlers
+        if trial:
+            chip.logger.setLevel(log_level)
     else:
         sel_inputs = _get_pruned_node_inputs(chip, flow, (step, index))
 
@@ -862,9 +877,16 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                     stderr_writer.close()
                     stderr_writer = sys.stdout
 
+                # Handle logger stdout suppression if quiet
+                stdout_handler_level = chip.logger._console.level
+                if chip.get('option', 'quiet', step=step, index=index):
+                    chip.logger._console.setLevel(logging.CRITICAL)
+
                 with contextlib.redirect_stderr(stderr_writer), \
                         contextlib.redirect_stdout(stdout_writer):
                     retcode = run_func(chip)
+
+                chip.logger._console.setLevel(stdout_handler_level)
         except Exception as e:
             chip.logger.error(f'Failed in run() for {tool}/{task}: {e}')
             retcode = 1  # default to non-zero
@@ -1491,6 +1513,9 @@ def _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow):
     # Call this in case this was invoked without __main__
     multiprocessing.freeze_support()
 
+    # Log queue for logging messages
+    log_queue = multiprocessing.Queue(-1)
+
     init_funcs = set()
     for (step, index) in nodes_to_execute(chip, flow):
         node = (step, index)
@@ -1525,13 +1550,16 @@ def _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow):
         process["proc"] = multiprocessing.Process(
             target=_runtask,
             args=(chip, flow, step, index, exec_func),
-            kwargs={"pipe": process["child_pipe"]})
+            kwargs={"pipe": process["child_pipe"],
+                    "queue": log_queue})
 
         processes[node] = process
 
     for init_func in init_funcs:
         init_func(chip)
 
+    return log_queue
+
 
 def _check_node_dependencies(chip, node, deps, deps_was_successful):
     had_deps = len(deps) > 0
@@ -1600,6 +1628,8 @@ def _launch_nodes(chip, nodes_to_run, processes, local_processes):
     if _get_callback('pre_run'):
         _get_callback('pre_run')(chip)
 
+    start_times = {None: time.time()}
+
     while len(nodes_to_run) > 0 or len(running_nodes) > 0:
         changed = _process_completed_nodes(chip, processes, running_nodes)
 
@@ -1624,6 +1654,7 @@ def _launch_nodes(chip, nodes_to_run, processes, local_processes):
                         _get_callback('pre_node')(chip, *node)
 
                     chip.set('record', 'status', NodeStatus.RUNNING, step=node[0], index=node[1])
+                    start_times[node] = time.time()
                     changed = True
 
                     processes[node]["proc"].start()
@@ -1640,7 +1671,7 @@ def _launch_nodes(chip, nodes_to_run, processes, local_processes):
 
         if chip._dash and changed:
             # Update dashboard if the manifest changed
-            chip._dash.update_manifest()
+            chip._dash.update_manifest(payload={"starttimes": start_times})
 
         if len(running_nodes) == 1:
             # if there is only one node running, just join the thread

siliconcompiler/scheduler/docker_runner.py

@@ -115,7 +115,8 @@ def run(chip, step, index, replay):
     start_cwd = os.getcwd()
 
     # Remove handlers from logger
-    chip.logger.handlers.clear()
+    for handler in chip.logger.handlers.copy():
+        chip.logger.removeHandler(handler)
 
     # Reinit logger
     chip._init_logger(step=step, index=index, in_run=True)

siliconcompiler/sphinx_ext/dynamicgen.py

@@ -325,7 +325,7 @@ class DynamicGen(SphinxDirective):
 
         return True
 
-    def _document_free_params(self, cfg, type, key_path, reference_prefix, s):
+    def _document_free_params(self, cfg, type, key_path, reference_prefix, s, show_type=False):
         if type in cfg:
             cfg = cfg[type]
         else:
@@ -337,8 +337,12 @@ class DynamicGen(SphinxDirective):
             type_heading = "Files"
         elif type == "dir":
             type_heading = "Directories"
+        else:
+            raise ValueError(type)
 
         table = [[strong('Parameters'), strong('Help')]]
+        if show_type:
+            table[0].insert(1, strong('Type'))
         for key, params in cfg.items():
             if key == "default":
                 continue
@@ -346,7 +350,10 @@ class DynamicGen(SphinxDirective):
             key_node = nodes.paragraph()
             key_node += keypath(key_path + [key], self.env.docname,
                                 key_text=["...", f"'{type}'", f"'{key}'"])
-            table.append([key_node, para(params["help"])])
+            entry = [key_node, para(params["help"])]
+            if show_type:
+                entry.insert(1, code(params["type"]))
+            table.append(entry)
 
         if len(table) > 1:
             s += build_section(type_heading, self.get_ref(*reference_prefix, type))
@@ -710,14 +717,6 @@ class ToolGen(DynamicGen):
         self.__tool = None
         self.__task = None
 
-        # Annotate the target used for default values
-        if chip.valid('option', 'target') and chip.get('option', 'target'):
-            p = docutils.nodes.inline('')
-            target = chip.get('option', 'target').split('.')[-1]
-            targetid = get_ref_id(DynamicGen.get_ref_key(TargetGen.REF_PREFIX, target))
-            self.parse_rst(f"Built using target: :ref:`{target}<{targetid}>`", p)
-            s += p
-
         try:
             task_setup(chip)
 
@@ -780,7 +779,8 @@ class ToolGen(DynamicGen):
 
     def document_free_params(self, cfg, reference_prefix, s):
         key_path = ['tool', '<tool>', 'task', '<task>']
-        self._document_free_params(cfg, 'var', key_path + ['var'], reference_prefix, s)
+        self._document_free_params(cfg, 'var', key_path + ['var'], reference_prefix, s,
+                                   show_type=True)
         self._document_free_params(cfg, 'file', key_path + ['file'], reference_prefix, s)
         self._document_free_params(cfg, 'dir', key_path + ['dir'], reference_prefix, s)
 

siliconcompiler/templates/replay/replay.sh.j2

@@ -60,10 +60,10 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-if [ $SKIPEXPORT == 0 ]; then
+{% if exports|length > 0 %}if [ $SKIPEXPORT == 0 ]; then
     # Environmental variables{% for key, value in exports.items() %}
     export {{ key }}="{{ value }}"{% endfor %}
-fi
+fi{% endif %}
 
 # Switch to the working directory
 cd "$CD_WORK"