siliconcompiler 0.32.2__py3-none-any.whl → 0.33.0__py3-none-any.whl

siliconcompiler/report/dashboard/web/__init__.py

@@ -0,0 +1,196 @@
+import os
+import time
+import tempfile
+import json
+
+import multiprocessing
+import subprocess
+import atexit
+import shutil
+import fasteners
+import signal
+import socketserver
+
+from siliconcompiler.report.dashboard import AbstractDashboard
+from siliconcompiler.report.dashboard.web import utils
+
+
+class WebDashboard(AbstractDashboard):
+    __port = 8501
+
+    @staticmethod
+    def __signal_handler(signal, frame):
+        # used to avoid issues during shutdown
+        pass
+
+    def __init__(self, chip, port=None, graph_chips=None):
+        try:
+            from streamlit.web import bootstrap  # noqa: F401
+        except ModuleNotFoundError:
+            raise NotImplementedError('streamlit is not available')
+
+        super().__init__(chip)
+
+        if not port:
+            port = WebDashboard.get_next_port()
+        if not port:
+            port = WebDashboard.__port
+
+        self.__dashboard = None
+        self.__chip = chip
+        self.__directory = tempfile.mkdtemp(prefix='sc_dashboard_',
+                                            suffix=f'_{self.__chip.design}')
+        self.__manifest = os.path.join(self.__directory, 'manifest.json')
+        self.__manifest_lock = os.path.join(self.__directory, 'manifest.lock')
+        self.__port = port
+        dirname = os.path.dirname(__file__)
+        self.__streamlit_file = os.path.join(dirname, 'viewer.py')
+
+        self.__streamlit_args = [
+            ("browser.gatherUsageStats", False),
+            ("browser.serverPort", self.__port),
+            ("logger.level", 'error'),
+            ("runner.fastReruns", True),
+            ("server.port", self.__port),
+            ("client.toolbarMode", "viewer")
+        ]
+        if "PYTEST_CURRENT_TEST" in os.environ:
+            self.__streamlit_args.append(("server.headless", True))
+
+        # pass in a json object called __graph_chips
+        # the key is the chip_name and value is the filepath
+        # if another argument is passed
+
+        # use of list is to preserve order
+        self.__graph_chips = []
+        graph_chips_config = []
+        if graph_chips:
+            for chip_object_and_name in graph_chips:
+                chip_file_path = \
+                    os.path.join(self.__directory,
+                                 f"{chip_object_and_name['name']}.json")
+                self.__graph_chips.append({
+                    'chip': chip_object_and_name['chip'],
+                    'name': chip_file_path
+                })
+                graph_chips_config.append({
+                    "path": chip_file_path,
+                    "cwd": utils.get_chip_cwd(
+                        chip_object_and_name['chip'],
+                        chip_object_and_name['cfg_path'])
+                })
+
+        self.__config = {
+            "manifest": self.__manifest,
+            "lock": self.__manifest_lock,
+            "graph_chips": graph_chips_config
+        }
+
+        self.__sleep_time = 0.5
+        self.__signal_handler = None
+
+        self.__lock = fasteners.InterProcessLock(self.__manifest_lock)
+
+        atexit.register(self.__cleanup)
+
+    def open_dashboard(self):
+        with open(self.__get_config_file(), 'w') as f:
+            json.dump(self.__config, f, indent=4)
+
+        self.update_manifest()
+
+        self.update_graph_manifests()
+
+        self.__dashboard = multiprocessing.Process(
+            target=self._run_streamlit_bootstrap)
+
+        self.__signal_handler = signal.signal(signal.SIGINT, WebDashboard.__signal_handler)
+
+        self.__dashboard.start()
+
+    def update_manifest(self, payload=None):
+        if not self.__manifest:
+            return
+
+        new_file = f"{self.__manifest}.new.json"
+        self.__chip.write_manifest(new_file)
+
+        with self.__lock:
+            shutil.move(new_file, self.__manifest)
+
+    def update_graph_manifests(self):
+        for chip_object_and_name in self.__graph_chips:
+            chip = chip_object_and_name['chip']
+            file_path = chip_object_and_name['name']
+            chip.write_manifest(file_path)
+
+    def __get_config_file(self):
+        return os.path.join(self.__directory, 'config.json')
+
+    def is_running(self):
+        if self.__dashboard is None:
+            return False
+
+        if self.__dashboard.is_alive():
+            return True
+
+        self.__dashboard = None
+        self.__manifest = None
+        return False
+
+    def end_of_run(self):
+        pass
+
+    def stop(self):
+        if not self.is_running():
+            return
+
+        while self.__dashboard.is_alive():
+            self.__dashboard.terminate()
+            self._sleep()
+
+        if self.__signal_handler:
+            signal.signal(signal.SIGINT, self.__signal_handler)
+
+        self.__dashboard = None
+        self.__manifest = None
+        self.__signal_handler = None
+
+    def wait(self):
+        self.__dashboard.join()
+
+    def _sleep(self):
+        time.sleep(self.__sleep_time)
+
+    def _run_streamlit_bootstrap(self):
+        from streamlit.web import bootstrap
+        from streamlit import config as _config
+
+        for config, val in self.__streamlit_args:
+            _config.set_option(config, val)
+
+        bootstrap.run(self.__streamlit_file,
+                      False,
+                      [self.__get_config_file()],
+                      flag_options={})
+
+    def __run_streamlit_subproc(self):
+        cmd = ['streamlit', 'run',
+               self.__streamlit_file, self.__get_config_file()]
+        for config, val in self.__streamlit_args:
+            cmd.append(f'--{config}')
+            cmd.append(val)
+
+        subprocess.Popen(cmd)
+
+    def __cleanup(self):
+        self.stop()
+
+        if os.path.exists(self.__directory):
+            shutil.rmtree(self.__directory)
+
+    @staticmethod
+    def get_next_port():
+        with socketserver.TCPServer(("localhost", 0), None) as s:
+            return s.server_address[1]
+        return None

siliconcompiler/report/dashboard/{components → web/components}/__init__.py

@@ -1,7 +1,6 @@
 import base64
 import json
 import os
-import pandas
 
 import streamlit
 from streamlit_agraph import agraph
@@ -14,10 +13,10 @@ from siliconcompiler import __version__ as sc_version
 from siliconcompiler import utils
 from siliconcompiler.report import report
 
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import layouts
-from siliconcompiler.report.dashboard.utils import file_utils
-from siliconcompiler.report.dashboard.components import flowgraph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import layouts
+from siliconcompiler.report.dashboard.web.utils import file_utils
+from siliconcompiler.report.dashboard.web.components import flowgraph
 
 
 SC_ABOUT = [
@@ -302,7 +301,7 @@ def manifest_viewer(
             if streamlit.checkbox(
                     'Raw manifest',
                     help='Click here to see the manifest before it was made more readable'):
-                manifest_to_show = chip.schema.cfg
+                manifest_to_show = chip.schema.getdict()
             else:
                 manifest_to_show = report.make_manifest(chip)
 
@@ -327,7 +326,7 @@ def manifest_viewer(
         streamlit.download_button(
             label='Download',
             file_name='manifest.json',
-            data=json.dumps(chip.schema.cfg, indent=2),
+            data=json.dumps(chip.schema.getdict(), indent=2),
             mime="application/json",
             use_container_width=True)
 
@@ -491,6 +490,8 @@ def node_file_tree_viewer(chip, step, index):
 
 
 def node_viewer(chip, step, index, metric_dataframe, height=None):
+    from pandas import DataFrame
+
     metrics_col, records_col, logs_and_reports_col = streamlit.columns(3, gap='small')
 
     node_name = f'{step}{index}'
@@ -507,7 +508,7 @@ def node_viewer(chip, step, index, metric_dataframe, height=None):
         nodes = {}
         nodes[step + index] = report.get_flowgraph_nodes(chip, step, index)
         streamlit.dataframe(
-            pandas.DataFrame.from_dict(nodes),
+            DataFrame.from_dict(nodes),
             use_container_width=True,
             height=height)
     with logs_and_reports_col:

siliconcompiler/report/dashboard/{components → web/components}/flowgraph.py

@@ -1,5 +1,4 @@
 from siliconcompiler.report import report
-from siliconcompiler.flowgraph import _get_flowgraph_exit_nodes, _get_flowgraph_entry_nodes
 from siliconcompiler.tools._common import get_tool_task
 from siliconcompiler import NodeStatus
 
@@ -45,8 +44,9 @@ def get_nodes_and_edges(chip):
     node_dependencies = report.get_flowgraph_edges(chip)
     successful_path = report.get_flowgraph_path(chip)
 
-    entry_exit_nodes = _get_flowgraph_entry_nodes(chip, chip.get('option', 'flow')) + \
-        _get_flowgraph_exit_nodes(chip, chip.get('option', 'flow'))
+    flow = chip.get('option', 'flow')
+    entry_exit_nodes = chip.schema.get("flowgraph", flow, field="schema").get_entry_nodes() + \
+        chip.schema.get("flowgraph", flow, field="schema").get_exit_nodes()
 
     for step, index in node_dependencies:
         # Build node

siliconcompiler/report/dashboard/{components → web/components}/graph.py

@@ -1,11 +1,10 @@
 import altair
 import math
-import pandas
 import streamlit
 
 from siliconcompiler.report import report
 
-from siliconcompiler.report.dashboard import state
+from siliconcompiler.report.dashboard.web import state
 
 
 def _get_report_chips():
@@ -20,9 +19,11 @@ def job_selector():
     Displays a dataframe that can be edited to select specific jobs to include
     in the analysis.
     """
+    from pandas import DataFrame
+
     jobs = state.get_chips()
 
-    all_jobs = pandas.DataFrame({
+    all_jobs = DataFrame({
         'job names': jobs,
         'selected jobs': [True] * len(jobs)
     })
@@ -114,6 +115,8 @@ def settings(metrics, nodes, graph_number):
 
 
 def graph(metrics, nodes, node_to_step_index_map, graph_number):
+    from pandas import DataFrame
+
     metric, selected_nodes, log_scale, transpose, chart_type = \
         settings(metrics, nodes, graph_number)
 
@@ -167,7 +170,7 @@ def graph(metrics, nodes, node_to_step_index_map, graph_number):
 
     color = color_label
 
-    alt_chart = altair.Chart(pandas.DataFrame(filtered_data).dropna(), height=500)
+    alt_chart = altair.Chart(DataFrame(filtered_data).dropna(), height=500)
 
     if chart_type == 'line':
         chart_mark = alt_chart.mark_line(point=True)

siliconcompiler/report/dashboard/{layouts → web/layouts}/__init__.py

@@ -1,6 +1,6 @@
-from siliconcompiler.report.dashboard.layouts import vertical_flowgraph
-from siliconcompiler.report.dashboard.layouts import vertical_flowgraph_sac_tabs
-from siliconcompiler.report.dashboard.layouts import vertical_flowgraph_node_tab
+from siliconcompiler.report.dashboard.web.layouts import vertical_flowgraph
+from siliconcompiler.report.dashboard.web.layouts import vertical_flowgraph_sac_tabs
+from siliconcompiler.report.dashboard.web.layouts import vertical_flowgraph_node_tab
 
 __LAYOUTS = {
     "vertical_flowgraph": vertical_flowgraph.layout,

siliconcompiler/report/dashboard/{layouts → web/layouts}/_common.py

@@ -1,6 +1,6 @@
 import streamlit_antd_components as sac
 
-from siliconcompiler.report.dashboard import state
+from siliconcompiler.report.dashboard.web import state
 
 
 def check_rerun():

siliconcompiler/report/dashboard/{layouts → web/layouts}/vertical_flowgraph.py

@@ -1,11 +1,11 @@
 import os
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard.components import graph
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
-from siliconcompiler.report.dashboard.layouts import _common
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web.components import graph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
+from siliconcompiler.report.dashboard.web.layouts import _common
 
 
 def layout():

siliconcompiler/report/dashboard/{layouts → web/layouts}/vertical_flowgraph_node_tab.py

@@ -1,12 +1,12 @@
 import os
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard.components import graph
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
-from siliconcompiler.report.dashboard.utils import file_utils
-from siliconcompiler.report.dashboard.layouts import _common
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web.components import graph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
+from siliconcompiler.report.dashboard.web.utils import file_utils
+from siliconcompiler.report.dashboard.web.layouts import _common
 import streamlit_antd_components as sac
 
 

siliconcompiler/report/dashboard/{layouts → web/layouts}/vertical_flowgraph_sac_tabs.py

@@ -1,12 +1,12 @@
 import os
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard.components import graph
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
-from siliconcompiler.report.dashboard.utils import file_utils
-from siliconcompiler.report.dashboard.layouts import _common
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web.components import graph
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
+from siliconcompiler.report.dashboard.web.utils import file_utils
+from siliconcompiler.report.dashboard.web.layouts import _common
 import streamlit_antd_components as sac
 
 

siliconcompiler/report/dashboard/{state.py → web/state.py}

@@ -58,7 +58,7 @@ def update_manifest():
 
         for history in chip.getkeys('history'):
             history_chip = Chip(design='')
-            history_chip.schema.cfg = chip.getdict('history', history)
+            history_chip.schema = chip.schema.history(history).copy()
             history_chip.set('design', chip.design)
             add_chip(history, history_chip)
 

siliconcompiler/report/dashboard/{utils → web/utils}/__init__.py

@@ -1,5 +1,4 @@
 from siliconcompiler import NodeStatus
-from siliconcompiler.flowgraph import _get_flowgraph_nodes
 
 from pathlib import Path
 
@@ -25,7 +24,8 @@ def make_node_to_step_index_map(chip, metric_dataframe):
     '''
     node_to_step_index_map = {}
     if chip.get('option', 'flow'):
-        for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
+        for step, index in chip.schema.get("flowgraph", chip.get('option', 'flow'),
+                                           field="schema").get_nodes():
             node_to_step_index_map[f'{step}{index}'] = (step, index)
 
     # concatenate step and index
@@ -57,7 +57,8 @@ def make_metric_to_metric_unit_map(metric_dataframe):
 def is_running(chip):
     if not chip.get('option', 'flow'):
         return False
-    for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
+    for step, index in chip.schema.get("flowgraph", chip.get('option', 'flow'),
+                                       field="schema").get_nodes():
         state = chip.get('record', 'status', step=step, index=index)
         if not NodeStatus.is_done(state):
             return True

siliconcompiler/report/dashboard/{viewer.py → web/viewer.py}

@@ -1,9 +1,9 @@
 import streamlit
 
-from siliconcompiler.report.dashboard import components
-from siliconcompiler.report.dashboard import layouts
-from siliconcompiler.report.dashboard import state
-from siliconcompiler.report.dashboard import utils
+from siliconcompiler.report.dashboard.web import components
+from siliconcompiler.report.dashboard.web import layouts
+from siliconcompiler.report.dashboard.web import state
+from siliconcompiler.report.dashboard.web import utils
 
 import streamlit_autorefresh
 

siliconcompiler/report/html_report.py

@@ -21,8 +21,7 @@ def _generate_html_report(chip, flow, flowgraph_nodes, results_html):
             del flowgraph_nodes[index]
 
     schema = chip.schema.copy()
-    schema.prune()
-    pruned_cfg = schema.cfg
+    pruned_cfg = schema.getdict()
     if 'history' in pruned_cfg:
         del pruned_cfg['history']
     if 'library' in pruned_cfg:
@@ -51,7 +50,7 @@ def _generate_html_report(chip, flow, flowgraph_nodes, results_html):
             metrics=metrics,
             metrics_unit=metrics_unit,
             reports=reports,
-            manifest=chip.schema.cfg,
+            manifest=chip.schema.getdict(),
             pruned_cfg=pruned_cfg,
             metric_keys=metrics_to_show,
             img_data=img_data,

siliconcompiler/report/report.py

@@ -1,10 +1,9 @@
 import fnmatch
-import pandas
 import os
 from siliconcompiler import Schema
-from siliconcompiler.schema.utils import PerNode
+from siliconcompiler.schema import PerNode
 from siliconcompiler.report import utils
-from siliconcompiler.flowgraph import nodes_to_execute
+from siliconcompiler.utils.flowgraph import nodes_to_execute
 from siliconcompiler.tools._common import get_tool_task
 
 
@@ -22,10 +21,12 @@ def make_metric_dataframe(chip):
         >>> make_metric_dataframe(chip)
         Returns pandas dataframe of tracked metrics.
     '''
+    from pandas import DataFrame
+
     nodes, errors, metrics, metrics_unit, metrics_to_show, reports = utils._collect_data(chip)
     # converts from 2d dictionary to pandas DataFrame, transposes so
     # orientation is correct, and filters based on the metrics we track
-    data = (pandas.DataFrame.from_dict(metrics, orient='index').transpose())
+    data = (DataFrame.from_dict(metrics, orient='index').transpose())
     data = data.loc[metrics_to_show]
     # include metrics_unit
     data.index = data.index.map(lambda x: (x, metrics_unit[x]))
@@ -106,6 +107,11 @@ def make_manifest_helper(manifest_subsect, modified_manifest_subsect):
         'default' nodes.
     '''
 
+    def _is_leaf(cfg):
+        # 'shorthelp' chosen arbitrarily: any mandatory field with a consistent
+        # type would work.
+        return 'shorthelp' in cfg and isinstance(cfg['shorthelp'], str)
+
     def build_leaf(manifest_subsect):
         if PerNode(manifest_subsect['pernode']) == PerNode.NEVER:
             if Schema.GLOBAL_KEY in manifest_subsect['node'] and \
@@ -132,7 +138,7 @@ def make_manifest_helper(manifest_subsect, modified_manifest_subsect):
                             node_values[step + index] = value
             return node_values
 
-    if Schema._is_leaf(manifest_subsect):
+    if _is_leaf(manifest_subsect):
         if PerNode(manifest_subsect['pernode']) == PerNode.NEVER:
             if Schema.GLOBAL_KEY in manifest_subsect['node']:
                 value = manifest_subsect['node'][Schema.GLOBAL_KEY][Schema.GLOBAL_KEY]['value']
@@ -157,7 +163,7 @@ def make_manifest_helper(manifest_subsect, modified_manifest_subsect):
 
     for key, key_dict in manifest_subsect.items():
         if key != 'default':
-            if Schema._is_leaf(key_dict):
+            if _is_leaf(key_dict):
                 modified_manifest_subsect[key] = build_leaf(key_dict)
             else:
                 modified_manifest_subsect[key] = {}
@@ -175,7 +181,7 @@ def make_manifest(chip):
         >>> make_manifest(chip)
         Returns tree/json of manifest.
     '''
-    manifest = chip.schema.cfg
+    manifest = chip.schema.getdict()
     modified_manifest = {}
     make_manifest_helper(manifest, modified_manifest)
     return modified_manifest

siliconcompiler/report/summary_image.py

@@ -2,7 +2,7 @@ import os
 import string
 from PIL import Image, ImageFont, ImageDraw
 
-from siliconcompiler import units
+from siliconcompiler.utils import units
 from siliconcompiler.report.utils import _find_summary_image, _find_summary_metrics
 
 

siliconcompiler/report/summary_table.py

@@ -1,4 +1,3 @@
-import pandas
 import shutil
 
 from siliconcompiler.report.utils import _collect_data, _get_flowgraph_path
@@ -10,6 +9,7 @@ def _show_summary_table(chip, flow, flowgraph_nodes, show_all_indices):
     '''
     Prints the end of run summary table
     '''
+    from pandas import DataFrame
 
     # Display data
     column_width = 15
@@ -85,7 +85,7 @@ def _show_summary_table(chip, flow, flowgraph_nodes, show_all_indices):
             info_list.append(f"partname : {fpga_partname}")
 
     libraries = set()
-    for val, step, index in chip.schema._getvals('asic', 'logiclib'):
+    for val, step, index in chip.schema.get('asic', 'logiclib', field=None).getvalues():
         if not step or (step, index) in flowgraph_nodes:
             libraries.update(val)
     if libraries:
@@ -96,7 +96,7 @@ def _show_summary_table(chip, flow, flowgraph_nodes, show_all_indices):
     print("-" * max_line_width)
     print(info, "\n")
 
-    df = pandas.DataFrame(data, row_labels, column_labels)
+    df = DataFrame(data, row_labels, column_labels)
     if not df.empty:
         print(df.to_string(line_width=max_line_width, col_space=2))
     else:

siliconcompiler/report/utils.py

@@ -1,9 +1,11 @@
 from siliconcompiler import NodeStatus
-from siliconcompiler import units
-from siliconcompiler.flowgraph import _get_flowgraph_execution_order, _get_flowgraph_exit_nodes, \
+from siliconcompiler.utils import units
+from siliconcompiler.utils.flowgraph import _get_flowgraph_execution_order, \
     nodes_to_execute
 from siliconcompiler.tools._common import get_tool_task
 
+from siliconcompiler.flowgraph import RuntimeFlowgraph
+
 
 def _find_summary_image(chip, ext='png'):
     for nodes in reversed(_get_flowgraph_execution_order(chip, chip.get('option', 'flow'))):
@@ -26,9 +28,7 @@ def _find_summary_metrics(chip, metrics_map):
 
                 data = chip.get('metric', metric, step=step, index=index)
                 if data is not None:
-                    unit = None
-                    if chip.schema.has_field('metric', metric, 'unit'):
-                        unit = chip.get('metric', metric, field='unit')
+                    unit = chip.get('metric', metric, field='unit')
                     if formatter:
                         metrics[name] = formatter(data, unit)
                     else:
@@ -44,7 +44,7 @@ def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
         return [], {}, {}, {}, [], {}
 
     if not flowgraph_nodes:
-        flowgraph_nodes = nodes_to_execute(chip)
+        flowgraph_nodes = list(nodes_to_execute(chip))
         # only report tool based steps functions
         for (step, index) in list(flowgraph_nodes):
             tool, task = get_tool_task(chip, step, '0', flow=flow)
@@ -79,9 +79,7 @@ def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
     for metric in chip.getkeys('metric'):
 
         # Get the unit associated with the metric
-        metric_unit = None
-        if chip.schema.has_field('metric', metric, 'unit'):
-            metric_unit = chip.get('metric', metric, field='unit')
+        metric_unit = chip.get('metric', metric, field='unit')
         metric_type = chip.get('metric', metric, field='type')
 
         show_metric = False
@@ -143,7 +141,10 @@ def _get_flowgraph_path(chip, flow, nodes_to_execute, only_include_successful=Fa
     to_search = []
     # Start search with any successful leaf nodes.
     flowgraph_steps = list(map(lambda node: node[0], nodes_to_execute))
-    end_nodes = _get_flowgraph_exit_nodes(chip, flow, steps=flowgraph_steps)
+    runtime = RuntimeFlowgraph(chip.schema.get("flowgraph", flow, field='schema'),
+                               from_steps=flowgraph_steps,
+                               to_steps=flowgraph_steps)
+    end_nodes = runtime.get_exit_nodes()
     for node in end_nodes:
         if only_include_successful:
             if NodeStatus.is_success(chip.get('record', 'status', step=node[0], index=node[1])):