siliconcompiler 0.28.2__py3-none-any.whl → 0.28.4__py3-none-any.whl

siliconcompiler/report/dashboard/layouts/vertical_flowgraph_sac_tabs.py

@@ -0,0 +1,110 @@
+import os
+import streamlit
+
+from siliconcompiler.report.dashboard import components
+from siliconcompiler.report.dashboard.components import graph
+from siliconcompiler.report.dashboard import state
+from siliconcompiler.report.dashboard import utils
+from siliconcompiler.report.dashboard.utils import file_utils
+from siliconcompiler.report.dashboard.layouts import _common
+import streamlit_antd_components as sac
+
+
+def layout():
+    chip = state.get_chip()
+    metric_dataframe, node_to_step_index_map, metric_to_metric_unit_map = \
+        utils.generate_metric_dataframe(chip)
+
+    components.page_header()
+
+    tab_headings = [
+        sac.TabsItem(
+            "Metrics",
+            icon='stack'),
+        sac.TabsItem(
+            "Manifest",
+            icon=file_utils.get_file_icon('manifest.pkg.json')),
+        sac.TabsItem(
+            "File Viewer",
+            icon=file_utils.get_file_icon(state.get_key(state.SELECTED_FILE))),
+        sac.TabsItem(
+            "Design Preview",
+            icon=file_utils.get_file_icon('design.png'),
+            disabled=not os.path.isfile(f'{chip.getworkdir()}/{chip.design}.png')),
+        sac.TabsItem(
+            "Graphs",
+            icon='graph-up',
+            disabled=len(state.get_key(state.LOADED_CHIPS)) == 1)
+    ]
+
+    tab_selected = _common.sac_tabs(tab_headings)
+
+    if tab_selected == "Metrics":
+        # Add flowgraph
+        if state.get_key(state.DISPLAY_FLOWGRAPH):
+            default_flowgraph_width_in_percent = 0.4
+            flowgraph_col_width_in_pixels = 520
+            flowgraph_col_width_in_percent = \
+                state.compute_component_size(
+                    default_flowgraph_width_in_percent,
+                    flowgraph_col_width_in_pixels)
+
+            flowgraph_col, metrics_container = \
+                streamlit.columns(
+                    [flowgraph_col_width_in_percent, 1 - flowgraph_col_width_in_percent],
+                    gap="large")
+
+            with flowgraph_col:
+                header_col, flowgraph_toggle_container = streamlit.columns(2, gap="large")
+                with header_col:
+                    streamlit.header('Flowgraph')
+                components.flowgraph_viewer(chip)
+        else:
+            flowgraph_toggle_container = streamlit.container()
+            metrics_container = streamlit.container()
+
+        with flowgraph_toggle_container:
+            streamlit.markdown("")
+            streamlit.markdown("")
+
+            if state.set_key(state.DISPLAY_FLOWGRAPH, not streamlit.checkbox(
+                    'Hide flowgraph',
+                    help='Click here to hide the flowgraph')):
+                state.set_key(state.APP_RERUN, "Flowgraph")
+
+        with metrics_container:
+            components.metrics_viewer(metric_dataframe, metric_to_metric_unit_map)
+
+            header_col, settings_col = \
+                streamlit.columns(
+                    [0.7, 0.3],
+                    gap='small')
+            with header_col:
+                streamlit.header('Node Information')
+            with settings_col:
+                components.node_selector(list(node_to_step_index_map.keys()))
+
+            if state.get_selected_node():
+                step, index = node_to_step_index_map[state.get_selected_node()]
+                current_file = state.get_key(state.SELECTED_FILE)
+                components.node_viewer(chip, step, index, metric_dataframe)
+                if state.get_key(state.SELECTED_FILE) and \
+                        current_file != state.get_key(state.SELECTED_FILE):
+                    state.set_key(state.APP_RERUN, "File")
+
+    if tab_selected == "Manifest":
+        components.manifest_viewer(chip)
+
+    if tab_selected == "File Viewer":
+        components.file_viewer(
+            chip,
+            state.get_key(state.SELECTED_FILE),
+            page_key=state.SELECTED_FILE_PAGE)
+
+    if tab_selected == "Design Preview":
+        components.file_viewer(chip, f'{chip.getworkdir()}/{chip.design}.png')
+
+    if tab_selected == "Graphs":
+        graph.viewer(metric_dataframe, node_to_step_index_map, metric_to_metric_unit_map)
+
+    _common.check_rerun()

siliconcompiler/report/dashboard/state.py

@@ -0,0 +1,217 @@
+import argparse
+import json
+import os
+import streamlit
+import streamlit_javascript
+import fasteners
+
+from siliconcompiler import Chip
+
+
+DISPLAY_FLOWGRAPH = "show_flowgraph"
+SELECTED_JOB = "selected_job"
+SELECTED_NODE = "selected_node"
+# This is needed until the graph supports setting a selected node
+SELECTED_FLOWGRAPH_NODE = "selected_flowgraph_node"
+SELECTED_SELECTOR_NODE = "selected_selector_node"
+NODE_SOURCE = "node_source"
+SELECTED_FILE = "selected_file"
+SELECTED_FILE_PAGE = "selected_file_page"
+LOADED_CHIPS = "loaded_chips"
+UI_WIDTH = "ui_width"
+MANIFEST_FILE = "manifest_file"
+MANIFEST_LOCK = "manifest_lock"
+MANIFEST_TIME = "manifest_time"
+IS_RUNNING = "is_flow_running"
+GRAPH_JOBS = "graph_jobs"
+APP_LAYOUT = "app_layout"
+APP_RERUN = "app_rerun"
+APP_RUNNING_REFRESH = "app_running_refresh"
+APP_STOPPED_REFRESH = "app_stopped_refresh"
+MAX_DICT_ITEMS_TO_SHOW = "max_dict_items"
+MAX_FILE_LINES_TO_SHOW = "max_file_lines"
+SELECT_TAB = "select_tab"
+TAB_INDEX = "tab-index"
+TAB_STATE = "tab-state"
+
+_DEBUG = False
+DEVELOPER = False
+
+
+def _add_default(key, value):
+    if key not in streamlit.session_state:
+        streamlit.session_state[key] = value
+
+
+def update_manifest():
+    file_time = os.stat(get_key(MANIFEST_FILE)).st_mtime
+
+    if get_key(MANIFEST_TIME) != file_time:
+        chip = Chip(design='')
+
+        with get_key(MANIFEST_LOCK):
+            chip.read_manifest(get_key(MANIFEST_FILE))
+        set_key(MANIFEST_TIME, file_time)
+        debug_print("Read manifest", get_key(MANIFEST_FILE))
+
+        add_chip("default", chip)
+
+        for history in chip.getkeys('history'):
+            history_chip = Chip(design='')
+            history_chip.schema.cfg = chip.getdict('history', history)
+            history_chip.set('design', chip.design)
+            add_chip(history, history_chip)
+
+        return True
+    return False
+
+
+def init():
+    _add_default(DISPLAY_FLOWGRAPH, True)
+    _add_default(SELECTED_JOB, None)
+    _add_default(SELECTED_NODE, None)
+    _add_default(SELECTED_FLOWGRAPH_NODE, None)
+    _add_default(SELECTED_SELECTOR_NODE, None)
+    _add_default(NODE_SOURCE, None)
+    _add_default(SELECTED_FILE, None)
+    _add_default(SELECTED_FILE_PAGE, None)
+    _add_default(LOADED_CHIPS, {})
+    _add_default(MANIFEST_FILE, None)
+    _add_default(MANIFEST_LOCK, None)
+    _add_default(MANIFEST_TIME, None)
+    _add_default(IS_RUNNING, False)
+    _add_default(GRAPH_JOBS, None)
+    _add_default(UI_WIDTH, None)
+    _add_default(APP_LAYOUT, "vertical_flowgraph_sac_tabs")
+    _add_default(APP_RERUN, None)
+    _add_default(APP_RUNNING_REFRESH, 2 * 1000)
+    _add_default(APP_STOPPED_REFRESH, 30 * 1000)
+    _add_default(MAX_DICT_ITEMS_TO_SHOW, 100)
+    _add_default(MAX_FILE_LINES_TO_SHOW, 100)
+    _add_default(SELECT_TAB, None)
+    _add_default(TAB_INDEX, 0)
+
+    parser = argparse.ArgumentParser('dashboard')
+    parser.add_argument('cfg', nargs='?')
+    args = parser.parse_args()
+
+    if not args.cfg:
+        raise ValueError('configuration not provided')
+
+    if not get_key(LOADED_CHIPS):
+        # First time through
+
+        with open(args.cfg, 'r') as f:
+            config = json.load(f)
+
+        set_key(MANIFEST_FILE, config["manifest"])
+        set_key(MANIFEST_LOCK, fasteners.InterProcessLock(config["lock"]))
+
+        update_manifest()
+        chip = get_chip("default")
+        for graph_info in config['graph_chips']:
+            file_path = graph_info['path']
+            graph_chip = Chip(design='')
+            graph_chip.read_manifest(file_path)
+            graph_chip.unset('arg', 'step')
+            graph_chip.unset('arg', 'index')
+
+            if graph_info['cwd']:
+                graph_chip.cwd = graph_info['cwd']
+
+            add_chip(os.path.basename(file_path), graph_chip)
+
+        chip_step = chip.get('arg', 'step')
+        chip_index = chip.get('arg', 'index')
+
+        if chip_step and chip_index:
+            set_key(SELECTED_NODE, f'{chip_step}{chip_index}')
+
+    chip = get_chip("default")
+    chip.unset('arg', 'step')
+    chip.unset('arg', 'index')
+
+    if not get_key(SELECTED_JOB):
+        set_key(SELECTED_JOB, "default")
+
+
+def setup():
+    with streamlit.empty():
+        # get width
+        set_key(UI_WIDTH, streamlit_javascript.st_javascript("window.innerWidth"))
+        # replace with a empty container to avoid adding a gap at the top
+        streamlit.empty()
+
+    set_key(NODE_SOURCE, None)
+
+
+def get_chip(job=None):
+    if not job:
+        job = get_key(SELECTED_JOB)
+    return get_key(LOADED_CHIPS)[job]
+
+
+def add_chip(name, chip):
+    streamlit.session_state[LOADED_CHIPS][name] = chip
+
+
+def get_chips():
+    chips = list(get_key(LOADED_CHIPS).keys())
+    chips.remove('default')
+    chips.insert(0, 'default')
+    return chips
+
+
+def get_selected_node():
+    if get_key(NODE_SOURCE) == "flowgraph":
+        return get_key(SELECTED_FLOWGRAPH_NODE)
+
+    if get_key(NODE_SOURCE) == "selector":
+        return get_key(SELECTED_SELECTOR_NODE)
+
+    return get_key(SELECTED_NODE)
+
+
+def get_key(key):
+    return streamlit.session_state[key]
+
+
+def set_key(key, value):
+    changed = value != streamlit.session_state[key]
+    if changed:
+        debug_print("set_key()", key, "changed", streamlit.session_state[key], "->", value)
+        streamlit.session_state[key] = value
+        return True
+    return False
+
+
+def del_key(key):
+    debug_print("del_key()", key)
+    if key in streamlit.session_state:
+        del streamlit.session_state[key]
+
+
+def compute_component_size(minimum, requested_px):
+    ui_width = get_key(UI_WIDTH)
+
+    if ui_width > 0:
+        return min(requested_px / ui_width, minimum)
+
+    return minimum
+
+
+def debug_print(*args):
+    if not _DEBUG:
+        return
+
+    print(*args)
+
+
+def debug_print_state():
+    if not _DEBUG:
+        return
+
+    for n, key in enumerate(sorted(streamlit.session_state.keys())):
+        value = streamlit.session_state[key]
+        print("state", n, key, type(value), value)
+    print()

siliconcompiler/report/dashboard/utils/__init__.py

@@ -0,0 +1,73 @@
+from siliconcompiler import NodeStatus
+from siliconcompiler.flowgraph import _get_flowgraph_nodes
+
+from pathlib import Path
+
+
+def get_chip_cwd(chip, manifest):
+    build_dir = Path(chip.get('option', 'builddir'))
+
+    manifest_path = Path(manifest)
+    for path in manifest_path.parents:
+        if build_dir.name == path.name:
+            return str(path.parent)
+
+    return None
+
+
+def make_node_to_step_index_map(chip, metric_dataframe):
+    '''
+    Returns a map from the name of a node to the associated step, index pair.
+
+    Args:
+        metric_dataframe (pandas.DataFrame) : A dataframe full of all metrics and all
+            nodes of the selected chip
+    '''
+    node_to_step_index_map = {}
+    for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
+        node_to_step_index_map[f'{step}{index}'] = (step, index)
+
+    # concatenate step and index
+    metric_dataframe.columns = metric_dataframe.columns.map(lambda x: f'{x[0]}{x[1]}')
+    return node_to_step_index_map, metric_dataframe
+
+
+def make_metric_to_metric_unit_map(metric_dataframe):
+    '''
+    Returns a map from the name of a metric to the associated metric and unit in
+    the form f'{x[0]} ({x[1]})'
+
+    Args:
+        metric_dataframe (pandas.DataFrame) : A dataframe full of all metrics and all
+            nodes of the selected chip.
+    '''
+    metric_to_metric_unit_map = {}
+    for metric, unit in metric_dataframe.index.tolist():
+        if unit != '':
+            metric_to_metric_unit_map[f'{metric} ({unit})'] = metric
+        else:
+            metric_to_metric_unit_map[metric] = metric
+    # concatenate metric and unit
+    metric_dataframe.index = metric_dataframe.index.map(lambda x: f'{x[0]} ({x[1]})'
+                                                        if x[1] else x[0])
+    return metric_to_metric_unit_map, metric_dataframe
+
+
+def is_running(chip):
+    for step, index in _get_flowgraph_nodes(chip, chip.get('option', 'flow')):
+        state = chip.get('record', 'status', step=step, index=index)
+        if not NodeStatus.is_done(state):
+            return True
+    return False
+
+
+def generate_metric_dataframe(chip):
+    from siliconcompiler.report import report
+
+    metric_dataframe = report.make_metric_dataframe(chip)
+    node_to_step_index_map, metric_dataframe = \
+        make_node_to_step_index_map(chip, metric_dataframe)
+    metric_to_metric_unit_map, metric_dataframe = \
+        make_metric_to_metric_unit_map(metric_dataframe)
+
+    return metric_dataframe, node_to_step_index_map, metric_to_metric_unit_map

siliconcompiler/report/dashboard/utils/file_utils.py

@@ -0,0 +1,120 @@
+import gzip
+import os
+from siliconcompiler import utils, sc_open
+
+
+def is_file_is_binary(path, compressed):
+    # Read first chunk and check for non characters
+    try:
+        if compressed:
+            with gzip.open(path, 'rt') as f:
+                f.read(8196)
+        else:
+            with open(path, "r") as f:
+                f.read(8196)
+    except UnicodeDecodeError:
+        return True
+    return False
+
+
+def read_file(path, max_lines):
+    _, compressed_file_extension = os.path.splitext(path.lower())
+    file_info = []
+    honor_max_file = max_lines is not None
+
+    def read_file(fid):
+        for line in fid:
+            file_info.append(line.rstrip())
+            if honor_max_file and len(file_info) >= max_lines:
+                file_info.append('... truncated ...')
+                return
+
+    is_compressed = compressed_file_extension == '.gz'
+    if is_file_is_binary(path, is_compressed):
+        return "Binary file"
+
+    if is_compressed:
+        with gzip.open(path, 'rt') as fid:
+            read_file(fid)
+    else:
+        with sc_open(path) as fid:
+            read_file(fid)
+    return "\n".join(file_info)
+
+
+def get_file_type(ext):
+    if ext in ("v", "vh", "sv", "svh", "vg"):
+        return "verilog"
+    if ext in ("vhdl", "vhd"):
+        return "vhdl"
+    if ext in ("tcl", "sdc", "xdc"):
+        return "tcl"
+    if ext in ("c", "cpp", "cc", "h"):
+        return "cpp"
+    if ext in ("csv",):
+        return "csv"
+    if ext in ("md",):
+        return "markdown"
+    if ext in ("sh",):
+        return "bash"
+    return "log"
+
+
+def get_file_icon(file):
+    if not file:
+        return 'file'
+    ext = utils.get_file_ext(file)
+    file_type = get_file_type(ext)
+    if file.endswith('.pkg.json'):
+        return 'boxes'
+    elif ext in ('png', 'jpg', 'jpeg'):
+        return 'file-image'
+    elif ext == 'json':
+        return 'file-json'
+    elif file_type in ('verilog', 'tcl', 'vhdl', 'cpp', 'bash'):
+        return 'file-code'
+    elif ext in ('log', 'rpt', 'drc', 'warnings', 'errors'):
+        return 'file-text'
+    return 'file'
+
+
+def convert_filepaths_to_select_tree(logs_and_reports):
+    """
+    Converts the logs_and_reports found to the structure
+    required by streamlit_tree_select. Success is predicated on the order of
+    logs_and_reports outlined in report.get_files.
+
+    Args:
+        logs_and_reports (list) : A list of 3-tuples with order of a path name,
+            folder in the..., and files in the....
+    """
+    if not logs_and_reports:
+        return []
+
+    all_files = {}
+    for path_name, folders, files in logs_and_reports:
+        all_files[path_name] = {
+            'files': list(files),
+            'folders': list(folders)
+        }
+
+    def organize_node(base_folder):
+        nodes = []
+
+        for folder in all_files[base_folder]['folders']:
+            path = os.path.join(base_folder, folder)
+            nodes.append({
+                'value': path,
+                'label': folder,
+                'children': organize_node(path)
+            })
+        for file in all_files[base_folder]['files']:
+            nodes.append({
+                'value': os.path.join(base_folder, file),
+                'label': file
+            })
+
+        return nodes
+
+    starting_path_name = logs_and_reports[0][0]
+    return organize_node(starting_path_name)

siliconcompiler/report/dashboard/viewer.py

@@ -0,0 +1,36 @@
+import streamlit
+
+from siliconcompiler.report.dashboard import components
+from siliconcompiler.report.dashboard import layouts
+from siliconcompiler.report.dashboard import state
+from siliconcompiler.report.dashboard import utils
+
+import streamlit_autorefresh
+
+
+if __name__ == "__main__":
+    # opened by running command in siliconcompiler/apps/sc_dashboard.py
+    state.init()
+
+    components.setup_page()
+    state.setup()
+
+    layout = layouts.get_layout(state.get_key(state.APP_LAYOUT))
+    layout()
+
+    reload = False
+    if state.get_key(state.SELECTED_JOB) == 'default':
+        reload = state.set_key(state.IS_RUNNING, utils.is_running(state.get_chip()))
+
+    if state.get_key(state.IS_RUNNING):
+        update_interval = state.get_key(state.APP_RUNNING_REFRESH)
+    else:
+        update_interval = state.get_key(state.APP_STOPPED_REFRESH)
+
+    streamlit_autorefresh.st_autorefresh(interval=update_interval)
+
+    state.debug_print_state()
+
+    if reload or state.update_manifest() or state.get_key(state.APP_RERUN):
+        state.set_key(state.APP_RERUN, None)
+        streamlit.rerun()

siliconcompiler/report/report.py

@@ -1,3 +1,4 @@
+import fnmatch
 import pandas
 import os
 from siliconcompiler import Schema
@@ -104,7 +105,8 @@ def make_manifest_helper(manifest_subsect, modified_manifest_subsect):
 
     def build_leaf(manifest_subsect):
         if manifest_subsect['pernode'] == 'never':
-            if Schema.GLOBAL_KEY in manifest_subsect['node']:
+            if Schema.GLOBAL_KEY in manifest_subsect['node'] and \
+                    Schema.GLOBAL_KEY in manifest_subsect['node'][Schema.GLOBAL_KEY]:
                 value = manifest_subsect['node'][Schema.GLOBAL_KEY][Schema.GLOBAL_KEY]['value']
             else:
                 value = manifest_subsect['node']['default']['default']['value']
@@ -201,7 +203,7 @@ def search_manifest_keys(manifest, key):
     '''
     filtered_manifest = {}
     for dict_key in manifest:
-        if key in dict_key:
+        if fnmatch.fnmatch(dict_key, key):
             filtered_manifest[dict_key] = manifest[dict_key]
         elif isinstance(manifest[dict_key], dict):
             result = search_manifest_keys(manifest[dict_key], key)
@@ -225,8 +227,16 @@ def search_manifest_values(manifest, value):
             result = search_manifest_values(manifest[key], value)
             if result:  # result is non-empty
                 filtered_manifest[key] = result
-        elif isinstance(manifest[key], str) and value in manifest[key]:
-            filtered_manifest[key] = manifest[key]
+        else:
+            if manifest[key] is None:
+                continue
+
+            if isinstance(manifest[key], (list, tuple)):
+                if fnmatch.filter([str(v) for v in manifest[key]], value):
+                    filtered_manifest[key] = manifest[key]
+            else:
+                if fnmatch.fnmatch(str(manifest[key]), value):
+                    filtered_manifest[key] = manifest[key]
     return filtered_manifest
 
 
@@ -251,8 +261,12 @@ def search_manifest(manifest, key_search=None, value_search=None):
     '''
     return_manifest = manifest
     if key_search:
+        if '*' not in key_search or '?' not in key_search:
+            key_search = f'*{key_search}*'
         return_manifest = search_manifest_keys(return_manifest, key_search)
     if value_search:
+        if '*' not in value_search or '?' not in value_search:
+            value_search = f'*{value_search}*'
         return_manifest = search_manifest_values(return_manifest, value_search)
     return return_manifest
 
@@ -286,7 +300,11 @@ def get_metrics_source(chip, step, index):
     '''
     file_to_metric = {}
     tool, task = get_tool_task(chip, step, index)
+    if not chip.valid('tool', tool, 'task', task, 'report'):
+        return file_to_metric
+
     metrics = chip.getkeys('tool', tool, 'task', task, 'report')
+
     for metric in metrics:
         sources = chip.get('tool', tool, 'task', task, 'report', metric, step=step, index=index)
         for source in sources:

siliconcompiler/report/summary_table.py

@@ -12,10 +12,9 @@ def _show_summary_table(chip, flow, flowgraph_nodes, show_all_indices):
     '''
 
     # Display data
-    max_line_width = 135
     column_width = 15
 
-    max_line_width = max(max_line_width, int(0.9*shutil.get_terminal_size((80, 20)).columns))
+    max_line_width = max(2 * column_width, int(0.95*shutil.get_terminal_size().columns))
 
     nodes, _, metrics, metrics_unit, metrics_to_show, _ = \
         _collect_data(chip, flow, flowgraph_nodes)

siliconcompiler/report/utils.py

@@ -143,8 +143,7 @@ def _get_flowgraph_path(chip, flow, nodes_to_execute, only_include_successful=Fa
     end_nodes = _get_flowgraph_exit_nodes(chip, flow, steps=flowgraph_steps)
     for node in end_nodes:
         if only_include_successful:
-            if chip.get('record', 'status', step=node[0], index=node[1]) == \
-               NodeStatus.SUCCESS:
+            if NodeStatus.is_success(chip.get('record', 'status', step=node[0], index=node[1])):
                 selected_nodes.add(node)
                 to_search.append(node)
         else: