siliconcompiler 0.32.2__py3-none-any.whl → 0.32.3__py3-none-any.whl

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.32.2'
+version = '0.32.3'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'
@@ -21,7 +21,8 @@ authors = [
     'Aulihan Teng',
     'Peter Grossmann',
     'Gabriel Aguirre',
-    'Martin Troiber'
+    'Martin Troiber',
+    'Ciprian Antoci'
 ]
 
 # CLI entry banner autogenerated using pyfiglet.

siliconcompiler/core.py

@@ -17,16 +17,20 @@ import graphviz
 import codecs
 import copy
 from inspect import getfullargspec
-from siliconcompiler.remote import client
 from siliconcompiler.schema import Schema, SCHEMA_VERSION
 from siliconcompiler.schema import utils as schema_utils
 from siliconcompiler import utils
-from siliconcompiler.utils.logging import LoggerFormatter, ColorStreamFormatter
+from siliconcompiler.utils.logging import SCColorLoggerFormatter, \
+    SCLoggerFormatter, SCInRunLoggerFormatter, \
+    SCDebugLoggerFormatter, SCDebugInRunLoggerFormatter, \
+    SCBlankLoggerFormatter
 from siliconcompiler import _metadata
 from siliconcompiler import NodeStatus, SiliconCompilerError
 from siliconcompiler.report import _show_summary_table
 from siliconcompiler.report import _generate_summary_image, _open_summary_image
-from siliconcompiler.report import Dashboard
+from siliconcompiler.report.dashboard.web import WebDashboard
+from siliconcompiler.report.dashboard.cli import CliDashboard
+from siliconcompiler.report.dashboard import DashboardType
 from siliconcompiler import package as sc_package
 import glob
 from siliconcompiler.scheduler import run as sc_runner
@@ -238,62 +242,39 @@ class Chip:
             self.logger._console = stream_handler
             self.logger.addHandler(stream_handler)
 
-            self.logger._support_color = ColorStreamFormatter.supports_color(stream_handler)
+            self.logger._support_color = SCColorLoggerFormatter.supports_color(stream_handler)
 
         self._init_logger_formats(loglevel=loglevel)
 
     def _init_logger_formats(self, loglevel=None):
         if not loglevel:
-            self.schema.get('option', 'loglevel',
-                            step=self.logger._in_step, index=self.logger._in_index)
-
-        level_format = '%(levelname)-7s'
-        log_format = [level_format]
-        if loglevel == 'debug':
-            log_format.append('%(funcName)-10s')
-            log_format.append('%(lineno)-4s')
-
-        if self.logger._in_run:
-            max_column_width = 20
-            # Figure out how wide to make step and index fields
-            max_step_len = 1
-            max_index_len = 1
-            nodes_to_run = _get_flowgraph_nodes(self, flow=self.get('option', 'flow'))
-            if self.get('option', 'remote'):
-                nodes_to_run.append((client.remote_step_name, '0'))
-            for future_step, future_index in nodes_to_run:
-                max_step_len = max(len(future_step), max_step_len)
-                max_index_len = max(len(future_index), max_index_len)
-            max_step_len = min(max_step_len, max_column_width)
-            max_index_len = min(max_index_len, max_column_width)
-
-            jobname = self.get('option', 'jobname')
-
-            step = self.logger._in_step
-            index = self.logger._in_index
-
-            if step is None:
-                step = '-' * max(max_step_len // 4, 1)
-            if index is None:
-                index = '-' * max(max_index_len // 4, 1)
-
-            log_format.append(utils.truncate_text(jobname, max_column_width))
-            log_format.append(f'{utils.truncate_text(step, max_step_len): <{max_step_len}}')
-            log_format.append(f'{utils.truncate_text(index, max_step_len): >{max_index_len}}')
-
-        log_formatprefix = "| "
-        if loglevel == "quiet":
-            log_format = []
-            log_formatprefix = ""
-
-        log_format.append('%(message)s')
-        stream_logformat = log_formatprefix + ' | '.join(log_format[1:])
+            loglevel = self.schema.get('option', 'loglevel',
+                                       step=self.logger._in_step, index=self.logger._in_index)
+
+        if loglevel == 'quiet':
+            base_format = SCBlankLoggerFormatter()
+        elif self.logger._in_run:
+            if loglevel == 'debug':
+                base_format = SCDebugInRunLoggerFormatter(
+                    self,
+                    self.get('option', 'jobname'),
+                    self.logger._in_step, self.logger._in_index)
+            else:
+                base_format = SCInRunLoggerFormatter(
+                    self,
+                    self.get('option', 'jobname'),
+                    self.logger._in_step, self.logger._in_index)
+        else:
+            if loglevel == 'debug':
+                base_format = SCDebugLoggerFormatter()
+            else:
+                base_format = SCLoggerFormatter()
 
         for handler in self.logger.handlers.copy():
             if handler == self.logger._console and self.logger._support_color:
-                formatter = ColorStreamFormatter(log_formatprefix, level_format, stream_logformat)
+                formatter = SCColorLoggerFormatter(base_format)
             else:
-                formatter = LoggerFormatter(log_formatprefix, level_format, stream_logformat)
+                formatter = base_format
             handler.setFormatter(formatter)
 
     ###########################################################################
@@ -2867,7 +2848,7 @@ class Chip:
         return hashlist
 
     ###########################################################################
-    def dashboard(self, wait=True, port=None, graph_chips=None):
+    def dashboard(self, wait=True, port=None, graph_chips=None, type=DashboardType.WEB):
         '''
         Open a session of the dashboard.
 
@@ -2881,6 +2862,8 @@ class Chip:
                 dashboard to.
             graph_chips (list): A list of dictionaries of the format
                 {'chip': chip object, 'name': chip name}
+            type (enum): A string specifying what kind of dashboard to
+                launch. Available options: 'cli', 'web'.
 
         Examples:
             >>> chip.dashboard()
@@ -2891,7 +2874,14 @@ class Chip:
             self._dash.stop()
             self._dash = None
 
-        self._dash = Dashboard(self, port=port, graph_chips=graph_chips)
+        # Select dashboard type
+        type = DashboardType(type)
+        if type == DashboardType.WEB:
+            self._dash = WebDashboard(self, port=port, graph_chips=graph_chips)
+        elif type == DashboardType.CLI:
+            self._dash = CliDashboard(self)
+            wait = False
+
         self._dash.open_dashboard()
 
         if wait:
@@ -3017,7 +3007,7 @@ class Chip:
             flow (str): Flow name
             step (str): Step name
             task (module/str): Task to associate with this node
-            index (int): Step index
+            index (int/str): Step index
 
         Examples:
             >>> import siliconcomiler.tools.openroad.place as place
@@ -3076,8 +3066,8 @@ class Chip:
             flow (str): Name of flow
             tail (str): Name of tail node
             head (str): Name of head node
-            tail_index (int): Index of tail node to connect
-            head_index (int): Index of head node to connect
+            tail_index (int/str): Index of tail node to connect
+            head_index (int/str): Index of head node to connect
 
         Examples:
             >>> chip.edge('place', 'cts')
@@ -3107,7 +3097,7 @@ class Chip:
         Args:
             flow (str): Flow name
             step (str): Step name
-            index (int): Step index
+            index (int/str): Step index
         '''
 
         if flow not in self.getkeys('flowgraph'):
@@ -3219,6 +3209,12 @@ class Chip:
                 raise e
             self.logger.error(str(e))
             return False
+        finally:
+            # Update dashboard if running
+            if self._dash:
+                self._dash.update_manifest()
+                self._dash.end_of_run()
+
         return True
 
     ###########################################################################

siliconcompiler/issue.py

@@ -273,7 +273,9 @@ def generate_testcase(chip,
         full_archive_path = os.path.join(archive_directory, archive_name)
     full_archive_path = os.path.abspath(full_archive_path)
     # Build archive
-    arch_base_dir = os.path.basename(archive_name).split('.')[0]
+    arch_base_dir = os.path.basename(archive_name)
+    while arch_base_dir.lower().split('.')[-1] in ('gz', 'tar'):
+        arch_base_dir = '.'.join(arch_base_dir.split('.')[0:-1])
     with tarfile.open(full_archive_path, "w:gz") as tar:
         # Add individual files
         add_files = [manifest_path,

siliconcompiler/remote/client.py

@@ -10,11 +10,12 @@ import tarfile
 import tempfile
 import multiprocessing
 
-from siliconcompiler import utils, SiliconCompilerError
+from siliconcompiler import utils, SiliconCompilerError, NodeStatus
 from siliconcompiler import NodeStatus as SCNodeStatus
 from siliconcompiler._metadata import default_server
 from siliconcompiler.flowgraph import nodes_to_execute
 from siliconcompiler.remote import JobStatus
+from siliconcompiler.report.dashboard import DashboardType
 
 # Step name to use while logging
 remote_step_name = 'remote'
@@ -298,18 +299,19 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             self.__logger.info(nodes_log)
 
     def _report_job_status(self, info):
+        completed = []
+        starttimes = {}
+
         if not info['busy']:
             # Job is not running
-            return [], False
+            return completed, starttimes, False
 
         try:
             # Decode response JSON, if possible.
             job_info = json.loads(info['message'])
         except json.JSONDecodeError as e:
             self.__logger.warning(f"Job is still running: {e}")
-            return [], True
-
-        completed = []
+            return completed, starttimes, True
 
         nodes_to_log = {}
         for node, node_info in job_info.items():
@@ -335,7 +337,16 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 self.__log_node_status(stat, nodes)
 
         # Running / in-progress flowgraph nodes should all be printed:
+        base_time = time.time()
         for stat, nodes in nodes_to_log.items():
+            for node, node_info in nodes:
+                if 'elapsed_time' in node_info:
+                    runtime = 0
+                    for part in node_info['elapsed_time'].split(":"):
+                        runtime = 60 * runtime + float(part)
+                    starttimes[(self.__node_information[node]["step"],
+                                self.__node_information[node]["index"])] = base_time - runtime
+
             if SCNodeStatus.is_running(stat):
                 self.__logger.info(f'  {stat.title()} ({len(nodes)}):')
                 for node, node_info in nodes:
@@ -349,7 +360,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             if SCNodeStatus.is_waiting(stat):
                 self.__log_node_status(stat, nodes)
 
-        return completed, True
+        return completed, starttimes, True
 
     def __check(self):
         def post_action(url):
@@ -438,6 +449,9 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             raise SiliconCompilerError('Cannot pass [arg,index] parameter into remote flow.',
                                        chip=self.__chip)
 
+        if not self.__chip._dash:
+            self.__chip.dashboard(type=DashboardType.CLI)
+
         # Only run the pre-process step if the job doesn't already have a remote ID.
         if not remote_resume:
             self.__run_preprocess()
@@ -450,10 +464,12 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         self.__request_run()
 
         # Run the main 'check_progress' loop to monitor job status until it finishes.
-        self._run_loop()
-
-        # Restore logger
-        self.__chip._init_logger(in_run=True)
+        try:
+            self._run_loop()
+        finally:
+            # Restore logger
+            self.__chip._dash.end_of_run()
+            self.__chip._init_logger(in_run=True)
 
     def __request_run(self):
         '''
@@ -559,7 +575,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             self.__logger.info(f'To cancel this job use: {cancel_cmd}')
             raise SiliconCompilerError('Job canceled by user keyboard interrupt')
 
-    def __import_run_manifests(self):
+    def __import_run_manifests(self, starttimes):
         changed = False
         for _, node_info in self.__node_information.items():
             if node_info["imported"]:
@@ -577,6 +593,16 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 except:  # noqa E722
                     # Import may fail if file is still getting written
                     pass
+            elif self.__chip.get('record', 'status',
+                                 step=node_info["step"], index=node_info["index"]) \
+                    == NodeStatus.SKIPPED:
+                node_info["imported"] = True
+                changed = True
+
+        if changed and self.__chip._dash:
+            # Update dashboard if active
+            self.__chip._dash.update_manifest({"starttimes": starttimes})
+
         return changed
 
     def __ensure_run_loop_information(self):
@@ -605,17 +631,30 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         # Check the job's progress periodically until it finishes.
         running = True
 
+        starttimes = {}
+
         while running:
-            time.sleep(self.__check_interval)
-            self.__import_run_manifests()
+            sleepremaining = self.__check_interval
+            while any([nodeinfo["fetched"] and not nodeinfo["imported"]
+                       for nodeinfo in self.__node_information.values()]):
+                self.__import_run_manifests(starttimes)
+                sleepremaining -= 1
+                if sleepremaining <= 0:
+                    break
+                time.sleep(1)
+            if sleepremaining > 0:
+                time.sleep(sleepremaining)
 
             # Check progress
             job_info = self.check_job_status()
-            completed, running = self._report_job_status(job_info)
+            completed, new_starttimes, running = self._report_job_status(job_info)
+
+            # preserve old starttimes
+            starttimes = {**starttimes, **new_starttimes}
 
             if self.__chip._dash:
                 # Update dashboard if active
-                self.__chip._dash.update_manifest()
+                self.__chip._dash.update_manifest({"starttimes": starttimes})
 
             nodes_to_fetch = []
             for node in completed:
@@ -648,7 +687,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             self.__download_pool.join()
             self.__download_pool = None
 
-        self.__import_run_manifests()
+        self.__import_run_manifests({})
 
     def __schedule_fetch_result(self, node):
         self.__node_information[node]["fetched"] = True

siliconcompiler/report/__init__.py

@@ -1,7 +1,8 @@
 from .summary_image import _generate_summary_image, _open_summary_image
 from .html_report import _generate_html_report, _open_html_report
 from .summary_table import _show_summary_table
-from .dashboard import Dashboard
+from .dashboard.web import WebDashboard
+# from .dashboard import Dashboard
 
 __all__ = [
     "_generate_summary_image",
@@ -9,5 +10,5 @@ __all__ = [
     "_generate_html_report",
     "_open_html_report",
     "_show_summary_table",
-    "Dashboard"
+    "WebDashboard"
 ]

siliconcompiler/report/dashboard/__init__.py

@@ -1,190 +1,81 @@
-import os
-import time
-import tempfile
-import json
+from abc import ABC, abstractmethod
+from enum import Enum
 
-import multiprocessing
-import subprocess
-import atexit
-import shutil
-import fasteners
-import signal
-import socketserver
 
-from siliconcompiler.report.dashboard import utils
+class DashboardType(Enum):
+    WEB = 'web'
+    CLI = 'cli'
 
-try:
-    from streamlit.web import bootstrap
-    from streamlit import config as _config
-except ModuleNotFoundError:
-    bootstrap = None
-    _config = None
 
+class AbstractDashboard(ABC):
+    """
+    Abstract base class defining the interface for dashboard implementations.
+    Any concrete dashboard implementation should inherit from this class and
+    implement all abstract methods.
+    """
 
-class Dashboard():
-    __port = 8501
+    @abstractmethod
+    def __init__(self, chip):
+        """
+        Initialize the dashboard.
 
-    @staticmethod
-    def __signal_handler(signal, frame):
-        # used to avoid issues during shutdown
-        pass
-
-    def __init__(self, chip, port=None, graph_chips=None):
-        if not bootstrap:
-            raise NotImplementedError('streamlit is not available')
-
-        if not port:
-            port = Dashboard.get_next_port()
-        if not port:
-            port = Dashboard.__port
-
-        self.__dashboard = None
-        self.__chip = chip
-        self.__directory = tempfile.mkdtemp(prefix='sc_dashboard_',
-                                            suffix=f'_{self.__chip.design}')
-        self.__manifest = os.path.join(self.__directory, 'manifest.json')
-        self.__manifest_lock = os.path.join(self.__directory, 'manifest.lock')
-        self.__port = port
-        dirname = os.path.dirname(__file__)
-        self.__streamlit_file = os.path.join(dirname, 'viewer.py')
-
-        self.__streamlit_args = [
-            ("browser.gatherUsageStats", False),
-            ("browser.serverPort", self.__port),
-            ("logger.level", 'error'),
-            ("runner.fastReruns", True),
-            ("server.port", self.__port),
-            ("client.toolbarMode", "viewer")
-        ]
-
-        # pass in a json object called __graph_chips
-        # the key is the chip_name and value is the filepath
-        # if another argument is passed
-
-        # use of list is to preserve order
-        self.__graph_chips = []
-        graph_chips_config = []
-        if graph_chips:
-            for chip_object_and_name in graph_chips:
-                chip_file_path = \
-                    os.path.join(self.__directory,
-                                 f"{chip_object_and_name['name']}.json")
-                self.__graph_chips.append({
-                    'chip': chip_object_and_name['chip'],
-                    'name': chip_file_path
-                })
-                graph_chips_config.append({
-                    "path": chip_file_path,
-                    "cwd": utils.get_chip_cwd(
-                        chip_object_and_name['chip'],
-                        chip_object_and_name['cfg_path'])
-                })
-
-        self.__config = {
-            "manifest": self.__manifest,
-            "lock": self.__manifest_lock,
-            "graph_chips": graph_chips_config
-        }
-
-        self.__sleep_time = 0.5
-        self.__signal_handler = None
-
-        self.__lock = fasteners.InterProcessLock(self.__manifest_lock)
-
-        atexit.register(self.__cleanup)
+        Args:
+            chip: The chip object to display in the dashboard
+        """
+        self._chip = chip
 
+    @abstractmethod
     def open_dashboard(self):
-        with open(self.__get_config_file(), 'w') as f:
-            json.dump(self.__config, f, indent=4)
-
-        self.update_manifest()
-
-        self.update_graph_manifests()
-
-        self.__dashboard = multiprocessing.Process(
-            target=self._run_streamlit_bootstrap)
-
-        self.__signal_handler = signal.signal(signal.SIGINT, Dashboard.__signal_handler)
-
-        self.__dashboard.start()
-
-    def update_manifest(self):
-        if not self.__manifest:
-            return
+        """
+        Open and start the dashboard service.
+        """
+        pass
 
-        new_file = f"{self.__manifest}.new.json"
-        self.__chip.write_manifest(new_file)
+    @abstractmethod
+    def update_manifest(self, payload=None):
+        """
+        Update the manifest file with the latest chip information.
 
-        with self.__lock:
-            shutil.move(new_file, self.__manifest)
+        Args:
+            payload (dict): Dictionary of metadata to pass along to dashboard.
+                {"starttimes" {<node>: time, ...}}
+        """
+        pass
 
+    @abstractmethod
     def update_graph_manifests(self):
-        for chip_object_and_name in self.__graph_chips:
-            chip = chip_object_and_name['chip']
-            file_path = chip_object_and_name['name']
-            chip.write_manifest(file_path)
-
-    def __get_config_file(self):
-        return os.path.join(self.__directory, 'config.json')
+        """
+        Update the manifest files for all graph chips.
+        """
+        pass
 
+    @abstractmethod
     def is_running(self):
-        if self.__dashboard is None:
-            return False
+        """
+        Check if the dashboard is currently running.
 
-        if self.__dashboard.is_alive():
-            return True
+        Returns:
+            bool: True if the dashboard is running, False otherwise
+        """
+        pass
 
-        self.__dashboard = None
-        self.__manifest = None
-        return False
+    @abstractmethod
+    def end_of_run(self):
+        """
+        Announce that a run has completed
+        """
+        pass
 
+    @abstractmethod
     def stop(self):
-        if not self.is_running():
-            return
-
-        while self.__dashboard.is_alive():
-            self.__dashboard.terminate()
-            self._sleep()
-
-        if self.__signal_handler:
-            signal.signal(signal.SIGINT, self.__signal_handler)
-
-        self.__dashboard = None
-        self.__manifest = None
-        self.__signal_handler = None
+        """
+        Stop the dashboard service if it's running.
+        """
+        pass
 
+    @abstractmethod
     def wait(self):
-        self.__dashboard.join()
-
-    def _sleep(self):
-        time.sleep(self.__sleep_time)
-
-    def _run_streamlit_bootstrap(self):
-        for config, val in self.__streamlit_args:
-            _config.set_option(config, val)
-
-        bootstrap.run(self.__streamlit_file,
-                      False,
-                      [self.__get_config_file()],
-                      flag_options={})
-
-    def __run_streamlit_subproc(self):
-        cmd = ['streamlit', 'run',
-               self.__streamlit_file, self.__get_config_file()]
-        for config, val in self.__streamlit_args:
-            cmd.append(f'--{config}')
-            cmd.append(val)
-
-        subprocess.Popen(cmd)
-
-    def __cleanup(self):
-        self.stop()
-
-        if os.path.exists(self.__directory):
-            shutil.rmtree(self.__directory)
-
-    @staticmethod
-    def get_next_port():
-        with socketserver.TCPServer(("localhost", 0), None) as s:
-            return s.server_address[1]
-        return None
+        """
+        Wait for the dashboard service to terminate.
+        """
+        pass