siliconcompiler 0.33.0__py3-none-any.whl → 0.33.1__py3-none-any.whl

siliconcompiler/_common.py

@@ -14,6 +14,7 @@ class NodeStatus():
     SKIPPED = 'skipped'
     TIMEOUT = 'timeout'
 
+    @staticmethod
     def is_done(status):
         return status in (
             NodeStatus.SUCCESS,
@@ -22,23 +23,27 @@ class NodeStatus():
             NodeStatus.TIMEOUT
         )
 
+    @staticmethod
     def is_running(status):
         return status in (
             NodeStatus.QUEUED,
             NodeStatus.RUNNING
         )
 
+    @staticmethod
     def is_waiting(status):
         return status in (
             NodeStatus.PENDING,
         )
 
+    @staticmethod
     def is_success(status):
         return status in (
             NodeStatus.SUCCESS,
             NodeStatus.SKIPPED
         )
 
+    @staticmethod
     def is_error(status):
         return status in (
             NodeStatus.ERROR,

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.33.0'
+version = '0.33.1'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_install.py

@@ -42,6 +42,13 @@ def install_tool(tool, script, build_dir, prefix):
     # setup environment
     env = os.environ.copy()
     env["PREFIX"] = prefix
+    env["USE_SUDO_INSTALL"] = "no"
+    try:
+        os.makedirs(prefix, exist_ok=True)
+    except PermissionError:
+        env["USE_SUDO_INSTALL"] = "yes"
+    if not os.access(prefix, os.W_OK):
+        env["USE_SUDO_INSTALL"] = "yes"
 
     # run
     ret = subprocess.call(script, env=env, cwd=build_dir)

siliconcompiler/apps/sc_remote.py

@@ -6,7 +6,7 @@ from siliconcompiler import Chip
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler.remote.client import Client, ConfigureClient
 from siliconcompiler.scheduler import _finalize_run
-from siliconcompiler.utils.flowgraph import nodes_to_execute
+from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def main():
@@ -170,7 +170,12 @@ To delete a job, use:
             return 1
 
         # Wrap up run
-        for step, index in nodes_to_execute(chip):
+        runtime = RuntimeFlowgraph(
+            chip.schema.get("flowgraph", flow, field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+        for step, index in runtime.get_nodes():
             manifest = os.path.join(chip.getworkdir(step=step, index=index),
                                     'outputs',
                                     f'{chip.design}.pkg.json')

siliconcompiler/core.py

@@ -37,10 +37,7 @@ from siliconcompiler.report.dashboard import DashboardType
 from siliconcompiler import package as sc_package
 import glob
 from siliconcompiler.scheduler import run as sc_runner
-from siliconcompiler.utils.flowgraph import nodes_to_execute, \
-    _get_pruned_node_inputs, \
-    _get_flowgraph_execution_order, _check_flowgraph_io, \
-    _get_flowgraph_information
+from siliconcompiler.utils.flowgraph import _check_flowgraph_io, _get_flowgraph_information
 from siliconcompiler.tools._common import get_tool_task
 from types import FunctionType, ModuleType
 from siliconcompiler.flowgraph import RuntimeFlowgraph
@@ -1622,11 +1619,26 @@ class Chip:
             error = True
             self.logger.error(f"flowgraph {flow} not defined.")
 
-        nodes = [node for node in nodes_to_execute(self)
+        runtime = RuntimeFlowgraph(
+            self.schema.get("flowgraph", flow, field='schema'),
+            args=(self.get('arg', 'step'), self.get('arg', 'index')),
+            from_steps=self.get('option', 'from'),
+            to_steps=self.get('option', 'to'),
+            prune_nodes=self.get('option', 'prune'))
+
+        nodes = [node for node in runtime.get_nodes()
                  if self.get('record', 'status', step=node[0], index=node[1])
                  != NodeStatus.SKIPPED]
+        flow_schema = self.schema.get("flowgraph", flow, field="schema")
+        runtime_io = RuntimeFlowgraph(
+            flow_schema,
+            args=(self.get('arg', 'step'), self.get('arg', 'index')),
+            from_steps=set([step for step, _ in flow_schema.get_entry_nodes()]),
+            prune_nodes=self.get('option', 'prune'))
+
         for (step, index) in nodes:
-            for in_step, in_index in _get_pruned_node_inputs(self, flow, (step, index)):
+            for in_step, in_index in runtime_io.get_node_inputs(
+                    step, index, record=self.schema.get("record", field="schema")):
                 if (in_step, in_index) in nodes:
                     # we're gonna run this step, OK
                     continue
@@ -1717,7 +1729,7 @@ class Chip:
                 error = True
                 self.logger.error(f'No executable or run() function specified for {tool}/{task}')
 
-        if not _check_flowgraph_io(self, nodes=nodes):
+        if not error and not _check_flowgraph_io(self, nodes=nodes):
             error = True
 
         return not error
@@ -2748,7 +2760,13 @@ class Chip:
             flow = self.get('option', 'flow')
             flowgraph_nodes = [(step, index) for index in self.getkeys("flowgraph", flow, step)]
         else:
-            flowgraph_nodes = nodes_to_execute(self)
+            flow = self.get('option', 'flow')
+            runtime = RuntimeFlowgraph(
+                self.schema.get("flowgraph", flow, field='schema'),
+                from_steps=self.get('option', 'from'),
+                to_steps=self.get('option', 'to'),
+                prune_nodes=self.get('option', 'prune'))
+            flowgraph_nodes = runtime.get_nodes()
 
         if not archive_name:
             if step and index:
@@ -3231,14 +3249,18 @@ class Chip:
                 search_nodes.append((sc_step, sc_index))
             elif sc_step:
                 if flow is not None:
-                    for check_step, check_index in nodes_to_execute(self, flow):
+                    runtime = RuntimeFlowgraph(
+                        self.schema.get("flowgraph", flow, field='schema'),
+                        from_steps=self.get('option', 'from'),
+                        to_steps=self.get('option', 'to'),
+                        prune_nodes=self.get('option', 'prune'))
+                    for check_step, check_index in runtime.get_nodes():
                         if sc_step == check_step:
                             search_nodes.append((check_step, check_index))
             else:
                 if flow is not None:
-                    for nodes in _get_flowgraph_execution_order(self,
-                                                                flow,
-                                                                reverse=True):
+                    for nodes in self.schema.get(
+                            "flowgraph", flow, field="schema").get_execution_order(reverse=True):
                         search_nodes.extend(nodes)
 
             for ext in self._showtools.keys():

siliconcompiler/metric.py

@@ -3,6 +3,7 @@ from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
 from siliconcompiler.schema.utils import trim
 
 from siliconcompiler.utils.units import convert
+from siliconcompiler.record import RecordTime
 
 
 class MetricSchema(BaseSchema):
@@ -43,6 +44,64 @@ class MetricSchema(BaseSchema):
 
         return self.set(metric, value, step=step, index=str(index))
 
+    def record_totaltime(self, step, index, flow, record):
+        """
+        Record the total time for this node
+
+        Args:
+            step (str): step to record
+            index (str/int): index to record
+            flow (:class:`FlowgraphSchema`): flowgraph to lookup nodes in
+            record (:class:`RecordSchema`): record to lookup data in
+        """
+        all_nodes = flow.get_nodes()
+        node_times = [
+            (record.get_recorded_time(*node, RecordTime.START),
+             record.get_recorded_time(*node, RecordTime.END)) for node in all_nodes
+        ]
+
+        # Remove incomplete records
+        node_times = [times for times in node_times if times[0] is not None]
+
+        if len(node_times) == 0:
+            return False
+
+        node_end = record.get_recorded_time(step, index, RecordTime.END)
+        if node_end is None:
+            return False
+
+        node_times = sorted(node_times)
+        if len(node_times) > 1:
+            new_times = []
+            for n in range(len(node_times)):
+                if not new_times:
+                    new_times.append(node_times[n])
+                    continue
+                prev_start_time, prev_end_time = new_times[-1]
+                start_time, end_time = node_times[n]
+
+                new_start = min(prev_start_time, start_time)
+
+                if prev_end_time is None:
+                    new_times[-1] = (new_start, end_time)
+                elif prev_end_time >= start_time:
+                    if end_time is not None:
+                        new_end = max(prev_end_time, end_time)
+                    else:
+                        new_end = prev_end_time
+                    new_times[-1] = (new_start, new_end)
+                else:
+                    new_times.append(node_times[n])
+            node_times = new_times
+
+        total_time = 0
+        for start_time, end_time in node_times:
+            if start_time > node_end:
+                continue
+            total_time += min(end_time, node_end) - start_time
+
+        return self.record(step, index, "totaltime", total_time, unit="s")
+
 
 ###########################################################################
 # Metrics to Track

siliconcompiler/record.py

@@ -259,7 +259,7 @@ class RecordSchema(BaseSchema):
 
     def get_recorded_time(self, step, index, type):
         '''
-        Returns the time recorded for a given record.
+        Returns the time recorded for a given record, or None if nothing is recorded.
 
         Args:
             step (str): Step name to associate.
@@ -267,8 +267,12 @@ class RecordSchema(BaseSchema):
             type (:class:`RecordTime`): type of time to record
         '''
         type = RecordTime(type)
+        record_time = self.get(type.value, step=step, index=index)
+        if record_time is None:
+            return None
+
         return datetime.strptime(
-            self.get(type.value, step=step, index=index)+"+0000",
+            record_time+"+0000",
             RecordSchema.__TIMEFORMAT+"%z").timestamp()
 
     def record_tool(self, step, index, info, type):

siliconcompiler/remote/client.py

@@ -13,9 +13,10 @@ import multiprocessing
 from siliconcompiler import utils, SiliconCompilerError, NodeStatus
 from siliconcompiler import NodeStatus as SCNodeStatus
 from siliconcompiler._metadata import default_server
-from siliconcompiler.utils.flowgraph import nodes_to_execute
 from siliconcompiler.remote import JobStatus
 from siliconcompiler.report.dashboard import DashboardType
+from siliconcompiler.flowgraph import RuntimeFlowgraph
+from siliconcompiler.schema import JournalingSchema
 
 # Step name to use while logging
 remote_step_name = 'remote'
@@ -590,7 +591,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 f'{self.__chip.design}.pkg.json')
             if os.path.exists(manifest):
                 try:
-                    self.__chip.schema.read_journal(manifest)
+                    JournalingSchema(self.__chip.schema).read_journal(manifest)
                     node_info["imported"] = True
                     changed = True
                 except:  # noqa E722
@@ -618,7 +619,13 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             self.__check_interval = check_info['progress_interval']
 
         self.__node_information = {}
-        for step, index in nodes_to_execute(self.__chip):
+        runtime = RuntimeFlowgraph(
+            self.__chip.schema.get("flowgraph", self.__chip.get('option', 'flow'), field='schema'),
+            from_steps=self.__chip.get('option', 'from'),
+            to_steps=self.__chip.get('option', 'to'),
+            prune_nodes=self.__chip.get('option', 'prune'))
+
+        for step, index in runtime.get_nodes():
             done = SCNodeStatus.is_done(self.__chip.get('record', 'status', step=step, index=index))
             node_info = {
                 "step": step,

siliconcompiler/remote/server.py

@@ -21,7 +21,7 @@ from siliconcompiler.schema import SCHEMA_VERSION as sc_schema_version
 from siliconcompiler.remote.schema import ServerSchema
 from siliconcompiler.remote import banner, JobStatus
 from siliconcompiler.scheduler.slurm import get_configuration_directory
-from siliconcompiler.utils.flowgraph import nodes_to_execute
+from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 # Compile validation code for API request bodies.
@@ -422,7 +422,12 @@ class Server:
         chip.run()
 
         # Archive each task.
-        for (step, index) in nodes_to_execute(chip):
+        runtime = RuntimeFlowgraph(
+            chip.schema.get("flowgraph", chip.get('option', 'flow'), field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+        for (step, index) in runtime.get_nodes():
             chip.cwd = os.path.join(chip.get('option', 'builddir'), '..')
             tf = tarfile.open(os.path.join(self.nfs_mount,
                                            job_hash,

siliconcompiler/report/dashboard/cli/__init__.py

@@ -66,6 +66,8 @@ class CliDashboard(AbstractDashboard):
         """
         self._dashboard.end_of_run(self._chip)
 
+        self._dashboard.stop()
+
         # Restore logger
         if self.__logger_console:
             self._logger.removeHandler(self._dashboard._log_handler)

siliconcompiler/report/dashboard/cli/board.py

@@ -22,6 +22,8 @@ from siliconcompiler import SiliconCompilerError, NodeStatus
 from siliconcompiler.utils.logging import SCColorLoggerFormatter
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
+import atexit
+
 
 class LogBufferHandler(logging.Handler):
     def __init__(self, sync_queue, n=50, event=None):
@@ -246,6 +248,15 @@ class Board(metaclass=BoardSingleton):
 
     def _init_singleton(self):
         self._console = Console(theme=Board.__theme)
+
+        self.live = Live(
+            console=self._console,
+            screen=False,
+            auto_refresh=True,
+        )
+
+        atexit.register(self._stop_on_exit)
+
         self._active = self._console.is_terminal
         if not self._active:
             self._console = None
@@ -279,6 +290,9 @@ class Board(metaclass=BoardSingleton):
 
         self._metrics = ("warnings", "errors")
 
+    def _stop_on_exit(self):
+        self.stop()
+
     def open_dashboard(self):
         """Starts the dashboard rendering thread if it is not already running."""
 
@@ -289,7 +303,7 @@ class Board(metaclass=BoardSingleton):
             self._update_render_data(None)
 
             with self._job_data_lock:
-                if not self._render_thread:
+                if not self._render_thread or not self._render_thread.is_alive():
                     self._render_thread = threading.Thread(target=self._render, daemon=True)
                     self._render_event.clear()
                     self._render_stop_event.clear()
@@ -328,7 +342,6 @@ class Board(metaclass=BoardSingleton):
             return
 
         self._update_render_data(chip, complete=True)
-        self.stop()
 
     def stop(self):
         """
@@ -337,18 +350,20 @@ class Board(metaclass=BoardSingleton):
         if not self.is_running():
             return
 
-        # check for running jobs
-        with self._job_data_lock:
-            if self._job_data:
-                if any([not job.complete for job in self._job_data.values()]):
-                    return
+        if self._job_data:
+            if any([not job.complete for job in self._job_data.values()]):
+                return
 
-            self._render_stop_event.set()
-            self._render_event.set()
+        self._render_stop_event.set()
+        self._render_event.set()
 
         # Wait for rendering to finish
         self.wait()
 
+        # Restore terminal
+        self.live.stop()
+        self._console.show_cursor()
+
     def wait(self):
         """Waits for the dashboard rendering thread to finish."""
         if not self.is_running():
@@ -595,18 +610,11 @@ class Board(metaclass=BoardSingleton):
                 # Catch any multiprocessing errors
                 return True
 
-        live = None
         try:
             update_data()
-            live = Live(
-                self._get_rendable(),
-                console=self._console,
-                screen=False,
-                # transient=True,
-                auto_refresh=True,
-                # refresh_per_second=60,
-            )
-            live.start()
+
+            if not self.live.is_started:
+                self.live.start(refresh=True)
 
             while not check_stop_event():
                 try:
@@ -620,18 +628,14 @@ class Board(metaclass=BoardSingleton):
                     break
 
                 update_data()
-                live.update(self._get_rendable(), refresh=True)
+                self.live.update(self._get_rendable(), refresh=True)
+
         finally:
-            try:
-                update_data()
-                if live:
-                    live.update(self._get_rendable(), refresh=True)
-                    live.stop()
-                else:
-                    self._console.print(self._get_rendable())
-            finally:
-                # Restore the prompt
-                print("\033[?25h", end="")
+            update_data()
+            if self.live:
+                self.live.update(self._get_rendable(), refresh=True)
+            else:
+                self._console.print(self._get_rendable())
 
     def _update_layout(self):
         with self._render_data_lock:

siliconcompiler/report/report.py

@@ -3,8 +3,8 @@ import os
 from siliconcompiler import Schema
 from siliconcompiler.schema import PerNode
 from siliconcompiler.report import utils
-from siliconcompiler.utils.flowgraph import nodes_to_execute
 from siliconcompiler.tools._common import get_tool_task
+from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def make_metric_dataframe(chip):
@@ -199,7 +199,12 @@ def get_flowgraph_path(chip):
         Returns the "winning" path for that job.
     '''
     flow = chip.get('option', 'flow')
-    return utils._get_flowgraph_path(chip, flow, nodes_to_execute(chip))
+    runtime = RuntimeFlowgraph(
+        chip.schema.get("flowgraph", flow, field='schema'),
+        from_steps=chip.get('option', 'from'),
+        to_steps=chip.get('option', 'to'),
+        prune_nodes=chip.get('option', 'prune'))
+    return utils._get_flowgraph_path(chip, flow, runtime.get_nodes())
 
 
 def search_manifest_keys(manifest, key):
@@ -289,9 +294,9 @@ def get_total_manifest_key_count(manifest):
         acc (int) : An accumulator of the current number of folders and files.
     '''
     acc = len(manifest)
-    for dictKeys in manifest:
-        if isinstance(manifest[dictKeys], dict):
-            acc += get_total_manifest_key_count(manifest[dictKeys])
+    for key in manifest:
+        if isinstance(manifest[key], dict):
+            acc += get_total_manifest_key_count(manifest[key])
     return acc
 
 

siliconcompiler/report/utils.py

@@ -1,14 +1,13 @@
 from siliconcompiler import NodeStatus
 from siliconcompiler.utils import units
-from siliconcompiler.utils.flowgraph import _get_flowgraph_execution_order, \
-    nodes_to_execute
 from siliconcompiler.tools._common import get_tool_task
 
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def _find_summary_image(chip, ext='png'):
-    for nodes in reversed(_get_flowgraph_execution_order(chip, chip.get('option', 'flow'))):
+    for nodes in reversed(chip.schema.get(
+            "flowgraph", chip.get('option', 'flow'), field="schema").get_execution_order()):
         for step, index in nodes:
             layout_img = chip.find_result(ext, step=step, index=index)
             if layout_img:
@@ -18,7 +17,8 @@ def _find_summary_image(chip, ext='png'):
 
 def _find_summary_metrics(chip, metrics_map):
     metrics = {}
-    for nodes in reversed(_get_flowgraph_execution_order(chip, chip.get('option', 'flow'))):
+    for nodes in reversed(chip.schema.get(
+            "flowgraph", chip.get('option', 'flow'), field="schema").get_execution_order()):
         for step, index in nodes:
             for name, metric_info in metrics_map.items():
                 if name in metrics:
@@ -44,7 +44,13 @@ def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
         return [], {}, {}, {}, [], {}
 
     if not flowgraph_nodes:
-        flowgraph_nodes = list(nodes_to_execute(chip))
+        runtime = RuntimeFlowgraph(
+            chip.schema.get("flowgraph", flow, field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+
+        flowgraph_nodes = list(runtime.get_nodes())
         # only report tool based steps functions
         for (step, index) in list(flowgraph_nodes):
             tool, task = get_tool_task(chip, step, '0', flow=flow)
@@ -64,7 +70,7 @@ def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
     reports = {}
 
     # Build ordered list of nodes in flowgraph
-    for level_nodes in _get_flowgraph_execution_order(chip, flow):
+    for level_nodes in chip.schema.get("flowgraph", flow, field="schema").get_execution_order():
         nodes.extend(sorted(level_nodes))
     nodes = [node for node in nodes if node in flowgraph_nodes]
     for (step, index) in nodes: