siliconcompiler 0.33.0__py3-none-any.whl → 0.33.2__py3-none-any.whl

siliconcompiler/_common.py

@@ -14,6 +14,7 @@ class NodeStatus():
     SKIPPED = 'skipped'
     TIMEOUT = 'timeout'
 
+    @staticmethod
     def is_done(status):
         return status in (
             NodeStatus.SUCCESS,
@@ -22,23 +23,27 @@ class NodeStatus():
             NodeStatus.TIMEOUT
         )
 
+    @staticmethod
     def is_running(status):
         return status in (
             NodeStatus.QUEUED,
             NodeStatus.RUNNING
         )
 
+    @staticmethod
     def is_waiting(status):
         return status in (
             NodeStatus.PENDING,
         )
 
+    @staticmethod
     def is_success(status):
         return status in (
             NodeStatus.SUCCESS,
             NodeStatus.SKIPPED
         )
 
+    @staticmethod
     def is_error(status):
         return status in (
             NodeStatus.ERROR,

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.33.0'
+version = '0.33.2'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_install.py

@@ -42,6 +42,13 @@ def install_tool(tool, script, build_dir, prefix):
     # setup environment
     env = os.environ.copy()
     env["PREFIX"] = prefix
+    env["USE_SUDO_INSTALL"] = "no"
+    try:
+        os.makedirs(prefix, exist_ok=True)
+    except PermissionError:
+        env["USE_SUDO_INSTALL"] = "yes"
+    if not os.access(prefix, os.W_OK):
+        env["USE_SUDO_INSTALL"] = "yes"
 
     # run
     ret = subprocess.call(script, env=env, cwd=build_dir)

siliconcompiler/apps/sc_remote.py

@@ -6,7 +6,7 @@ from siliconcompiler import Chip
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler.remote.client import Client, ConfigureClient
 from siliconcompiler.scheduler import _finalize_run
-from siliconcompiler.utils.flowgraph import nodes_to_execute
+from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def main():
@@ -170,7 +170,12 @@ To delete a job, use:
             return 1
 
         # Wrap up run
-        for step, index in nodes_to_execute(chip):
+        runtime = RuntimeFlowgraph(
+            chip.schema.get("flowgraph", flow, field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+        for step, index in runtime.get_nodes():
             manifest = os.path.join(chip.getworkdir(step=step, index=index),
                                     'outputs',
                                     f'{chip.design}.pkg.json')

siliconcompiler/apps/utils/replay.py

@@ -18,6 +18,7 @@ import siliconcompiler
 from siliconcompiler.apps._common import UNSET_DESIGN
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler import utils
+from siliconcompiler.record import RecordTime
 
 
 def make_bytes(data):
@@ -127,11 +128,10 @@ def main():
     path = os.path.abspath(args['file'])
     os.makedirs(os.path.dirname(path), exist_ok=True)
 
-    starttimes = set()
-    for starttime, step, index in chip.schema.get('history', jobname, 'record', 'starttime',
-                                                  field=None).getvalues():
-        starttimes.add(datetime.strptime(starttime, '%Y-%m-%d %H:%M:%S'))
-    starttime = min(starttimes).strftime('%Y-%m-%d %H:%M:%S')
+    record_schema = chip.schema.get('history', jobname, 'record', field="schema")
+    starttime = datetime.fromtimestamp(
+        record_schema.get_earliest_time(RecordTime.START)).strftime(
+            '%Y-%m-%d %H:%M:%S')
 
     with io.StringIO() as fd:
         fd.write(utils.get_file_template('replay/requirements.txt').render(

siliconcompiler/core.py

@@ -37,10 +37,7 @@ from siliconcompiler.report.dashboard import DashboardType
 from siliconcompiler import package as sc_package
 import glob
 from siliconcompiler.scheduler import run as sc_runner
-from siliconcompiler.utils.flowgraph import nodes_to_execute, \
-    _get_pruned_node_inputs, \
-    _get_flowgraph_execution_order, _check_flowgraph_io, \
-    _get_flowgraph_information
+from siliconcompiler.utils.flowgraph import _check_flowgraph_io, _get_flowgraph_information
 from siliconcompiler.tools._common import get_tool_task
 from types import FunctionType, ModuleType
 from siliconcompiler.flowgraph import RuntimeFlowgraph
@@ -1622,11 +1619,26 @@ class Chip:
             error = True
             self.logger.error(f"flowgraph {flow} not defined.")
 
-        nodes = [node for node in nodes_to_execute(self)
+        runtime = RuntimeFlowgraph(
+            self.schema.get("flowgraph", flow, field='schema'),
+            args=(self.get('arg', 'step'), self.get('arg', 'index')),
+            from_steps=self.get('option', 'from'),
+            to_steps=self.get('option', 'to'),
+            prune_nodes=self.get('option', 'prune'))
+
+        nodes = [node for node in runtime.get_nodes()
                  if self.get('record', 'status', step=node[0], index=node[1])
                  != NodeStatus.SKIPPED]
+        flow_schema = self.schema.get("flowgraph", flow, field="schema")
+        runtime_io = RuntimeFlowgraph(
+            flow_schema,
+            args=(self.get('arg', 'step'), self.get('arg', 'index')),
+            from_steps=set([step for step, _ in flow_schema.get_entry_nodes()]),
+            prune_nodes=self.get('option', 'prune'))
+
         for (step, index) in nodes:
-            for in_step, in_index in _get_pruned_node_inputs(self, flow, (step, index)):
+            for in_step, in_index in runtime_io.get_node_inputs(
+                    step, index, record=self.schema.get("record", field="schema")):
                 if (in_step, in_index) in nodes:
                     # we're gonna run this step, OK
                     continue
@@ -1717,7 +1729,11 @@ class Chip:
                 error = True
                 self.logger.error(f'No executable or run() function specified for {tool}/{task}')
 
-        if not _check_flowgraph_io(self, nodes=nodes):
+        runtime_full = RuntimeFlowgraph(
+            self.schema.get("flowgraph", flow, field='schema'),
+            to_steps=self.get('option', 'to'),
+            prune_nodes=self.get('option', 'prune'))
+        if not error and not _check_flowgraph_io(self, nodes=runtime_full.get_nodes()):
             error = True
 
         return not error
@@ -2748,7 +2764,13 @@ class Chip:
             flow = self.get('option', 'flow')
             flowgraph_nodes = [(step, index) for index in self.getkeys("flowgraph", flow, step)]
         else:
-            flowgraph_nodes = nodes_to_execute(self)
+            flow = self.get('option', 'flow')
+            runtime = RuntimeFlowgraph(
+                self.schema.get("flowgraph", flow, field='schema'),
+                from_steps=self.get('option', 'from'),
+                to_steps=self.get('option', 'to'),
+                prune_nodes=self.get('option', 'prune'))
+            flowgraph_nodes = runtime.get_nodes()
 
         if not archive_name:
             if step and index:
@@ -3231,14 +3253,18 @@ class Chip:
                 search_nodes.append((sc_step, sc_index))
             elif sc_step:
                 if flow is not None:
-                    for check_step, check_index in nodes_to_execute(self, flow):
+                    runtime = RuntimeFlowgraph(
+                        self.schema.get("flowgraph", flow, field='schema'),
+                        from_steps=self.get('option', 'from'),
+                        to_steps=self.get('option', 'to'),
+                        prune_nodes=self.get('option', 'prune'))
+                    for check_step, check_index in runtime.get_nodes():
                         if sc_step == check_step:
                             search_nodes.append((check_step, check_index))
             else:
                 if flow is not None:
-                    for nodes in _get_flowgraph_execution_order(self,
-                                                                flow,
-                                                                reverse=True):
+                    for nodes in self.schema.get(
+                            "flowgraph", flow, field="schema").get_execution_order(reverse=True):
                         search_nodes.extend(nodes)
 
             for ext in self._showtools.keys():

siliconcompiler/data/templates/replay/replay.sh.j2

@@ -9,6 +9,7 @@ CD_WORK="{{ work_dir }}"
 PRINT=""
 CMDPREFIX=""
 SKIPEXPORT=0
+DONODE={{ node_only }}
 while [[ $# -gt 0 ]]; do
     case $1 in
         --which)
@@ -40,6 +41,11 @@ while [[ $# -gt 0 ]]; do
             shift
             shift
             ;;
+        --node)
+            DONODE=1
+            shift
+            shift
+            ;;
         -h|--help)
             echo "Usage: $0"
             echo "  Options:"
@@ -49,7 +55,8 @@ while [[ $# -gt 0 ]]; do
             echo "    --command         print the execution command"
             echo "    --skipcd          do not change directory into replay directory"
             echo "    --skipexports     do not export environmental variables"
-            echo "    --cmdprefix <cmd> prefix to add to the replay command, such as gdb"
+            echo "    --cmdprefix <cmd> prefix to add to the replay command, such as dgb"
+            echo "    --node            execute entire node"
             echo "    -h,--help         print this help"
             exit 0
             ;;
@@ -87,6 +94,16 @@ case $PRINT in
         ;;
 esac
 
+if [ $DONODE == 1 ]; then
+python3 -m siliconcompiler.scheduler.run_node \
+    -cfg "{{ cfg_file }}" \
+    -builddir "${PWD}/../../../../" \
+    -step "{{ step }}" \
+    -index "{{ index }}" \
+    -cwd "$PWD" \
+    -replay
+{% if cmds|length > 0 %}else
 # Command execution
 $CMDPREFIX \{% for cmd in cmds %}
 {% if not loop.first %}  {% endif %}{{ cmd }}{% if not loop.last %} \{% endif %}{% endfor %}
+{% endif %}fi

siliconcompiler/metric.py

@@ -3,6 +3,7 @@ from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
 from siliconcompiler.schema.utils import trim
 
 from siliconcompiler.utils.units import convert
+from siliconcompiler.record import RecordTime
 
 
 class MetricSchema(BaseSchema):
@@ -43,6 +44,83 @@ class MetricSchema(BaseSchema):
 
         return self.set(metric, value, step=step, index=str(index))
 
+    def record_tasktime(self, step, index, record):
+        """
+        Record the task time for this node
+
+        Args:
+            step (str): step to record
+            index (str/int): index to record
+            record (:class:`RecordSchema`): record to lookup data in
+        """
+        start_time, end_time = [
+            record.get_recorded_time(step, index, RecordTime.START),
+            record.get_recorded_time(step, index, RecordTime.END)
+        ]
+
+        if start_time is None or end_time is None:
+            return False
+
+        return self.record(step, index, "tasktime", end_time-start_time, unit="s")
+
+    def record_totaltime(self, step, index, flow, record):
+        """
+        Record the total time for this node
+
+        Args:
+            step (str): step to record
+            index (str/int): index to record
+            flow (:class:`FlowgraphSchema`): flowgraph to lookup nodes in
+            record (:class:`RecordSchema`): record to lookup data in
+        """
+        all_nodes = flow.get_nodes()
+        node_times = [
+            (record.get_recorded_time(*node, RecordTime.START),
+             record.get_recorded_time(*node, RecordTime.END)) for node in all_nodes
+        ]
+
+        # Remove incomplete records
+        node_times = [times for times in node_times if times[0] is not None]
+
+        if len(node_times) == 0:
+            return False
+
+        node_end = record.get_recorded_time(step, index, RecordTime.END)
+        if node_end is None:
+            return False
+
+        node_times = sorted(node_times)
+        if len(node_times) > 1:
+            new_times = []
+            for n in range(len(node_times)):
+                if not new_times:
+                    new_times.append(node_times[n])
+                    continue
+                prev_start_time, prev_end_time = new_times[-1]
+                start_time, end_time = node_times[n]
+
+                new_start = min(prev_start_time, start_time)
+
+                if prev_end_time is None:
+                    new_times[-1] = (new_start, end_time)
+                elif prev_end_time >= start_time:
+                    if end_time is not None:
+                        new_end = max(prev_end_time, end_time)
+                    else:
+                        new_end = prev_end_time
+                    new_times[-1] = (new_start, new_end)
+                else:
+                    new_times.append(node_times[n])
+            node_times = new_times
+
+        total_time = 0
+        for start_time, end_time in node_times:
+            if start_time > node_end:
+                continue
+            total_time += min(end_time, node_end) - start_time
+
+        return self.record(step, index, "totaltime", total_time, unit="s")
+
 
 ###########################################################################
 # Metrics to Track

siliconcompiler/package/git.py

@@ -81,4 +81,4 @@ def clone_from_git(chip, package, path, ref, url, data_path):
     chip.logger.info(f'Checking out {ref}')
     repo.git.checkout(ref)
     for submodule in repo.submodules:
-        submodule.update(init=True)
+        submodule.update(recursive=True, init=True, force=True)

siliconcompiler/record.py

@@ -30,13 +30,25 @@ class RecordTool(Enum):
 
 
 class RecordSchema(BaseSchema):
-    __TIMEFORMAT = "%Y-%m-%d %H:%M:%S"
+    __TIMEFORMAT = "%Y-%m-%d %H:%M:%S.%f"
 
     def __init__(self):
         super().__init__()
 
         schema_record(self)
 
+    def _from_dict(self, manifest, keypath, version=None):
+        ret = super()._from_dict(manifest, keypath, version)
+
+        # Correct for change specification
+        if version and version < (0, 50, 4):
+            for timekey in RecordTime:
+                start_param = self.get(timekey.value, field=None)
+                for value, step, index in start_param.getvalues():
+                    start_param.set(f"{value}.000000", step=step, index=index)
+
+        return ret
+
     def clear(self, step, index, keep=None):
         '''
         Clear all saved metrics for a given step and index
@@ -259,7 +271,7 @@ class RecordSchema(BaseSchema):
 
     def get_recorded_time(self, step, index, type):
         '''
-        Returns the time recorded for a given record.
+        Returns the time recorded for a given record, or None if nothing is recorded.
 
         Args:
             step (str): Step name to associate.
@@ -267,10 +279,52 @@ class RecordSchema(BaseSchema):
             type (:class:`RecordTime`): type of time to record
         '''
         type = RecordTime(type)
+        record_time = self.get(type.value, step=step, index=index)
+        if record_time is None:
+            return None
+
         return datetime.strptime(
-            self.get(type.value, step=step, index=index)+"+0000",
+            record_time+"+0000",
             RecordSchema.__TIMEFORMAT+"%z").timestamp()
 
+    def get_earliest_time(self, type):
+        '''
+        Returns the earliest recorded time.
+
+        Args:
+            type (:class:`RecordTime`): type of time to record
+        '''
+        type = RecordTime(type)
+        record_param = self.get(type.value, field=None)
+
+        times = set()
+        for _, step, index in record_param.getvalues():
+            times.add(self.get_recorded_time(step, index, type))
+
+        if not times:
+            return None
+
+        return min(times)
+
+    def get_latest_time(self, type):
+        '''
+        Returns the last recorded time.
+
+        Args:
+            type (:class:`RecordTime`): type of time to record
+        '''
+        type = RecordTime(type)
+        record_param = self.get(type.value, field=None)
+
+        times = set()
+        for _, step, index in record_param.getvalues():
+            times.add(self.get_recorded_time(step, index, type))
+
+        if not times:
+            return None
+
+        return max(times)
+
     def record_tool(self, step, index, info, type):
         '''
         Record information about the tool used during this record.
@@ -316,11 +370,13 @@ def schema_record(schema):
                         'x86_64',
                         '(x86_64, rv64imafdc)'],
                'starttime': ['start time',
-                             '\"2021-09-06 12:20:20\"',
-                             'Time is reported in the ISO 8601 format YYYY-MM-DD HR:MIN:SEC'],
+                             '\"2021-09-06 12:20:20.000000\"',
+                             'Time is recorded with the format YYYY-MM-DD HR:MIN:SEC.MICROSEC for '
+                             'UTC'],
                'endtime': ['end time',
-                           '\"2021-09-06 12:20:20\"',
-                           'Time is reported in the ISO 8601 format YYYY-MM-DD HR:MIN:SEC'],
+                           '\"2021-09-06 12:20:20.000000\"',
+                           'Time is recorded with the format YYYY-MM-DD HR:MIN:SEC.MICROSEC for '
+                           'UTC'],
                'region': ['cloud region',
                           '\"US Gov Boston\"',
                           """Recommended naming methodology:

siliconcompiler/remote/client.py

@@ -10,12 +10,15 @@ import tarfile
 import tempfile
 import multiprocessing
 
-from siliconcompiler import utils, SiliconCompilerError, NodeStatus
+import os.path
+
+from siliconcompiler import utils, SiliconCompilerError
 from siliconcompiler import NodeStatus as SCNodeStatus
 from siliconcompiler._metadata import default_server
-from siliconcompiler.utils.flowgraph import nodes_to_execute
-from siliconcompiler.remote import JobStatus
+from siliconcompiler.remote import JobStatus, NodeStatus
 from siliconcompiler.report.dashboard import DashboardType
+from siliconcompiler.flowgraph import RuntimeFlowgraph
+from siliconcompiler.schema import JournalingSchema
 
 # Step name to use while logging
 remote_step_name = 'remote'
@@ -309,19 +312,29 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         try:
             # Decode response JSON, if possible.
             job_info = json.loads(info['message'])
-        except json.JSONDecodeError as e:
-            self.__logger.warning(f"Job is still running: {e}")
+            if "null" in job_info:
+                job_info[None] = job_info["null"]
+                del job_info["null"]
+        except json.JSONDecodeError:
+            self.__logger.warning(f"Job is still running: {info['message']}")
             return completed, starttimes, True
 
         nodes_to_log = {}
         for node, node_info in job_info.items():
             status = node_info['status']
-            nodes_to_log.setdefault(status, []).append((node, node_info))
+
+            if status == NodeStatus.UPLOADED:
+                status = SCNodeStatus.PENDING
 
             if SCNodeStatus.is_done(status):
                 # collect completed
                 completed.append(node)
 
+            if not node:
+                continue
+
+            nodes_to_log.setdefault(status, []).append((node, node_info))
+
             if self.__node_information and node in self.__node_information:
                 self.__chip.set('record', 'status', status,
                                 step=self.__node_information[node]["step"],
@@ -579,6 +592,22 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             raise SiliconCompilerError('Job canceled by user keyboard interrupt')
 
     def __import_run_manifests(self, starttimes):
+        if not self.__setup_information_loaded:
+            if self.__setup_information_fetched:
+                manifest = os.path.join(self.__chip.getworkdir(), f'{self.__chip.design}.pkg.json')
+                if os.path.exists(manifest):
+                    try:
+                        JournalingSchema(self.__chip.schema).read_journal(manifest)
+                        self.__setup_information_loaded = True
+                        changed = True
+                    except:  # noqa E722
+                        # Import may fail if file is still getting written
+                        pass
+
+        if not self.__setup_information_loaded:
+            # Dont do anything until this has been loaded
+            return
+
         changed = False
         for _, node_info in self.__node_information.items():
             if node_info["imported"]:
@@ -590,7 +619,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 f'{self.__chip.design}.pkg.json')
             if os.path.exists(manifest):
                 try:
-                    self.__chip.schema.read_journal(manifest)
+                    JournalingSchema(self.__chip.schema).read_journal(manifest)
                     node_info["imported"] = True
                     changed = True
                 except:  # noqa E722
@@ -598,7 +627,7 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                     pass
             elif self.__chip.get('record', 'status',
                                  step=node_info["step"], index=node_info["index"]) \
-                    == NodeStatus.SKIPPED:
+                    == SCNodeStatus.SKIPPED:
                 node_info["imported"] = True
                 changed = True
 
@@ -617,8 +646,17 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
             check_info = self.__check()
             self.__check_interval = check_info['progress_interval']
 
+        self.__setup_information_fetched = False
+        self.__setup_information_loaded = False
+
         self.__node_information = {}
-        for step, index in nodes_to_execute(self.__chip):
+        runtime = RuntimeFlowgraph(
+            self.__chip.schema.get("flowgraph", self.__chip.get('option', 'flow'), field='schema'),
+            from_steps=self.__chip.get('option', 'from'),
+            to_steps=self.__chip.get('option', 'to'),
+            prune_nodes=self.__chip.get('option', 'prune'))
+
+        for step, index in runtime.get_nodes():
             done = SCNodeStatus.is_done(self.__chip.get('record', 'status', step=step, index=index))
             node_info = {
                 "step": step,
@@ -659,6 +697,11 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
                 # Update dashboard if active
                 self.__chip._dash.update_manifest({"starttimes": starttimes})
 
+            if None in completed:
+                completed.remove(None)
+                if not self.__setup_information_fetched:
+                    self.__schedule_fetch_result(None)
+
             nodes_to_fetch = []
             for node in completed:
                 if not self.__node_information[node]["fetched"]:
@@ -674,7 +717,6 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         for node, node_info in self.__node_information.items():
             if not node_info["fetched"]:
                 self.__schedule_fetch_result(node)
-        self.__schedule_fetch_result(node)
 
         self._finalize_loop()
 
@@ -693,11 +735,12 @@ service, provided by SiliconCompiler, is not intended to process proprietary IP.
         self.__import_run_manifests({})
 
     def __schedule_fetch_result(self, node):
-        self.__node_information[node]["fetched"] = True
+        if node:
+            self.__node_information[node]["fetched"] = True
+            self.__logger.info(f'    {node}')
+        else:
+            self.__setup_information_fetched = True
         self.__download_pool.apply_async(Client._fetch_result, (self, node))
-        if node is None:
-            node = 'final result'
-        self.__logger.info(f'    {node}')
 
     def _fetch_result(self, node):
         '''