siliconcompiler 0.33.1__py3-none-any.whl → 0.34.0__py3-none-any.whl

siliconcompiler/__init__.py

@@ -3,6 +3,7 @@ from siliconcompiler._common import NodeStatus, SiliconCompilerError
 from siliconcompiler.utils import sc_open
 from siliconcompiler.schema_obj import SchemaTmp as Schema
 
+from siliconcompiler.design import DesignSchema
 from siliconcompiler.record import RecordSchema
 from siliconcompiler.metric import MetricSchema
 from siliconcompiler.pdk import PDKSchema
@@ -30,6 +31,7 @@ __all__ = [
     "Checklist",
     "Schema",
     "sc_open",
+    "DesignSchema",
     "RecordSchema",
     "MetricSchema",
     "PDKSchema",

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.33.1'
+version = '0.34.0'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_issue.py

@@ -4,7 +4,7 @@ import os
 import siliconcompiler
 import tarfile
 import json
-from siliconcompiler.scheduler import _runtask, _executenode
+from siliconcompiler.scheduler.schedulernode import SchedulerNode
 from siliconcompiler.utils.issue import generate_testcase
 from siliconcompiler.tools._common import get_tool_task
 
@@ -153,8 +153,10 @@ To run a testcase, use:
         # Run task
         # Rerun setup task, assumed to be running in its own thread so
         # multiprocess is not needed
-        flow = chip.get('option', 'flow')
-        _runtask(chip, flow, step, index, _executenode, replay=True)
+        SchedulerNode(chip,
+                      step,
+                      index,
+                      replay=True).run()
 
         return 0
 

siliconcompiler/apps/sc_remote.py

@@ -1,12 +1,9 @@
 # Copyright 2023 Silicon Compiler Authors. All Rights Reserved.
-import os
 import sys
 
 from siliconcompiler import Chip
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler.remote.client import Client, ConfigureClient
-from siliconcompiler.scheduler import _finalize_run
-from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def main():
@@ -169,20 +166,6 @@ To delete a job, use:
             chip.logger.error(f'{e}')
             return 1
 
-        # Wrap up run
-        runtime = RuntimeFlowgraph(
-            chip.schema.get("flowgraph", flow, field='schema'),
-            from_steps=chip.get('option', 'from'),
-            to_steps=chip.get('option', 'to'),
-            prune_nodes=chip.get('option', 'prune'))
-        for step, index in runtime.get_nodes():
-            manifest = os.path.join(chip.getworkdir(step=step, index=index),
-                                    'outputs',
-                                    f'{chip.design}.pkg.json')
-            if os.path.exists(manifest):
-                chip.schema.read_journal(manifest)
-        _finalize_run(chip)
-
         # Summarize the run.
         chip.summary()
 

siliconcompiler/apps/utils/replay.py

@@ -18,6 +18,7 @@ import siliconcompiler
 from siliconcompiler.apps._common import UNSET_DESIGN
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler import utils
+from siliconcompiler.record import RecordTime
 
 
 def make_bytes(data):
@@ -127,11 +128,10 @@ def main():
     path = os.path.abspath(args['file'])
     os.makedirs(os.path.dirname(path), exist_ok=True)
 
-    starttimes = set()
-    for starttime, step, index in chip.schema.get('history', jobname, 'record', 'starttime',
-                                                  field=None).getvalues():
-        starttimes.add(datetime.strptime(starttime, '%Y-%m-%d %H:%M:%S'))
-    starttime = min(starttimes).strftime('%Y-%m-%d %H:%M:%S')
+    record_schema = chip.schema.get('history', jobname, 'record', field="schema")
+    starttime = datetime.fromtimestamp(
+        record_schema.get_earliest_time(RecordTime.START)).strftime(
+            '%Y-%m-%d %H:%M:%S')
 
     with io.StringIO() as fd:
         fd.write(utils.get_file_template('replay/requirements.txt').render(

siliconcompiler/checklist.py

@@ -4,7 +4,7 @@ from siliconcompiler.schema.utils import trim
 
 
 class ChecklistSchema(NamedSchema):
-    def __init__(self, name=None):
+    def __init__(self, name):
         super().__init__(name=name)
 
         schema_checklist(self)

siliconcompiler/core.py

@@ -18,9 +18,9 @@ import csv
 import yaml
 from inspect import getfullargspec
 from siliconcompiler import Schema
-from siliconcompiler.schema import SCHEMA_VERSION, PerNode, JournalingSchema, EditableSchema
+from siliconcompiler.schema import SCHEMA_VERSION, PerNode, Journal, EditableSchema
 from siliconcompiler.schema.parametertype import NodeType
-from siliconcompiler.schema.parametervalue import FileNodeValue, PathNodeValue
+from siliconcompiler.schema.parametervalue import FileNodeValue
 from siliconcompiler.schema import utils as schema_utils
 from siliconcompiler import utils
 from siliconcompiler.utils.logging import SCColorLoggerFormatter, \
@@ -34,9 +34,8 @@ from siliconcompiler.report import _generate_summary_image, _open_summary_image
 from siliconcompiler.report.dashboard.web import WebDashboard
 from siliconcompiler.report.dashboard.cli import CliDashboard
 from siliconcompiler.report.dashboard import DashboardType
-from siliconcompiler import package as sc_package
 import glob
-from siliconcompiler.scheduler import run as sc_runner
+from siliconcompiler.scheduler.scheduler import Scheduler
 from siliconcompiler.utils.flowgraph import _check_flowgraph_io, _get_flowgraph_information
 from siliconcompiler.tools._common import get_tool_task
 from types import FunctionType, ModuleType
@@ -1102,7 +1101,8 @@ class Chip:
         '''
 
         if package:
-            filename = os.path.join(sc_package.path(self, package), filename)
+            resolvers = self.get("package", field="schema").get_resolvers()
+            filename = os.path.join(resolvers[package](), filename)
 
         if not os.path.isfile(filename):
             raise FileNotFoundError(filename)
@@ -1110,10 +1110,16 @@ class Chip:
         package_name = f'flist-{os.path.basename(filename)}'
         package_dir = os.path.dirname(os.path.abspath(filename))
 
-        env_vars = utils.get_env_vars(self, None, None)
-
         def __make_path(rel, path):
-            path = PathNodeValue.resolve_env_vars(path, envvars=env_vars)
+            env_save = os.environ.copy()
+            schema_env = {}
+            for env in self.getkeys('option', 'env'):
+                schema_env[env] = self.get('option', 'env', env)
+            os.environ.update(schema_env)
+            path = os.path.expandvars(path)
+            path = os.path.expanduser(path)
+            os.environ.clear()
+            os.environ.update(env_save)
             if os.path.isabs(path):
                 if path.startswith(rel):
                     return os.path.relpath(path, rel), package_name
@@ -1380,18 +1386,17 @@ class Chip:
         else:
             search_paths = [self.cwd]
 
-        env_vars = utils.get_env_vars(self, step, index)
+        resolvers = self.get("package", field="schema").get_resolvers()
         for (dependency, path) in zip(dependencies, paths):
             faux_param = FileNodeValue()
             faux_param.set(path)
             try:
                 if dependency:
                     faux_param.set(dependency, field='package')
-                    faux_search = [os.path.abspath(os.path.join(sc_package.path(self, dependency)))]
+                    faux_search = [resolvers[dependency]()]
                 else:
                     faux_search = search_paths
                 resolved = faux_param.resolve_path(
-                    envvars=env_vars,
                     search=faux_search,
                     collection_dir=collection_dir)
             except FileNotFoundError:
@@ -1547,39 +1552,19 @@ class Chip:
             True if all file paths are valid, otherwise False.
         '''
 
-        allkeys = self.allkeys()
-        error = False
-        for keypath in allkeys:
-            paramtype = self.get(*keypath, field='type')
-            is_file = 'file' in paramtype
-            is_dir = 'dir' in paramtype
-            is_list = paramtype.startswith('[')
-
-            if is_file or is_dir:
-                if keypath[-2:] == ('option', 'builddir'):
-                    # Skip ['option', 'builddir'] since it will get created by run() if it doesn't
-                    # exist
-                    continue
-
-                for check_files, step, index in self.schema.get(*keypath, field=None).getvalues():
-                    if not check_files:
-                        continue
+        ignore_keys = []
+        for keypath in self.allkeys():
+            if keypath[-2:] == ('option', 'builddir'):
+                ignore_keys.append(keypath)
 
-                    if not is_list:
-                        check_files = [check_files]
-
-                    for idx, check_file in enumerate(check_files):
-                        found_file = self.__find_files(*keypath,
-                                                       missing_ok=True,
-                                                       step=step, index=index,
-                                                       list_index=idx)
-                        if is_list:
-                            found_file = found_file[0]
-                        if not found_file:
-                            self.logger.error(f"Parameter {keypath} path {check_file} is invalid")
-                            error = True
+        package_map = self.get("package", field="schema").get_resolvers()
 
-        return not error
+        return self.schema.check_filepaths(
+            ignore_keys=ignore_keys,
+            logger=self.logger,
+            packages=package_map,
+            collection_dir=self._getcollectdir(),
+            cwd=self.cwd)
 
     ###########################################################################
     def check_manifest(self):
@@ -1729,7 +1714,11 @@ class Chip:
                 error = True
                 self.logger.error(f'No executable or run() function specified for {tool}/{task}')
 
-        if not error and not _check_flowgraph_io(self, nodes=nodes):
+        runtime_full = RuntimeFlowgraph(
+            self.schema.get("flowgraph", flow, field='schema'),
+            to_steps=self.get('option', 'to'),
+            prune_nodes=self.get('option', 'prune'))
+        if not error and not _check_flowgraph_io(self, nodes=runtime_full.get_nodes()):
             error = True
 
         return not error
@@ -1792,10 +1781,7 @@ class Chip:
             schema.write_manifest(filepath)
             return
 
-        tcl_record = False
-        if isinstance(schema, JournalingSchema):
-            tcl_record = "get" in schema.get_journaling_types()
-            schema = schema.get_base_schema()
+        tcl_record = "get" in Journal.access(schema).get_types()
 
         is_csv = re.search(r'(\.csv)(\.gz)*$', filepath)
 
@@ -3197,9 +3183,14 @@ class Chip:
             >>> run()
             Runs the execution flow defined by the flowgraph dictionary.
         '''
+        from siliconcompiler.remote.client import ClientScheduler
 
         try:
-            sc_runner(self)
+            if self.get('option', 'remote'):
+                scheduler = ClientScheduler(self)
+            else:
+                scheduler = Scheduler(self)
+            scheduler.run()
         except Exception as e:
             if raise_exception:
                 raise e

siliconcompiler/data/templates/replay/replay.sh.j2

@@ -9,6 +9,7 @@ CD_WORK="{{ work_dir }}"
 PRINT=""
 CMDPREFIX=""
 SKIPEXPORT=0
+DONODE={{ node_only }}
 while [[ $# -gt 0 ]]; do
     case $1 in
         --which)
@@ -40,6 +41,11 @@ while [[ $# -gt 0 ]]; do
             shift
             shift
             ;;
+        --node)
+            DONODE=1
+            shift
+            shift
+            ;;
         -h|--help)
             echo "Usage: $0"
             echo "  Options:"
@@ -49,7 +55,8 @@ while [[ $# -gt 0 ]]; do
             echo "    --command         print the execution command"
             echo "    --skipcd          do not change directory into replay directory"
             echo "    --skipexports     do not export environmental variables"
-            echo "    --cmdprefix <cmd> prefix to add to the replay command, such as gdb"
+            echo "    --cmdprefix <cmd> prefix to add to the replay command, such as dgb"
+            echo "    --node            execute entire node"
             echo "    -h,--help         print this help"
             exit 0
             ;;
@@ -87,6 +94,16 @@ case $PRINT in
         ;;
 esac
 
+if [ $DONODE == 1 ]; then
+python3 -m siliconcompiler.scheduler.run_node \
+    -cfg "{{ cfg_file }}" \
+    -builddir "${PWD}/../../../../" \
+    -step "{{ step }}" \
+    -index "{{ index }}" \
+    -cwd "$PWD" \
+    -replay
+{% if cmds|length > 0 %}else
 # Command execution
 $CMDPREFIX \{% for cmd in cmds %}
 {% if not loop.first %}  {% endif %}{{ cmd }}{% if not loop.last %} \{% endif %}{% endfor %}
+{% endif %}fi