siliconcompiler 0.34.0__py3-none-any.whl → 0.34.2__py3-none-any.whl

siliconcompiler/scheduler/taskscheduler.py

@@ -11,6 +11,7 @@ from siliconcompiler import SiliconCompilerError
 from siliconcompiler import utils
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
+from siliconcompiler.package import Resolver
 from siliconcompiler.schema import Journal
 
 from siliconcompiler.utils.logging import SCBlankLoggerFormatter
@@ -33,6 +34,7 @@ class TaskScheduler:
     def __init__(self, chip, tasks):
         self.__chip = chip
         self.__logger = self.__chip.logger
+        self.__logger_console_handler = self.__chip._logger_console
         self.__schema = self.__chip.schema
         self.__flow = self.__schema.get("flowgraph", self.__chip.get('option', 'flow'),
                                         field="schema")
@@ -73,13 +75,14 @@ class TaskScheduler:
             if self.__record.get('status', step=step, index=index) != NodeStatus.PENDING:
                 continue
 
-            threads = tasks[(step, index)].threads
+            with tasks[(step, index)].runtime():
+                threads = tasks[(step, index)].threads
             if not threads:
                 threads = self.__max_threads
             threads = max(1, min(threads, self.__max_threads))
 
             task = {
-                "name": f"{step}{index}",
+                "name": f"{step}/{index}",
                 "inputs": runtime.get_node_inputs(step, index, record=self.__record),
                 "proc": None,
                 "parent_pipe": None,
@@ -93,7 +96,6 @@ class TaskScheduler:
 
             task["parent_pipe"], pipe = multiprocessing.Pipe()
             task["node"].set_queue(pipe, self.__log_queue)
-            task["node"].init_state()  # reinit access to remove holdover access
 
             task["proc"] = multiprocessing.Process(target=task["node"].run)
             init_funcs.add(task["node"].init)
@@ -108,9 +110,9 @@ class TaskScheduler:
         multiprocessing.freeze_support()
 
         # Handle logs across threads
-        log_listener = QueueListener(self.__log_queue, self.__logger._console)
-        console_format = self.__logger._console.formatter
-        self.__logger._console.setFormatter(SCBlankLoggerFormatter())
+        log_listener = QueueListener(self.__log_queue, self.__logger_console_handler)
+        console_format = self.__logger_console_handler.formatter
+        self.__logger_console_handler.setFormatter(SCBlankLoggerFormatter())
         log_listener.start()
 
         # Update dashboard before run begins
@@ -130,7 +132,7 @@ class TaskScheduler:
 
         # Cleanup logger
         log_listener.stop()
-        self.__logger._console.setFormatter(console_format)
+        self.__logger_console_handler.setFormatter(console_format)
 
     def __run_loop(self):
         self.__startTimes = {None: time.time()}
@@ -199,7 +201,7 @@ class TaskScheduler:
                         packages = info["parent_pipe"].recv()
                         if isinstance(packages, dict):
                             for package, path in packages.items():
-                                self.__chip.get("package", field="schema")._set_cache(package, path)
+                                Resolver.set_cache(self.__chip, package, path)
                     except:  # noqa E722
                         pass
 

siliconcompiler/schema/__init__.py

@@ -5,7 +5,6 @@ from .editableschema import EditableSchema
 from .baseschema import BaseSchema
 from .cmdlineschema import CommandLineSchema
 from .namedschema import NamedSchema
-from .packageschema import PackageSchema
 
 from .schema_cfg import SCHEMA_VERSION
 
@@ -16,7 +15,6 @@ __all__ = [
     "EditableSchema",
     "CommandLineSchema",
     "NamedSchema",
-    "PackageSchema",
     "Parameter",
     "Scope",
     "PerNode",

siliconcompiler/schema/baseschema.py

@@ -4,6 +4,7 @@
 # SC dependencies outside of its directory, since it may be used by tool drivers
 # that have isolated Python environments.
 
+import contextlib
 import copy
 
 try:
@@ -21,7 +22,7 @@ except ModuleNotFoundError:
 
 import os.path
 
-from .parameter import Parameter
+from .parameter import Parameter, NodeValue
 from .journal import Journal
 
 
@@ -32,11 +33,21 @@ class BaseSchema:
     '''
 
     def __init__(self):
-        # Data storage for the schema
         self.__manifest = {}
         self.__default = None
         self.__journal = Journal()
         self.__parent = self
+        self.__active = None
+        self.__key = None
+
+    @property
+    def _keypath(self):
+        '''
+        Returns the key to the current section of the schema
+        '''
+        if self.__parent is self:
+            return tuple()
+        return tuple([*self.__parent._keypath, self.__key])
 
     def _from_dict(self, manifest, keypath, version=None):
         '''
@@ -106,6 +117,9 @@ class BaseSchema:
             return gzip.open(filepath, mode="rt" if is_read else "wt", encoding="utf-8")
         return open(filepath, mode="r" if is_read else "w", encoding="utf-8")
 
+    def __format_key(self, *key):
+        return f"[{','.join([*self._keypath, *key])}]"
+
     def read_manifest(self, filepath):
         """
         Reads a manifest from disk and replaces the current data with the data in the file.
@@ -217,21 +231,23 @@ class BaseSchema:
 
         try:
             require_leaf = True
+            insert_defaults = False
             if field == 'schema':
                 require_leaf = False
+                insert_defaults = True
             param = self.__search(
                 *keypath,
-                insert_defaults=False,
+                insert_defaults=insert_defaults,
                 use_default=True,
                 require_leaf=require_leaf)
             if field == 'schema':
                 if isinstance(param, Parameter):
-                    raise ValueError(f"[{','.join(keypath)}] is a complete keypath")
+                    raise ValueError(f"{self.__format_key(*keypath)} is a complete keypath")
                 self.__journal.record("get", keypath, field=field, step=step, index=index)
                 param.__journal = self.__journal.get_child(*keypath)
                 return param
         except KeyError:
-            raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
+            raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
         if field is None:
             return param
 
@@ -240,7 +256,7 @@ class BaseSchema:
             self.__journal.record("get", keypath, field=field, step=step, index=index)
             return get_ret
         except Exception as e:
-            new_msg = f"error while accessing [{','.join(keypath)}]: {e.args[0]}"
+            new_msg = f"error while accessing {self.__format_key(*keypath)}: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
             raise e
 
@@ -274,7 +290,7 @@ class BaseSchema:
         try:
             param = self.__search(*keypath, insert_defaults=True)
         except KeyError:
-            raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
+            raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
 
         try:
             set_ret = param.set(value, field=field, clobber=clobber,
@@ -282,9 +298,10 @@ class BaseSchema:
             if set_ret:
                 self.__journal.record("set", keypath, value=value, field=field,
                                       step=step, index=index)
+                self.__process_active(param, set_ret)
             return set_ret
         except Exception as e:
-            new_msg = f"error while setting [{','.join(keypath)}]: {e.args[0]}"
+            new_msg = f"error while setting {self.__format_key(*keypath)}: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
             raise e
 
@@ -317,16 +334,17 @@ class BaseSchema:
         try:
             param = self.__search(*keypath, insert_defaults=True)
         except KeyError:
-            raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
+            raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
 
         try:
             add_ret = param.add(value, field=field, step=step, index=index)
             if add_ret:
                 self.__journal.record("add", keypath, value=value, field=field,
                                       step=step, index=index)
+                self.__process_active(param, add_ret)
             return add_ret
         except Exception as e:
-            new_msg = f"error while adding to [{','.join(keypath)}]: {e.args[0]}"
+            new_msg = f"error while adding to {self.__format_key(*keypath)}: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
             raise e
 
@@ -359,13 +377,13 @@ class BaseSchema:
         try:
             param = self.__search(*keypath, use_default=True)
         except KeyError:
-            raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
+            raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
 
         try:
             param.unset(step=step, index=index)
             self.__journal.record("unset", keypath, step=step, index=index)
         except Exception as e:
-            new_msg = f"error while unsetting [{','.join(keypath)}]: {e.args[0]}"
+            new_msg = f"error while unsetting {self.__format_key(*keypath)}: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
             raise e
 
@@ -385,7 +403,7 @@ class BaseSchema:
         try:
             key_param = self.__search(*search_path, require_leaf=False)
         except KeyError:
-            raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
+            raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
 
         if removal_key not in key_param.__manifest:
             return
@@ -455,13 +473,13 @@ class BaseSchema:
             try:
                 key_param = self.__search(*keypath, require_leaf=False)
             except KeyError:
-                raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
+                raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
             if isinstance(key_param, Parameter):
                 return tuple()
         else:
             key_param = self
 
-        return tuple(key_param.__manifest.keys())
+        return tuple(sorted(key_param.__manifest.keys()))
 
     def allkeys(self, *keypath, include_default=True):
         '''
@@ -556,8 +574,33 @@ class BaseSchema:
         else:
             schema_copy.__parent = schema_copy
 
+        if key:
+            schema_copy.__key = key[-1]
+
         return schema_copy
 
+    def _find_files_search_paths(self, keypath, step, index):
+        """
+        Returns a list of paths to search during find files.
+
+        Args:
+            keypath (str): final component of keypath
+            step (str): Step name.
+            index (str): Index name.
+        """
+        return []
+
+    def _find_files_dataroot_resolvers(self):
+        """
+        Returns a dictionary of path resolevrs data directory handling for find_files
+
+        Returns:
+            dictionary of str to resolver mapping
+        """
+        if self.__parent is self:
+            return {}
+        return self.__parent._find_files_dataroot_resolvers()
+
     def find_files(self, *keypath, missing_ok=False, step=None, index=None,
                    packages=None, collection_dir=None, cwd=None):
         """
@@ -593,10 +636,13 @@ class BaseSchema:
             the schema.
         """
 
-        param = self.get(*keypath, field=None)
+        base_schema = self.get(*keypath[0:-1], field="schema")
+
+        param = base_schema.get(keypath[-1], field=None)
         paramtype = param.get(field='type')
         if 'file' not in paramtype and 'dir' not in paramtype:
-            raise TypeError(f'Cannot find files on [{",".join(keypath)}], must be a path type')
+            raise TypeError(
+                f'Cannot find files on {self.__format_key(*keypath)}, must be a path type')
 
         paths = param.get(field=None, step=step, index=index)
 
@@ -616,23 +662,26 @@ class BaseSchema:
             cwd = os.getcwd()
 
         if packages is None:
-            packages = {}
+            packages = base_schema._find_files_dataroot_resolvers()
 
         resolved_paths = []
+        root_search_paths = base_schema._find_files_search_paths(keypath[-1], step, index)
         for path in paths:
-            search_paths = []
+            search_paths = root_search_paths.copy()
 
             package = path.get(field="package")
             if package:
                 if package not in packages:
-                    raise ValueError(f"Resolver for {package} not provided")
+                    raise ValueError(f"Resolver for {package} not provided: "
+                                     f"{self.__format_key(*keypath)}")
                 package_path = packages[package]
                 if isinstance(package_path, str):
                     search_paths.append(os.path.abspath(package_path))
                 elif callable(package_path):
                     search_paths.append(package_path())
                 else:
-                    raise TypeError(f"Resolver for {package} is not a recognized type")
+                    raise TypeError(f"Resolver for {package} is not a recognized type: "
+                                    f"{self.__format_key(*keypath)}")
             else:
                 if cwd:
                     search_paths.append(os.path.abspath(cwd))
@@ -645,10 +694,11 @@ class BaseSchema:
                 if not missing_ok:
                     if package:
                         raise FileNotFoundError(
-                            f'Could not find "{path.get()}" in {package} [{",".join(keypath)}]')
+                            f'Could not find "{path.get()}" in {package} '
+                            f'{self.__format_key(*keypath)}')
                     else:
                         raise FileNotFoundError(
-                            f'Could not find "{path.get()}" [{",".join(keypath)}]')
+                            f'Could not find "{path.get()}" {self.__format_key(*keypath)}')
             resolved_paths.append(resolved_path)
 
         if not is_list:
@@ -716,10 +766,10 @@ class BaseSchema:
                                 if index is None:
                                     node_indicator = f" ({step})"
                                 else:
-                                    node_indicator = f" ({step}{index})"
+                                    node_indicator = f" ({step}/{index})"
 
-                            logger.error(f"Parameter [{','.join(keypath)}]{node_indicator} path "
-                                         f"{check_file} is invalid")
+                            logger.error(f"Parameter {self.__format_key(*keypath)}{node_indicator} "
+                                         f"path {check_file} is invalid")
 
         return not error
 
@@ -736,3 +786,75 @@ class BaseSchema:
         if self.__parent is self:
             return self
         return self.__parent._parent(root=root)
+
+    @contextlib.contextmanager
+    def _active(self, **kwargs):
+        '''
+        Use this context to temporarily set additional fields in :meth:`.set` and :meth:`.add`.
+        Additional fields can be specified which can be accessed by :meth:`._get_active`.
+
+        Args:
+            kwargs (dict of str): keyword arguments that are used for setting values
+
+        Example:
+            >>> with schema._active(package="lambdalib"):
+            ...     schema.set("file", "top.v")
+            Sets the file to top.v and associates lambdalib as the package.
+        '''
+        if self.__active:
+            orig_active = self.__active.copy()
+        else:
+            orig_active = None
+
+        if self.__active is None:
+            self.__active = {}
+
+        self.__active.update(kwargs)
+        try:
+            yield
+        finally:
+            self.__active = orig_active
+
+    def _get_active(self, field, defvalue=None):
+        '''
+        Get the value of a specific field.
+
+        Args:
+            field (str): if None, return the current active dictionary,
+                         otherwise the value, if the field is not present, defvalue is returned.
+            defvalue (any): value to return if the field is not present.
+        '''
+        if self.__active is None:
+            return defvalue
+
+        if field is None:
+            return self.__active.copy()
+
+        return self.__active.get(field, defvalue)
+
+    def __process_active(self, param, nodevalues):
+        if not self.__active:
+            return
+
+        if not isinstance(nodevalues, (list, set, tuple)):
+            # Make everything a list
+            nodevalues = [nodevalues]
+
+        if not all([isinstance(v, NodeValue) for v in nodevalues]):
+            nodevalues = []
+            nodevalues_fields = []
+        else:
+            nodevalues_fields = nodevalues[0].fields
+
+        key_fields = ("copy", "lock")
+
+        for field, value in self.__active.items():
+            if field in key_fields:
+                param.set(value, field=field)
+                continue
+
+            if field not in nodevalues_fields:
+                continue
+
+            for param in nodevalues:
+                param.set(value, field=field)

siliconcompiler/schema/editableschema.py

@@ -7,6 +7,8 @@
 from .parameter import Parameter
 from .baseschema import BaseSchema
 
+from typing import Union, Tuple
+
 
 class EditableSchema:
     '''
@@ -17,11 +19,15 @@ class EditableSchema:
         schema (:class:`BaseSchema`): schema to modify
     '''
 
-    def __init__(self, schema):
+    def __init__(self, schema: BaseSchema):
         # Grab manifest from base class
         self.__schema = schema
 
-    def __insert(self, keypath, value, fullkey, clobber):
+    def __insert(self,
+                 keypath: Tuple[str],
+                 value: Union[BaseSchema, Parameter],
+                 fullkey: Tuple[str],
+                 clobber: bool) -> None:
         key = keypath[0]
         keypath = keypath[1:]
 
@@ -31,6 +37,7 @@ class EditableSchema:
 
             if isinstance(value, BaseSchema):
                 value._BaseSchema__parent = self.__schema
+                value._BaseSchema__key = key
 
             if key == "default":
                 self.__schema._BaseSchema__default = value
@@ -47,9 +54,10 @@ class EditableSchema:
                 self.__schema._BaseSchema__default = new_schema
         else:
             new_schema = self.__schema._BaseSchema__manifest.setdefault(key, new_schema)
+        new_schema._BaseSchema__key = key
         EditableSchema(new_schema).__insert(keypath, value, fullkey, clobber)
 
-    def __remove(self, keypath, fullkey):
+    def __remove(self, keypath: Tuple[str], fullkey: Tuple[str]) -> None:
         key = keypath[0]
         keypath = keypath[1:]
 
@@ -69,7 +77,7 @@ class EditableSchema:
         else:
             EditableSchema(next_param).__remove(keypath, fullkey)
 
-    def insert(self, *args, clobber=False):
+    def insert(self, *args, clobber: bool = False) -> None:
         '''
         Inserts a :class:`Parameter` or a :class:`BaseSchema` to the schema,
         based on the keypath and value provided in the ``*args``.
@@ -98,7 +106,7 @@ class EditableSchema:
 
         self.__insert(keypath, value, keypath, clobber=clobber)
 
-    def remove(self, *keypath):
+    def remove(self, *keypath: str) -> None:
         '''
         Removes a keypath from the schema.
 
@@ -118,7 +126,7 @@ class EditableSchema:
 
         self.__remove(keypath, keypath)
 
-    def search(self, *keypath):
+    def search(self, *keypath: str) -> Union[BaseSchema, Parameter]:
         '''
         Finds an item in the schema. This will raise a KeyError if
         the path is not found.

siliconcompiler/schema/journal.py

@@ -1,6 +1,8 @@
 import copy
 import json
 
+from typing import Tuple, Set
+
 
 class Journal:
     """
@@ -12,7 +14,7 @@ class Journal:
         keyprefix (list of str): keypath to prefix on to recorded path
     """
 
-    def __init__(self, keyprefix=None):
+    def __init__(self, keyprefix: Tuple[str] = None):
         if not keyprefix:
             self.__keyprefix = tuple()
         else:
@@ -24,14 +26,14 @@ class Journal:
         self.stop()
 
     @property
-    def keypath(self):
+    def keypath(self) -> Tuple[str]:
         '''
         Returns the reference key path for this journal
         '''
 
         return self.__keyprefix
 
-    def get_child(self, *keypath):
+    def get_child(self, *keypath: Tuple[str]):
         '''
         Get a child journal based on a new keypath
 
@@ -60,26 +62,26 @@ class Journal:
 
         return copy.deepcopy(self.__parent.__journal)
 
-    def has_journaling(self):
+    def has_journaling(self) -> bool:
         """
         Returns true if the schema is currently setup and is the root of the journal and has data
         """
         return self is self.__parent and bool(self.__journal)
 
-    def is_journaling(self):
+    def is_journaling(self) -> bool:
         """
         Returns true if the schema is currently setup for journaling
         """
         return self.__parent.__journal is not None
 
-    def get_types(self):
+    def get_types(self) -> Set[str]:
         """
         Returns the current schema accesses that are being recorded
         """
 
         return self.__parent.__record_types.copy()
 
-    def add_type(self, value):
+    def add_type(self, value: str) -> None:
         """
         Adds a new access type to the journal record.
 
@@ -90,9 +92,9 @@ class Journal:
         if value not in ("set", "add", "remove", "unset", "get"):
             raise ValueError(f"{value} is not a valid type")
 
-        return self.__parent.__record_types.add(value)
+        self.__parent.__record_types.add(value)
 
-    def remove_type(self, value):
+    def remove_type(self, value: str) -> None:
         """
         Removes a new access type to the journal record.
 
@@ -105,7 +107,13 @@ class Journal:
         except KeyError:
             pass
 
-    def record(self, record_type, key, value=None, field=None, step=None, index=None):
+    def record(self,
+               record_type: str,
+               key: Tuple[str],
+               value=None,
+               field: str = None,
+               step: str = None,
+               index: str = None) -> None:
         '''
         Record the schema transaction
         '''
@@ -125,7 +133,7 @@ class Journal:
             "index": index
         })
 
-    def start(self):
+    def start(self) -> None:
         '''
         Start journaling the schema transactions
         '''
@@ -135,7 +143,7 @@ class Journal:
         self.add_type("remove")
         self.add_type("unset")
 
-    def stop(self):
+    def stop(self) -> None:
         '''
         Stop journaling the schema transactions
         '''
@@ -143,7 +151,7 @@ class Journal:
         self.__parent.__record_types.clear()
 
     @staticmethod
-    def replay_file(schema, filepath):
+    def replay_file(schema, filepath: str) -> None:
         '''
         Replay a journal into a schema from a manifest
 
@@ -160,7 +168,7 @@ class Journal:
         journal.from_dict(data["__journal__"])
         journal.replay(schema)
 
-    def replay(self, schema):
+    def replay(self, schema) -> None:
         '''
         Replay journal into a schema
 
@@ -196,7 +204,7 @@ class Journal:
                 raise ValueError(f'Unknown record type {record_type}')
 
     @staticmethod
-    def access(schema):
+    def access(schema) -> None:
         '''
         Access a journal from a schema