siliconcompiler 0.34.2__py3-none-any.whl → 0.34.3__py3-none-any.whl

siliconcompiler/schema/baseschema.py

@@ -6,6 +6,8 @@
 
 import contextlib
 import copy
+import importlib
+import logging
 
 try:
     import gzip
@@ -22,6 +24,9 @@ except ModuleNotFoundError:
 
 import os.path
 
+from functools import cache
+from typing import Dict, Type, Tuple, Union, Set, Callable, List
+
 from .parameter import Parameter, NodeValue
 from .journal import Journal
 
@@ -49,7 +54,88 @@ class BaseSchema:
             return tuple()
         return tuple([*self.__parent._keypath, self.__key])
 
-    def _from_dict(self, manifest, keypath, version=None):
+    @staticmethod
+    @cache
+    def __get_child_classes() -> Dict[str, Type["BaseSchema"]]:
+        """
+        Returns all known subclasses of BaseSchema
+        """
+        def recurse(cls):
+            subclss = set()
+            subclss.add(cls)
+            for subcls in cls.__subclasses__():
+                subclss.update(recurse(subcls))
+            return subclss
+
+        # Resolve true base
+        cls_mapping = {}
+        for cls in recurse(BaseSchema):
+            try:
+                cls_mapping.setdefault(cls._getdict_type(), set()).add(cls)
+            except NotImplementedError:
+                pass
+
+        # Build lookup table
+        cls_map = {}
+        for cls_type, clss in cls_mapping.items():
+            for cls in clss:
+                cls_map[f"{cls.__module__}/{cls.__name__}"] = cls
+
+            if len(clss) > 1:
+                found = False
+                for cls in clss:
+                    if cls.__name__ == cls_type:
+                        cls_map[cls_type] = cls
+                        found = True
+                        break
+                if not found:
+                    raise RuntimeError(f"fatal error at: {cls_type}")
+            else:
+                cls_map[cls_type] = list(clss)[0]
+        return cls_map
+
+    @staticmethod
+    @cache
+    def __load_schema_class(cls_name: str) -> Type["BaseSchema"]:
+        """
+        Load a schema class from a string
+        """
+        try:
+            module_name, cls_name = cls_name.split("/")
+        except (ValueError, AttributeError):
+            return None
+
+        try:
+            module = importlib.import_module(module_name)
+        except (ImportError, ModuleNotFoundError, SyntaxError):
+            return None
+
+        cls = getattr(module, cls_name, None)
+        if not cls:
+            return None
+        if not issubclass(cls, BaseSchema):
+            raise TypeError(f"{cls_name} must be a BaseSchema type")
+        return cls
+
+    @staticmethod
+    def __process_meta_section(meta: Dict[str, str]) -> Type["BaseSchema"]:
+        """
+        Handle __meta__ section of the schema by loading the appropriate class
+        """
+        cls_map = BaseSchema.__get_child_classes()
+
+        # Lookup object, use class first, then type
+        cls_name = meta.get("class", None)
+        cls = None
+        if cls_name:
+            cls = cls_map.get(cls_name, None)
+            if not cls:
+                cls = BaseSchema.__load_schema_class(cls_name)
+        if not cls:
+            cls = cls_map.get(meta.get("sctype", None), None)
+        return cls
+
+    def _from_dict(self, manifest: Dict, keypath: Tuple[str], version: str = None):
         '''
         Decodes a dictionary into a schema object
 
@@ -64,6 +150,10 @@ class BaseSchema:
 
         if "__journal__" in manifest:
             self.__journal.from_dict(manifest["__journal__"])
+            del manifest["__journal__"]
+
+        if "__meta__" in manifest:
+            del manifest["__meta__"]
 
         if self.__default:
             data = manifest.get("default", None)
@@ -74,9 +164,25 @@ class BaseSchema:
 
         for key, data in manifest.items():
             obj = self.__manifest.get(key, None)
+            if not obj and isinstance(data, dict) and "__meta__" in data:
+                # Lookup object, use class first, then type
+                cls = BaseSchema.__process_meta_section(data["__meta__"])
+                if cls is BaseSchema and self.__default:
+                    # Use default when BaseSchema is the class
+                    obj = self.__default.copy(key=keypath + [key])
+                    self.__manifest[key] = obj
+                elif cls:
+                    # Create object and connect to schema
+                    obj = cls()
+                    obj.__parent = self
+                    obj.__key = key
+                    self.__manifest[key] = obj
+
+            # Use default if it is available
             if not obj and self.__default:
-                obj = self.__default.copy()
+                obj = self.__default.copy(key=keypath + [key])
                 self.__manifest[key] = obj
+
             if obj:
                 obj._from_dict(data, keypath + [key], version=version)
                 handled.add(key)
@@ -87,7 +193,7 @@ class BaseSchema:
 
     # Manifest methods
     @classmethod
-    def from_manifest(cls, filepath=None, cfg=None):
+    def from_manifest(cls, filepath: str = None, cfg: Dict = None) -> "BaseSchema":
         '''
         Create a new schema based on the provided source files.
 
@@ -101,15 +207,24 @@ class BaseSchema:
         if not filepath and cfg is None:
             raise RuntimeError("filepath or dictionary is required")
 
-        schema = cls()
         if filepath:
-            schema.read_manifest(filepath)
-        if cfg:
-            schema._from_dict(cfg, [])
+            cfg = BaseSchema._read_manifest(filepath)
+
+        new_cls = None
+        if "__meta__" in cfg:
+            # Determine correct class
+            new_cls = BaseSchema.__process_meta_section(cfg["__meta__"])
+        if new_cls:
+            schema = new_cls()
+        else:
+            schema = cls()
+
+        schema._from_dict(cfg, [])
+
         return schema
 
     @staticmethod
-    def __open_file(filepath, is_read=True):
+    def __open_file(filepath: str, is_read: bool = True):
         _, ext = os.path.splitext(filepath)
         if ext.lower() == ".gz":
             if not _has_gzip:
@@ -117,10 +232,27 @@ class BaseSchema:
             return gzip.open(filepath, mode="rt" if is_read else "wt", encoding="utf-8")
         return open(filepath, mode="r" if is_read else "w", encoding="utf-8")
 
-    def __format_key(self, *key):
+    def __format_key(self, *key: str):
         return f"[{','.join([*self._keypath, *key])}]"
 
-    def read_manifest(self, filepath):
+    @staticmethod
+    def _read_manifest(filepath: str):
+        """
+        Reads a manifest from disk and returns dictionary.
+
+        Args:
+            filename (path): Path to a manifest file to be loaded.
+        """
+
+        fin = BaseSchema.__open_file(filepath)
+        try:
+            manifest = json.loads(fin.read())
+        finally:
+            fin.close()
+
+        return manifest
+
+    def read_manifest(self, filepath: str):
         """
         Reads a manifest from disk and replaces the current data with the data in the file.
 
@@ -132,13 +264,9 @@ class BaseSchema:
             Loads the file mychip.json into the current Schema object.
         """
 
-        fin = BaseSchema.__open_file(filepath)
-        manifest = json.loads(fin.read())
-        fin.close()
-
-        self._from_dict(manifest, [])
+        self._from_dict(BaseSchema._read_manifest(filepath), [])
 
-    def write_manifest(self, filepath):
+    def write_manifest(self, filepath: str):
         '''
         Writes the manifest to a file.
 
@@ -162,11 +290,11 @@ class BaseSchema:
 
     # Accessor methods
     def __search(self,
-                 *keypath,
-                 insert_defaults=False,
-                 use_default=False,
-                 require_leaf=True,
-                 complete_path=None):
+                 *keypath: str,
+                 insert_defaults: bool = False,
+                 use_default: bool = False,
+                 require_leaf: bool = True,
+                 complete_path: bool = None) -> Union["BaseSchema", Parameter]:
         if len(keypath) == 0:
             if require_leaf:
                 raise KeyError
@@ -204,7 +332,8 @@ class BaseSchema:
                                       complete_path=complete_path)
         return key_param
 
-    def get(self, *keypath, field='value', step=None, index=None):
+    def get(self, *keypath: str, field: str = 'value',
+            step: str = None, index: Union[int, str] = None):
         """
         Returns a parameter field from the schema.
 
@@ -260,7 +389,8 @@ class BaseSchema:
             e.args = (new_msg, *e.args[1:])
             raise e
 
-    def set(self, *args, field='value', clobber=True, step=None, index=None):
+    def set(self, *args: str, field: str = 'value', clobber: bool = True,
+            step: str = None, index: Union[int, str] = None):
         '''
         Sets a schema parameter field.
 
@@ -305,7 +435,8 @@ class BaseSchema:
             e.args = (new_msg, *e.args[1:])
             raise e
 
-    def add(self, *args, field='value', step=None, index=None):
+    def add(self, *args: str, field: str = 'value',
+            step: str = None, index: Union[int, str] = None):
         '''
         Adds item(s) to a schema parameter list.
 
@@ -348,7 +479,7 @@ class BaseSchema:
             e.args = (new_msg, *e.args[1:])
             raise e
 
-    def unset(self, *keypath, step=None, index=None):
+    def unset(self, *keypath: str, step: str = None, index: Union[int, str] = None):
         '''
         Unsets a schema parameter.
 
@@ -387,7 +518,7 @@ class BaseSchema:
             e.args = (new_msg, *e.args[1:])
             raise e
 
-    def remove(self, *keypath):
+    def remove(self, *keypath: str):
         '''
         Remove a schema parameter and its subparameters.
 
@@ -417,7 +548,8 @@ class BaseSchema:
         del key_param.__manifest[removal_key]
         self.__journal.record("remove", keypath)
 
-    def valid(self, *keypath, default_valid=False, check_complete=False):
+    def valid(self, *keypath: str, default_valid: bool = False,
+              check_complete: bool = False) -> bool:
         """
         Checks validity of a keypath.
 
@@ -451,7 +583,7 @@ class BaseSchema:
             return isinstance(param, Parameter)
         return True
 
-    def getkeys(self, *keypath):
+    def getkeys(self, *keypath: str) -> Tuple[str]:
         """
         Returns a tuple of schema dictionary keys.
 
@@ -481,7 +613,7 @@ class BaseSchema:
 
         return tuple(sorted(key_param.__manifest.keys()))
 
-    def allkeys(self, *keypath, include_default=True):
+    def allkeys(self, *keypath: str, include_default: bool = True) -> Set[Tuple[str]]:
         '''
         Returns all keypaths in the schema as a set of tuples.
 
@@ -510,7 +642,16 @@ class BaseSchema:
             add(keys, key, item)
         return set(keys)
 
-    def getdict(self, *keypath, include_default=True, values_only=False):
+    @classmethod
+    def _getdict_type(cls) -> str:
+        """
+        Returns the meta data for getdict
+        """
+
+        return "BaseSchema"
+
+    def getdict(self, *keypath: str, include_default: bool = True,
+                values_only: bool = False) -> Dict:
         """
         Returns a schema dictionary.
 
@@ -553,10 +694,19 @@ class BaseSchema:
         if not values_only and self.__journal.has_journaling():
             manifest["__journal__"] = self.__journal.get()
 
+        if not values_only and self.__class__ is not BaseSchema:
+            manifest["__meta__"] = {
+                "class": f"{self.__class__.__module__}/{self.__class__.__name__}"
+            }
+            try:
+                manifest["__meta__"]["sctype"] = self._getdict_type()
+            except NotImplementedError:
+                pass
+
         return manifest
 
     # Utility functions
-    def copy(self, key=None):
+    def copy(self, key: Tuple[str] = None):
         """
         Returns a copy of this schema.
 
@@ -579,7 +729,7 @@ class BaseSchema:
 
         return schema_copy
 
-    def _find_files_search_paths(self, keypath, step, index):
+    def _find_files_search_paths(self, keypath: Tuple[str], step: str, index: str):
         """
         Returns a list of paths to search during find files.
 
@@ -601,8 +751,11 @@ class BaseSchema:
             return {}
         return self.__parent._find_files_dataroot_resolvers()
 
-    def find_files(self, *keypath, missing_ok=False, step=None, index=None,
-                   packages=None, collection_dir=None, cwd=None):
+    def find_files(self, *keypath: str, missing_ok: bool = False,
+                   step: str = None, index: Union[int, str] = None,
+                   packages: Dict[str, Union[str, Callable]] = None,
+                   collection_dir: str = None,
+                   cwd: str = None) -> Union[str, List[str], Set[str]]:
         """
         Returns absolute paths to files or directories based on the keypath
         provided.
@@ -707,8 +860,11 @@ class BaseSchema:
             return resolved_paths[0]
         return resolved_paths
 
-    def check_filepaths(self, ignore_keys=None, logger=None,
-                        packages=None, collection_dir=None, cwd=None):
+    def check_filepaths(self, ignore_keys: bool = None,
+                        logger: logging.Logger = None,
+                        packages: Dict[str, Union[str, Callable]] = None,
+                        collection_dir: str = None,
+                        cwd: str = None) -> bool:
         '''
         Verifies that paths to all files in manifest are valid.
 
@@ -773,7 +929,7 @@ class BaseSchema:
 
         return not error
 
-    def _parent(self, root=False):
+    def _parent(self, root: bool = False) -> "BaseSchema":
         '''
         Returns the parent of this schema section, if root is true the root parent
         will be returned.
@@ -815,7 +971,7 @@ class BaseSchema:
         finally:
             self.__active = orig_active
 
-    def _get_active(self, field, defvalue=None):
+    def _get_active(self, field: str, defvalue=None):
         '''
         Get the value of a specific field.
 

siliconcompiler/schema/journal.py

@@ -1,7 +1,7 @@
 import copy
 import json
 
-from typing import Tuple, Set
+from typing import Tuple, Set, Dict, List
 
 
 class Journal:
@@ -45,7 +45,7 @@ class Journal:
         child.__parent = self.__parent
         return child
 
-    def from_dict(self, manifest):
+    def from_dict(self, manifest: Dict):
         '''
         Import a journal from a manifest dictionary
 
@@ -55,7 +55,7 @@ class Journal:
 
         self.__journal = manifest
 
-    def get(self):
+    def get(self) -> List[Dict]:
         """
         Returns a copy of the current journal
         """
@@ -124,6 +124,9 @@ class Journal:
         if record_type not in self.__parent.__record_types:
             return
 
+        if isinstance(value, set):
+            value = list(value)
+
         self.__parent.__journal.append({
             "type": record_type,
             "key": tuple([*self.__keyprefix, *key]),
@@ -204,7 +207,7 @@ class Journal:
                 raise ValueError(f'Unknown record type {record_type}')
 
     @staticmethod
-    def access(schema) -> None:
+    def access(schema) -> "Journal":
         '''
         Access a journal from a schema
 

siliconcompiler/schema/namedschema.py

@@ -4,6 +4,8 @@
 # SC dependencies outside of its directory, since it may be used by tool drivers
 # that have isolated Python environments.
 
+from typing import Dict, Tuple
+
 from .baseschema import BaseSchema
 
 
@@ -20,6 +22,7 @@ class NamedSchema(BaseSchema):
 
         self.set_name(name)
 
+    @property
     def name(self) -> str:
         '''
         Returns the name of the schema
@@ -40,26 +43,28 @@ class NamedSchema(BaseSchema):
             name (str): name for object
         """
 
-        if self.name() is not None:
+        if self.name is not None:
             raise RuntimeError("Cannot call set_name more than once.")
         if name is not None and "." in name:
             raise ValueError("Named schema object cannot contains: .")
         self.__name = name
 
-    def _reset(self) -> None:
+    def type(self) -> str:
         """
-        Resets the state of the object
+        Returns the type of this object
         """
-        pass
+        raise NotImplementedError("Must be implemented by the child classes.")
 
-    def type(self) -> str:
+    @classmethod
+    def _getdict_type(cls) -> str:
         """
-        Returns the type of this object
+        Returns the meta data for getdict
         """
+
         raise NotImplementedError("Must be implemented by the child classes.")
 
     @classmethod
-    def from_manifest(cls, name, filepath=None, cfg=None):
+    def from_manifest(cls, name: str, filepath: str = None, cfg: Dict = None):
         '''
         Create a new schema based on the provided source files.
 
@@ -74,20 +79,21 @@ class NamedSchema(BaseSchema):
         if not filepath and cfg is None:
             raise RuntimeError("filepath or dictionary is required")
 
-        schema = cls(name)
+        schema = cls()
+        schema.set_name(name)
         if filepath:
             schema.read_manifest(filepath)
         if cfg:
             schema._from_dict(cfg, [])
         return schema
 
-    def _from_dict(self, manifest, keypath, version=None):
+    def _from_dict(self, manifest: Dict, keypath: Tuple[str], version: str = None):
         if keypath:
             self.__name = keypath[-1]
 
         return super()._from_dict(manifest, keypath, version=version)
 
-    def copy(self, key=None):
+    def copy(self, key: Tuple[str] = None) -> "NamedSchema":
         copy = super().copy(key=key)
 
         if key and key[-1] != "default":

siliconcompiler/schema/parameter.py

@@ -574,14 +574,27 @@ class Parameter:
 
         if self.__pernode == PerNode.REQUIRED and (step is None or index is None):
             return None
-        if not self.__pernode.is_never():
-            value = self.get(step=step, index=index)
-        else:
-            value = self.get()
 
-        return NodeType.to_tcl(value, self.__type)
+        if isinstance(index, int):
+            index = str(index)
 
-    def getvalues(self, return_defvalue=True):
+        try:
+            return self.__node[step][index].gettcl()
+        except KeyError:
+            if self.__pernode == PerNode.REQUIRED:
+                return self.__defvalue.gettcl()
+
+        try:
+            return self.__node[step][Parameter.GLOBAL_KEY].gettcl()
+        except KeyError:
+            pass
+
+        try:
+            return self.__node[Parameter.GLOBAL_KEY][Parameter.GLOBAL_KEY].gettcl()
+        except KeyError:
+            return self.__defvalue.gettcl()
+
+    def getvalues(self, return_defvalue=True, return_values=True):
         """
         Returns all values (global and pernode) associated with a particular parameter.
 
@@ -599,10 +612,16 @@ class Parameter:
                 index_arg = None if index == Parameter.GLOBAL_KEY else index
                 if step_arg is None and index_arg is None:
                     has_global = True
-                vals.append((self.__node[step][index].get(), step_arg, index_arg))
+                if return_values:
+                    vals.append((self.__node[step][index].get(), step_arg, index_arg))
+                else:
+                    vals.append((self.__node[step][index], step_arg, index_arg))
 
-        if (self.__pernode != PerNode.REQUIRED) and not has_global and return_defvalue:
-            vals.append((self.__defvalue.get(), None, None))
+        if self.__pernode != PerNode.REQUIRED and not has_global and return_defvalue:
+            if return_values:
+                vals.append((self.__defvalue.get(), None, None))
+            else:
+                vals.append((self.__defvalue, None, None))
 
         return vals
 
@@ -656,6 +675,33 @@ class Parameter:
             index in self.__node[step] and \
             self.__node[step][index]
 
+    def has_value(self, step=None, index=None) -> bool:
+        '''
+        Returns whether the parameter as a value.
+
+        A value counts as set if a user has set a global value OR a value for
+        the provided step/index.
+        '''
+
+        if isinstance(index, int):
+            index = str(index)
+
+        try:
+            return self.__node[step][index].has_value
+        except KeyError:
+            if self.__pernode == PerNode.REQUIRED:
+                return self.__defvalue.has_value
+
+        try:
+            return self.__node[step][Parameter.GLOBAL_KEY].has_value
+        except KeyError:
+            pass
+
+        try:
+            return self.__node[Parameter.GLOBAL_KEY][Parameter.GLOBAL_KEY].has_value
+        except KeyError:
+            return self.__defvalue.has_value
+
     @property
     def default(self):
         """