siliconcompiler 0.35.2__py3-none-any.whl → 0.35.3__py3-none-any.whl

siliconcompiler/schema/baseschema.py

@@ -24,14 +24,42 @@ except ModuleNotFoundError:
 
 import os.path
 
+from enum import Enum, auto
 from functools import cache
-from typing import Dict, Type, Tuple, Union, Set, Callable, List, Optional, TextIO, Iterable
+from typing import Dict, Type, Tuple, Union, Set, Callable, List, Optional, TextIO, Iterable, Any
 
 from .parameter import Parameter, NodeValue
 from .journal import Journal
 from ._metadata import version
 
 
+class LazyLoad(Enum):
+    """
+    Controls manifest loading
+    """
+    OFF = auto()  # load entire schema immediately
+    ON = auto()  # store schema but do not load it
+    FORWARD = auto()  # load the current section but do not load children
+
+    @property
+    def next(self) -> "LazyLoad":
+        """
+        Returns the next state for lazy loading
+        """
+        if self == LazyLoad.ON:
+            return LazyLoad.ON
+        if self == LazyLoad.FORWARD:
+            return LazyLoad.ON
+        return LazyLoad.OFF
+
+    @property
+    def is_enforced(self) -> bool:
+        """
+        Returns true when the current section should not be loaded.
+        """
+        return self == LazyLoad.ON
+
+
 class BaseSchema:
     '''
     This class maintains the access and file IO operations for the schema.
@@ -42,21 +70,38 @@ class BaseSchema:
     __version = tuple([int(v) for v in version.split('.')])
 
     def __init__(self):
-        self.__manifest = {}
-        self.__default = None
-        self.__journal = Journal()
-        self.__parent = self
-        self.__active = None
-        self.__key = None
+        self.__manifest: Dict[str, Union["BaseSchema", Parameter]] = {}
+        self.__default: Optional[Union["BaseSchema", Parameter]] = None
+        self.__journal: Journal = Journal()
+        self.__parent: Optional["BaseSchema"] = None
+        self.__active: Optional[Dict] = None
+        self.__key: Optional[str] = None
+        self.__lazy: Optional[Tuple[Optional[Tuple[int, ...]], Dict]] = None
+
+    @property
+    def __is_root(self) -> bool:
+        '''
+        Returns true if the object is the root of the schema
+        '''
+        try:
+            return self.__parent is None
+        except AttributeError:
+            return True
 
     @property
     def _keypath(self) -> Tuple[str, ...]:
         '''
         Returns the key to the current section of the schema
         '''
-        if self.__parent is self:
+        if self.__is_root:
+            return tuple()
+        try:
+            parentpath = self.__parent._keypath
+            key = self.__key
+        except AttributeError:
+            # Guard against partially setup parents during serialization
             return tuple()
-        return tuple([*self.__parent._keypath, self.__key])
+        return tuple([*parentpath, key])
 
     @staticmethod
     @cache
@@ -146,13 +191,25 @@ class BaseSchema:
     def __extractversion(manifest: Dict) -> Optional[Tuple[int, ...]]:
         schema_version = manifest.get(BaseSchema._version_key, None)
         if schema_version:
-            param = Parameter.from_dict(schema_version, [BaseSchema._version_key], None)
+            param = Parameter.from_dict(schema_version,
+                                        tuple([BaseSchema._version_key]),
+                                        None)
             return tuple([int(v) for v in param.get().split('.')])
         return None
 
+    def __ensure_lazy_elab(self):
+        if not self.__lazy:
+            return
+
+        version, manifest = self.__lazy
+        self.__lazy = None
+
+        self._from_dict(manifest, self._keypath, version=version, lazyload=LazyLoad.FORWARD)
+
     def _from_dict(self, manifest: Dict,
                    keypath: Union[List[str], Tuple[str, ...]],
-                   version: Optional[Tuple[int, ...]] = None) \
+                   version: Optional[Tuple[int, ...]] = None,
+                   lazyload: LazyLoad = LazyLoad.ON) \
             -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
         '''
         Decodes a dictionary into a schema object
@@ -174,26 +231,35 @@ class BaseSchema:
 
         if "__journal__" in manifest:
             self.__journal.from_dict(manifest["__journal__"])
-            del manifest["__journal__"]
+            if lazyload != LazyLoad.ON:
+                del manifest["__journal__"]
+
+        if lazyload == LazyLoad.ON:
+            self.__lazy = (version, manifest)
+            return set(), set()
 
         if "__meta__" in manifest:
             del manifest["__meta__"]
 
         if self.__default:
-            data = manifest.get("default", None)
+            data = manifest.pop("default", None)
             if data:
-                del manifest["default"]
-                self.__default._from_dict(data, list(keypath) + ["default"], version=version)
+                if isinstance(self.__default, BaseSchema):
+                    self.__default._from_dict(data, tuple([*keypath, "default"]), version=version,
+                                              lazyload=lazyload.next)
+                else:
+                    self.__default._from_dict(data, tuple([*keypath, "default"]), version=version)
                 handled.add("default")
 
         for key, data in manifest.items():
+            data_keypath = tuple([*keypath, key])
             obj = self.__manifest.get(key, None)
             if not obj and isinstance(data, dict) and "__meta__" in data:
                 # Lookup object, use class first, then type
                 cls = BaseSchema.__process_meta_section(data["__meta__"])
                 if cls is BaseSchema and self.__default:
                     # Use default when BaseSchema is the class
-                    obj = self.__default.copy(key=list(keypath) + [key])
+                    obj = self.__default.copy(key=data_keypath)
                     self.__manifest[key] = obj
                 elif cls:
                     # Create object and connect to schema
@@ -204,11 +270,14 @@ class BaseSchema:
 
             # Use default if it is available
             if not obj and self.__default:
-                obj = self.__default.copy(key=list(keypath) + [key])
+                obj = self.__default.copy(key=data_keypath)
                 self.__manifest[key] = obj
 
             if obj:
-                obj._from_dict(data, list(keypath) + [key], version=version)
+                if isinstance(obj, BaseSchema):
+                    obj._from_dict(data, data_keypath, version=version, lazyload=lazyload.next)
+                else:
+                    obj._from_dict(data, data_keypath, version=version)
                 handled.add(key)
             else:
                 missing.add(key)
@@ -219,7 +288,8 @@ class BaseSchema:
     @classmethod
     def from_manifest(cls,
                       filepath: Union[None, str] = None,
-                      cfg: Union[None, Dict] = None) -> "BaseSchema":
+                      cfg: Union[None, Dict] = None,
+                      lazyload: bool = True) -> "BaseSchema":
         '''
         Create a new schema based on the provided source files.
 
@@ -245,7 +315,12 @@ class BaseSchema:
         else:
             schema = cls()
 
-        schema._from_dict(cfg, tuple())
+        if lazyload:
+            do_lazyload = LazyLoad.ON
+        else:
+            do_lazyload = LazyLoad.OFF
+
+        schema._from_dict(cfg, tuple(), lazyload=do_lazyload)
 
         return schema
 
@@ -321,13 +396,19 @@ class BaseSchema:
                  insert_defaults: bool = False,
                  use_default: bool = False,
                  require_leaf: bool = True,
-                 complete_path: Optional[List[str]] = None) -> Union["BaseSchema", Parameter]:
+                 complete_path: Optional[List[str]] = None,
+                 elaborate_leaf: bool = True) -> Union["BaseSchema", Parameter]:
+
         if len(keypath) == 0:
             if require_leaf:
                 raise KeyError
             else:
+                if elaborate_leaf:
+                    self.__ensure_lazy_elab()
                 return self
 
+        self.__ensure_lazy_elab()
+
         if complete_path is None:
             complete_path = []
         complete_path.append(keypath[0])
@@ -351,6 +432,8 @@ class BaseSchema:
                 if require_leaf:
                     raise KeyError
                 else:
+                    if elaborate_leaf:
+                        key_param.__ensure_lazy_elab()
                     return key_param
             return key_param.__search(*keypath[1:],
                                       insert_defaults=insert_defaults,
@@ -637,16 +720,13 @@ class BaseSchema:
             >>> keylist = schema.getkeys('pdk')
             Returns all keys for the [pdk] keypath.
         """
+        try:
+            key_param = self.__search(*keypath, require_leaf=False)
+        except KeyError:
+            raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
 
-        if keypath:
-            try:
-                key_param = self.__search(*keypath, require_leaf=False)
-            except KeyError:
-                raise KeyError(f"{self.__format_key(*keypath)} is not a valid keypath")
-            if isinstance(key_param, Parameter):
-                return tuple()
-        else:
-            key_param = self
+        if isinstance(key_param, Parameter):
+            return tuple()
 
         return tuple(sorted(key_param.__manifest.keys()))
 
@@ -658,14 +738,17 @@ class BaseSchema:
             keypath (list of str): Keypath prefix to search under. The
                 returned keypaths do not include the prefix.
         '''
+        try:
+            key_param = self.__search(*keypath, require_leaf=False)
+        except KeyError:
+            return set()
 
-        if keypath:
-            key_param = self.__manifest.get(keypath[0], None)
-            if not key_param or isinstance(key_param, Parameter):
-                return set()
-            return key_param.allkeys(*keypath[1:], include_default=include_default)
+        if isinstance(key_param, Parameter):
+            return set()
 
-        def add(keys, key, item):
+        def add(keys: List[Tuple[str, ...]],
+                key: str,
+                item: Union["BaseSchema", Parameter]) -> None:
             if isinstance(item, Parameter):
                 keys.append((key,))
             else:
@@ -673,9 +756,9 @@ class BaseSchema:
                     keys.append((key, *subkeypath))
 
         keys = []
-        if include_default and self.__default:
-            add(keys, "default", self.__default)
-        for key, item in self.__manifest.items():
+        if include_default and key_param.__default:
+            add(keys, "default", key_param.__default)
+        for key, item in key_param.__manifest.items():
             add(keys, key, item)
         return set(keys)
 
@@ -714,42 +797,52 @@ class BaseSchema:
             >>> pdk = schema.getdict('pdk')
             Returns the complete dictionary found for the keypath [pdk]
         """
+        try:
+            if not values_only and include_default:
+                key_param = self.__search(*keypath, require_leaf=False, elaborate_leaf=False)
 
-        if keypath:
-            key_param = self.__manifest.get(keypath[0], None)
-            if not key_param:
-                return {}
-            return key_param.getdict(*keypath[1:],
-                                     include_default=include_default,
-                                     values_only=values_only)
+                if isinstance(key_param, Parameter):
+                    return key_param.getdict(include_default=include_default,
+                                             values_only=values_only)
+
+                if key_param.__lazy:
+                    return key_param.__lazy[1]
+            else:
+                key_param = self.__search(*keypath, require_leaf=False)
+        except KeyError:
+            return {}
+
+        if isinstance(key_param, Parameter):
+            return key_param.getdict(include_default=include_default, values_only=values_only)
 
         manifest = {}
-        if include_default and self.__default:
-            manifest_dict = self.__default.getdict(include_default=include_default,
-                                                   values_only=values_only)
+        if include_default and key_param.__default:
+            manifest_dict = key_param.__default.getdict(include_default=include_default,
+                                                        values_only=values_only)
             if manifest_dict or not values_only:
                 manifest["default"] = manifest_dict
-        for key, item in self.__manifest.items():
+        for key, item in key_param.__manifest.items():
             manifest_dict = item.getdict(include_default=include_default,
                                          values_only=values_only)
             if manifest_dict or not values_only:
                 manifest[key] = manifest_dict
 
-        if not values_only and self.__journal.has_journaling():
-            manifest["__journal__"] = self.__journal.get()
+        if not values_only and key_param.__journal.has_journaling():
+            manifest["__journal__"] = key_param.__journal.get()
 
-        if not values_only and self.__class__ is not BaseSchema:
+        if not values_only and key_param.__class__ is not BaseSchema:
             manifest["__meta__"] = {}
 
             try:
-                cls_meta = self._getdict_meta()
+                cls_meta = key_param._getdict_meta()
                 manifest["__meta__"].update(cls_meta)
             except NotImplementedError:
                 pass
 
-            manifest["__meta__"]["class"] = f"{self.__class__.__module__}/{self.__class__.__name__}"
+            manifest["__meta__"]["class"] = \
+                f"{key_param.__class__.__module__}/{key_param.__class__.__name__}"
             try:
-                manifest["__meta__"]["sctype"] = self._getdict_type()
+                manifest["__meta__"]["sctype"] = key_param._getdict_type()
             except NotImplementedError:
                 pass
 
@@ -768,11 +861,7 @@ class BaseSchema:
         self.__parent = None
         schema_copy = copy.deepcopy(self)
         self.__parent = parent
-
-        if self is not self.__parent:
-            schema_copy.__parent = self.__parent
-        else:
-            schema_copy.__parent = schema_copy
+        schema_copy.__parent = self.__parent
 
         if key:
             schema_copy.__key = key[-1]
@@ -792,16 +881,20 @@ class BaseSchema:
         """
         return []
 
-    def _find_files_dataroot_resolvers(self) -> Dict[str, Union[str, Callable]]:
+    def _find_files_dataroot_resolvers(self, resolvers: bool = False) \
+            -> Dict[str, Union[str, Callable]]:
         """
         Returns a dictionary of path resolvers data directory handling for find_files
 
+        Args:
+            resolvers (bool, optional): Returns the resolvers instead of callables
+
         Returns:
             dictionary of str to resolver mapping
         """
-        if self.__parent is self:
+        if self.__is_root:
             return {}
-        return self.__parent._find_files_dataroot_resolvers()
+        return self.__parent._find_files_dataroot_resolvers(resolvers=resolvers)
 
     def _find_files(self, *keypath: str, missing_ok: bool = False,
                     step: Optional[str] = None, index: Optional[Union[int, str]] = None,
@@ -927,6 +1020,7 @@ class BaseSchema:
             search_paths = root_search_paths.copy()
 
             dataroot: Optional[str] = path.get(field="dataroot")
+            dataroot_except: Optional[Exception] = None
             if dataroot:
                 if dataroot not in dataroots:
                     raise ValueError(f"Resolver for {dataroot} not provided: "
@@ -935,7 +1029,10 @@ class BaseSchema:
                 if isinstance(dataroot_path, str):
                     search_paths.append(os.path.abspath(dataroot_path))
                 elif callable(dataroot_path):
-                    search_paths.append(dataroot_path())
+                    try:
+                        search_paths.append(dataroot_path())
+                    except Exception as e:
+                        dataroot_except = e
                 else:
                     raise TypeError(f"Resolver for {dataroot} is not a recognized type: "
                                     f"{self.__format_key(*keypath)}")
@@ -956,6 +1053,10 @@ class BaseSchema:
                 if not missing_ok:
                     report_paths = ", ".join(search_paths)
                     if dataroot:
+                        if dataroot_except:
+                            raise FileNotFoundError(
+                                f"Dataroot {dataroot} not found: {dataroot_except}") \
+                                    from dataroot_except
                         raise FileNotFoundError(
                             f'Could not find "{path.get()}" in {dataroot} '
                             f'{self.__format_key(*keypath)}: {report_paths}')
@@ -1036,7 +1137,7 @@ class BaseSchema:
                                     node_indicator = f" ({step}/{index})"
 
                             name = ""
-                            if self.__parent is self and hasattr(self, "name"):
+                            if hasattr(self, "name"):
                                 name = f"({self.name}) "
 
                             logger.error(f"Parameter {name}"
@@ -1101,10 +1202,11 @@ class BaseSchema:
         Args:
             root (bool): if true, returns the root of the schemas.
         '''
+        if self.__is_root:
+            return self
+
         if not root:
             return self.__parent
-        if self.__parent is self:
-            return self
         return self.__parent._parent(root=root)
 
     @contextlib.contextmanager
@@ -1122,30 +1224,18 @@ class BaseSchema:
             Sets the file to top.v and associates lambdalib as the dataroot.
         '''
 
-        # Collect child schemas
-        subkeys = set()
-        for key in self.allkeys():
-            for n in range(len(key)):
-                subkeys.add(key[:n])
-        subschemas = set()
-        subschemas.add(self)
-        for key in subkeys:
-            subschemas.add(self.get(*key, field="schema"))
-
-        orig_active = {}
-        for schema in subschemas:
-            if schema.__active:
-                orig_active[schema] = schema.__active.copy()
-            else:
-                orig_active[schema] = None
-                schema.__active = {}
-            schema.__active.update(kwargs)
+        if self.__active is None:
+            orig_active = None
+            self.__active = {}
+        else:
+            orig_active = self.__active.copy()
+
+        self.__active.update(kwargs)
 
         try:
             yield
         finally:
-            for schema, orig in orig_active.items():
-                schema.__active = orig
+            self.__active = orig_active
 
     def _get_active(self, field: str, defvalue=None):
         '''
@@ -1156,20 +1246,42 @@ class BaseSchema:
                          otherwise the value, if the field is not present, defvalue is returned.
             defvalue (any): value to return if the field is not present.
         '''
-        if self.__active is None:
+        actives = self.__get_active()
+        if actives is None:
             return defvalue
 
         if field is None:
-            return self.__active.copy()
+            return actives
+
+        return actives.get(field, defvalue)
+
+    def __get_active(self) -> Optional[Dict[str, Any]]:
+        '''
+        Get the actives this point in the schema
+        '''
+        schemas: List["BaseSchema"] = [self]
+
+        root = self._parent(root=True)
+        curr = schemas[-1]
+        while curr is not root:
+            schemas.append(curr._parent())
+            curr = schemas[-1]
 
-        return self.__active.get(field, defvalue)
+        active = {}
+        for schema in reversed(schemas):
+            if schema.__active:
+                active.update(schema.__active)
+        if not active:
+            return None
+        return active
 
     def __process_active(self, param: Parameter,
                          nodevalues: Optional[Union[List[NodeValue],
                                                     Set[NodeValue],
                                                     Tuple[NodeValue, ...],
                                                     NodeValue]]) -> None:
-        if not self.__active:
+        actives = self.__get_active()
+        if actives is None:
             return
 
         if not isinstance(nodevalues, (list, set, tuple)):
@@ -1184,7 +1296,7 @@ class BaseSchema:
 
         key_fields = ("copy", "lock")
 
-        for field, value in self.__active.items():
+        for field, value in actives.items():
             if field in key_fields:
                 param.set(value, field=field)
                 continue

siliconcompiler/schema/namedschema.py

@@ -6,7 +6,7 @@
 
 from typing import Dict, Tuple, Optional, Set, Union, List
 
-from .baseschema import BaseSchema
+from .baseschema import BaseSchema, LazyLoad
 
 
 class NamedSchema(BaseSchema):
@@ -71,10 +71,17 @@ class NamedSchema(BaseSchema):
         info["name"] = self.name
         return info
 
+    @staticmethod
+    def __get_meta_name(cfg: Optional[Dict]) -> Optional[str]:
+        if not cfg:
+            return None
+        return cfg.get("__meta__", {}).get("name", None)
+
     @classmethod
     def from_manifest(cls,
                       filepath: Union[None, str] = None,
                       cfg: Union[None, Dict] = None,
+                      lazyload: bool = True,
                       name: Optional[str] = None):
         '''
         Create a new schema based on the provided source files.
@@ -86,28 +93,29 @@ class NamedSchema(BaseSchema):
             cfg (dict): Initial configuration dictionary.
             name (str): name of the schema.
         '''
+        if filepath and not cfg:
+            cfg = BaseSchema._read_manifest(filepath)
 
-        if not filepath and cfg is None:
-            raise RuntimeError("filepath or dictionary is required")
+        meta_name = NamedSchema.__get_meta_name(cfg)
+        schema = super().from_manifest(filepath=None, cfg=cfg, lazyload=lazyload)
+        if name:
+            schema.__name = name
+        elif meta_name:
+            schema.__name = meta_name
 
-        schema = cls()
-        schema.set_name(name)
-        if filepath:
-            schema.read_manifest(filepath)
-        if cfg:
-            schema._from_dict(cfg, tuple())
         return schema
 
     def _from_dict(self, manifest: Dict,
                    keypath: Union[List[str], Tuple[str, ...]],
-                   version: Optional[Tuple[int, ...]] = None) \
+                   version: Optional[Tuple[int, ...]] = None,
+                   lazyload: LazyLoad = LazyLoad.ON) \
             -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
         if keypath:
             self.__name = keypath[-1]
-        elif not self.__name and "__meta__" in manifest and "name" in manifest["__meta__"]:
-            self.__name = manifest["__meta__"]["name"]
+        elif not self.__name:
+            self.__name = NamedSchema.__get_meta_name(manifest)
 
-        return super()._from_dict(manifest, keypath, version=version)
+        return super()._from_dict(manifest, keypath, version=version, lazyload=lazyload)
 
     def copy(self, key: Optional[Tuple[str, ...]] = None) -> "NamedSchema":
         copy: NamedSchema = super().copy(key=key)

siliconcompiler/schema/safeschema.py

@@ -7,7 +7,7 @@
 from typing import Dict, Tuple, Optional, Union, List, Set
 
 from .parameter import Parameter
-from .baseschema import BaseSchema
+from .baseschema import BaseSchema, LazyLoad
 
 
 class SafeSchema(BaseSchema):
@@ -35,7 +35,8 @@ class SafeSchema(BaseSchema):
 
     def _from_dict(self, manifest: Dict,
                    keypath: Union[List[str], Tuple[str, ...]],
-                   version: Optional[Tuple[int, ...]] = None) \
+                   version: Optional[Tuple[int, ...]] = None,
+                   lazyload: LazyLoad = LazyLoad.OFF) \
             -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
         if not isinstance(manifest, dict):
             return set(), set()
@@ -43,11 +44,13 @@ class SafeSchema(BaseSchema):
         if "__meta__" in manifest:
             del manifest["__meta__"]
 
+        lazyload = LazyLoad.OFF
+
         for key, data in manifest.items():
             obj = SafeSchema.__is_dict_leaf(data, list(keypath) + [key], version)
             if not obj:
                 obj = SafeSchema()
-                obj._from_dict(data, list(keypath) + [key], version)
+                obj._from_dict(data, list(keypath) + [key], version=version, lazyload=lazyload)
 
             if key == "default":
                 self._BaseSchema__default = obj
@@ -59,11 +62,19 @@ class SafeSchema(BaseSchema):
     @classmethod
     def from_manifest(cls,
                       filepath: Union[None, str] = None,
-                      cfg: Union[None, Dict] = None) -> "SafeSchema":
+                      cfg: Union[None, Dict] = None,
+                      lazyload: bool = False) -> "SafeSchema":
         if filepath:
             cfg = BaseSchema._read_manifest(filepath)
 
-        if cfg and "__meta__" in cfg:
-            del cfg["__meta__"]
+        def rm_meta(manifest):
+            if not isinstance(manifest, dict):
+                return
+            if manifest and "__meta__" in manifest:
+                del manifest["__meta__"]
+            for section in manifest.values():
+                rm_meta(section)
+
+        rm_meta(cfg)
 
-        return super().from_manifest(filepath=None, cfg=cfg)
+        return super().from_manifest(filepath=None, cfg=cfg, lazyload=False)