siliconcompiler 0.33.2__py3-none-any.whl → 0.34.0__py3-none-any.whl

siliconcompiler/schema/baseschema.py

@@ -22,6 +22,7 @@ except ModuleNotFoundError:
 import os.path
 
 from .parameter import Parameter
+from .journal import Journal
 
 
 class BaseSchema:
@@ -34,6 +35,8 @@ class BaseSchema:
         # Data storage for the schema
         self.__manifest = {}
         self.__default = None
+        self.__journal = Journal()
+        self.__parent = self
 
     def _from_dict(self, manifest, keypath, version=None):
         '''
@@ -48,6 +51,9 @@ class BaseSchema:
         handled = set()
         missing = set()
 
+        if "__journal__" in manifest:
+            self.__journal.from_dict(manifest["__journal__"])
+
         if self.__default:
             data = manifest.get("default", None)
             if data:
@@ -74,16 +80,17 @@ class BaseSchema:
         '''
         Create a new schema based on the provided source files.
 
-        The two arguments to this class are mutually exclusive.
+        The two arguments to this method are mutually exclusive.
 
         Args:
             filepath (path): Initial manifest.
             cfg (dict): Initial configuration dictionary.
         '''
 
-        schema = cls()
         if not filepath and cfg is None:
             raise RuntimeError("filepath or dictionary is required")
+
+        schema = cls()
         if filepath:
             schema.read_manifest(filepath)
         if cfg:
@@ -220,6 +227,8 @@ class BaseSchema:
             if field == 'schema':
                 if isinstance(param, Parameter):
                     raise ValueError(f"[{','.join(keypath)}] is a complete keypath")
+                self.__journal.record("get", keypath, field=field, step=step, index=index)
+                param.__journal = self.__journal.get_child(*keypath)
                 return param
         except KeyError:
             raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
@@ -227,7 +236,9 @@ class BaseSchema:
             return param
 
         try:
-            return param.get(field, step=step, index=index)
+            get_ret = param.get(field, step=step, index=index)
+            self.__journal.record("get", keypath, field=field, step=step, index=index)
+            return get_ret
         except Exception as e:
             new_msg = f"error while accessing [{','.join(keypath)}]: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
@@ -266,7 +277,12 @@ class BaseSchema:
             raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
 
         try:
-            return param.set(value, field=field, clobber=clobber, step=step, index=index)
+            set_ret = param.set(value, field=field, clobber=clobber,
+                                step=step, index=index)
+            if set_ret:
+                self.__journal.record("set", keypath, value=value, field=field,
+                                      step=step, index=index)
+            return set_ret
         except Exception as e:
             new_msg = f"error while setting [{','.join(keypath)}]: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
@@ -304,7 +320,11 @@ class BaseSchema:
             raise KeyError(f"[{','.join(keypath)}] is not a valid keypath")
 
         try:
-            return param.add(value, field=field, step=step, index=index)
+            add_ret = param.add(value, field=field, step=step, index=index)
+            if add_ret:
+                self.__journal.record("add", keypath, value=value, field=field,
+                                      step=step, index=index)
+            return add_ret
         except Exception as e:
             new_msg = f"error while adding to [{','.join(keypath)}]: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
@@ -343,6 +363,7 @@ class BaseSchema:
 
         try:
             param.unset(step=step, index=index)
+            self.__journal.record("unset", keypath, step=step, index=index)
         except Exception as e:
             new_msg = f"error while unsetting [{','.join(keypath)}]: {e.args[0]}"
             e.args = (new_msg, *e.args[1:])
@@ -376,6 +397,7 @@ class BaseSchema:
             return
 
         del key_param.__manifest[removal_key]
+        self.__journal.record("remove", keypath)
 
     def valid(self, *keypath, default_valid=False, check_complete=False):
         """
@@ -470,7 +492,7 @@ class BaseSchema:
             add(keys, key, item)
         return set(keys)
 
-    def getdict(self, *keypath, include_default=True):
+    def getdict(self, *keypath, include_default=True, values_only=False):
         """
         Returns a schema dictionary.
 
@@ -480,6 +502,7 @@ class BaseSchema:
         Args:
             keypath (list of str): Variable length ordered schema key list
             include_default (boolean): If true will include default key paths
+            values_only (boolean): If true will only return values
 
         Returns:
             A schema dictionary
@@ -493,13 +516,25 @@ class BaseSchema:
             key_param = self.__manifest.get(keypath[0], None)
             if not key_param:
                 return {}
-            return key_param.getdict(*keypath[1:], include_default=include_default)
+            return key_param.getdict(*keypath[1:],
+                                     include_default=include_default,
+                                     values_only=values_only)
 
         manifest = {}
         if include_default and self.__default:
-            manifest["default"] = self.__default.getdict(include_default=include_default)
+            manifest_dict = self.__default.getdict(include_default=include_default,
+                                                   values_only=values_only)
+            if manifest_dict or not values_only:
+                manifest["default"] = manifest_dict
         for key, item in self.__manifest.items():
-            manifest[key] = item.getdict(include_default=include_default)
+            manifest_dict = item.getdict(include_default=include_default,
+                                         values_only=values_only)
+            if manifest_dict or not values_only:
+                manifest[key] = manifest_dict
+
+        if not values_only and self.__journal.has_journaling():
+            manifest["__journal__"] = self.__journal.get()
+
         return manifest
 
     # Utility functions
@@ -511,4 +546,193 @@ class BaseSchema:
             key (list of str): keypath to this schema
         """
 
-        return copy.deepcopy(self)
+        parent = self.__parent
+        self.__parent = None
+        schema_copy = copy.deepcopy(self)
+        self.__parent = parent
+
+        if self is not self.__parent:
+            schema_copy.__parent = self.__parent
+        else:
+            schema_copy.__parent = schema_copy
+
+        return schema_copy
+
+    def find_files(self, *keypath, missing_ok=False, step=None, index=None,
+                   packages=None, collection_dir=None, cwd=None):
+        """
+        Returns absolute paths to files or directories based on the keypath
+        provided.
+
+        The keypath provided must point to a schema parameter of type file, dir,
+        or lists of either. Otherwise, it will trigger an error.
+
+        Args:
+            keypath (list of str): Variable length schema key list.
+            missing_ok (bool): If True, silently return None when files aren't
+                found. If False, print an error and set the error flag.
+            step (str): Step name to access for parameters that may be specified
+                on a per-node basis.
+            index (str): Index name to access for parameters that may be specified
+                on a per-node basis.
+            packages (dict of resolvers): dirctionary of path resolvers for package
+                paths, these can either be a path or a callable function
+            collection_dir (path): optional path to a collections directory
+            cwd (path): optional path to current working directory, this will default
+                to os.getcwd() if not provided.
+
+        Returns:
+            If keys points to a scalar entry, returns an absolute path to that
+            file/directory, or None if not found. It keys points to a list
+            entry, returns a list of either the absolute paths or None for each
+            entry, depending on whether it is found.
+
+        Examples:
+            >>> chip.find_files('input', 'verilog')
+            Returns a list of absolute paths to source files, as specified in
+            the schema.
+        """
+
+        param = self.get(*keypath, field=None)
+        paramtype = param.get(field='type')
+        if 'file' not in paramtype and 'dir' not in paramtype:
+            raise TypeError(f'Cannot find files on [{",".join(keypath)}], must be a path type')
+
+        paths = param.get(field=None, step=step, index=index)
+
+        is_list = True
+        if not isinstance(paths, list):
+            is_list = False
+            if paths.get():
+                paths = [paths]
+            else:
+                paths = []
+
+        # Ignore collection directory if it does not exist
+        if collection_dir and not os.path.exists(collection_dir):
+            collection_dir = None
+
+        if cwd is None:
+            cwd = os.getcwd()
+
+        if packages is None:
+            packages = {}
+
+        resolved_paths = []
+        for path in paths:
+            search_paths = []
+
+            package = path.get(field="package")
+            if package:
+                if package not in packages:
+                    raise ValueError(f"Resolver for {package} not provided")
+                package_path = packages[package]
+                if isinstance(package_path, str):
+                    search_paths.append(os.path.abspath(package_path))
+                elif callable(package_path):
+                    search_paths.append(package_path())
+                else:
+                    raise TypeError(f"Resolver for {package} is not a recognized type")
+            else:
+                if cwd:
+                    search_paths.append(os.path.abspath(cwd))
+
+            try:
+                resolved_path = path.resolve_path(search=search_paths,
+                                                  collection_dir=collection_dir)
+            except FileNotFoundError:
+                resolved_path = None
+                if not missing_ok:
+                    if package:
+                        raise FileNotFoundError(
+                            f'Could not find "{path.get()}" in {package} [{",".join(keypath)}]')
+                    else:
+                        raise FileNotFoundError(
+                            f'Could not find "{path.get()}" [{",".join(keypath)}]')
+            resolved_paths.append(resolved_path)
+
+        if not is_list:
+            if not resolved_paths:
+                return None
+            return resolved_paths[0]
+        return resolved_paths
+
+    def check_filepaths(self, ignore_keys=None, logger=None,
+                        packages=None, collection_dir=None, cwd=None):
+        '''
+        Verifies that paths to all files in manifest are valid.
+
+        Args:
+            ignore_keys (list of keypaths): list of keypaths to ignore while checking
+            logger (:class:`logging.Logger`): optional logger to use to report errors
+            packages (dict of resolvers): dirctionary of path resolvers for package
+                paths, these can either be a path or a callable function
+            collection_dir (path): optional path to a collections directory
+            cwd (path): optional path to current working directory, this will default
+                to os.getcwd() if not provided.
+
+        Returns:
+            True if all file paths are valid, otherwise False.
+        '''
+
+        if ignore_keys is None:
+            ignore_keys = set()
+        else:
+            ignore_keys = set([
+                tuple(keypath) for keypath in ignore_keys
+            ])
+
+        error = False
+
+        for keypath in self.allkeys():
+            if keypath in ignore_keys:
+                continue
+
+            param = self.get(*keypath, field=None)
+            paramtype = param.get(field='type')
+
+            if 'file' not in paramtype and 'dir' not in paramtype:
+                continue
+
+            for check_files, step, index in param.getvalues():
+                if not check_files:
+                    # nothing set so continue
+                    continue
+
+                found_files = BaseSchema.find_files(
+                    self, *keypath, missing_ok=True, step=step, index=index,
+                    packages=packages, collection_dir=collection_dir, cwd=cwd)
+
+                if not param.is_list():
+                    check_files = [check_files]
+                    found_files = [found_files]
+
+                for check_file, found_file in zip(check_files, found_files):
+                    if not found_file:
+                        error = True
+                        if logger:
+                            node_indicator = ""
+                            if step is not None:
+                                if index is None:
+                                    node_indicator = f" ({step})"
+                                else:
+                                    node_indicator = f" ({step}{index})"
+
+                            logger.error(f"Parameter [{','.join(keypath)}]{node_indicator} path "
+                                         f"{check_file} is invalid")
+
+        return not error
+
+    def _parent(self, root=False):
+        '''
+        Returns the parent of this schema section, if root is true the root parent
+        will be returned.
+
+        Args:
+            root (bool): if true, returns the root of the schemas.
+        '''
+        if not root:
+            return self.__parent
+        if self.__parent is self:
+            return self
+        return self.__parent._parent(root=root)

siliconcompiler/schema/editableschema.py

@@ -29,6 +29,9 @@ class EditableSchema:
             if key in self.__schema._BaseSchema__manifest and not clobber:
                 raise KeyError(f"[{','.join(fullkey)}] is already defined")
 
+            if isinstance(value, BaseSchema):
+                value._BaseSchema__parent = self.__schema
+
             if key == "default":
                 self.__schema._BaseSchema__default = value
             else:
@@ -36,6 +39,7 @@ class EditableSchema:
             return
 
         new_schema = BaseSchema()
+        new_schema._BaseSchema__parent = self.__schema
         if key == "default":
             if self.__schema._BaseSchema__default:
                 new_schema = self.__schema._BaseSchema__default

siliconcompiler/schema/journal.py

@@ -0,0 +1,210 @@
+import copy
+import json
+
+
+class Journal:
+    """
+    This class provides the ability to record the schema transactions:
+    :meth:`BaseSchema.set`, :meth:`BaseSchema.add`, :meth:`BaseSchema.remove`,
+    :meth:`BaseSchema.unset`, and :meth:`BaseSchema.get`.
+
+    Args:
+        keyprefix (list of str): keypath to prefix on to recorded path
+    """
+
+    def __init__(self, keyprefix=None):
+        if not keyprefix:
+            self.__keyprefix = tuple()
+        else:
+            self.__keyprefix = tuple(keyprefix)
+
+        self.__parent = self
+
+        self.__record_types = set()
+        self.stop()
+
+    @property
+    def keypath(self):
+        '''
+        Returns the reference key path for this journal
+        '''
+
+        return self.__keyprefix
+
+    def get_child(self, *keypath):
+        '''
+        Get a child journal based on a new keypath
+
+        Args:
+            keypath (list of str): keypath to prefix on to recorded path
+        '''
+
+        child = Journal(keyprefix=[*self.__keyprefix, *keypath])
+        child.__parent = self.__parent
+        return child
+
+    def from_dict(self, manifest):
+        '''
+        Import a journal from a manifest dictionary
+
+        Args:
+            manifest (dict): Manifest to decode.
+        '''
+
+        self.__journal = manifest
+
+    def get(self):
+        """
+        Returns a copy of the current journal
+        """
+
+        return copy.deepcopy(self.__parent.__journal)
+
+    def has_journaling(self):
+        """
+        Returns true if the schema is currently setup and is the root of the journal and has data
+        """
+        return self is self.__parent and bool(self.__journal)
+
+    def is_journaling(self):
+        """
+        Returns true if the schema is currently setup for journaling
+        """
+        return self.__parent.__journal is not None
+
+    def get_types(self):
+        """
+        Returns the current schema accesses that are being recorded
+        """
+
+        return self.__parent.__record_types.copy()
+
+    def add_type(self, value):
+        """
+        Adds a new access type to the journal record.
+
+        Args:
+            value (str): access type
+        """
+
+        if value not in ("set", "add", "remove", "unset", "get"):
+            raise ValueError(f"{value} is not a valid type")
+
+        return self.__parent.__record_types.add(value)
+
+    def remove_type(self, value):
+        """
+        Removes a new access type to the journal record.
+
+        Args:
+            value (str): access type
+        """
+
+        try:
+            self.__parent.__record_types.remove(value)
+        except KeyError:
+            pass
+
+    def record(self, record_type, key, value=None, field=None, step=None, index=None):
+        '''
+        Record the schema transaction
+        '''
+
+        if self.__parent.__journal is None:
+            return
+
+        if record_type not in self.__parent.__record_types:
+            return
+
+        self.__parent.__journal.append({
+            "type": record_type,
+            "key": tuple([*self.__keyprefix, *key]),
+            "value": value,
+            "field": field,
+            "step": step,
+            "index": index
+        })
+
+    def start(self):
+        '''
+        Start journaling the schema transactions
+        '''
+        self.__parent.__journal = []
+        self.add_type("set")
+        self.add_type("add")
+        self.add_type("remove")
+        self.add_type("unset")
+
+    def stop(self):
+        '''
+        Stop journaling the schema transactions
+        '''
+        self.__parent.__journal = None
+        self.__parent.__record_types.clear()
+
+    @staticmethod
+    def replay_file(schema, filepath):
+        '''
+        Replay a journal into a schema from a manifest
+
+        Args:
+            schema (:class:`BaseSchema`): schema to replay transactions to
+            filepath (path): path to manifest
+        '''
+        with open(filepath, "r") as fid:
+            data = json.load(fid)
+        if "__journal__" not in data:
+            return
+
+        journal = Journal()
+        journal.from_dict(data["__journal__"])
+        journal.replay(schema)
+
+    def replay(self, schema):
+        '''
+        Replay journal into a schema
+
+        Args:
+            schema (:class:`BaseSchema`): schema to replay transactions to
+        '''
+
+        from .baseschema import BaseSchema
+        if not isinstance(schema, BaseSchema):
+            raise TypeError(f"schema must be a BaseSchema, not {type(schema)}")
+
+        if not self.__parent.__journal:
+            return
+
+        for action in self.__parent.__journal:
+            record_type = action['type']
+            keypath = action['key']
+            value = action['value']
+            field = action['field']
+            step = action['step']
+            index = action['index']
+            if record_type == 'set':
+                schema.set(*keypath, value, field=field, step=step, index=index)
+            elif record_type == 'add':
+                schema.add(*keypath, value, field=field, step=step, index=index)
+            elif record_type == 'unset':
+                schema.unset(*keypath, step=step, index=index)
+            elif record_type == 'remove':
+                schema.remove(*keypath)
+            elif record_type == 'get':
+                continue
+            else:
+                raise ValueError(f'Unknown record type {record_type}')
+
+    @staticmethod
+    def access(schema):
+        '''
+        Access a journal from a schema
+
+        Args:
+            schema (:class:`BaseSchema`): schema to replay transactions to access journal
+        '''
+        from .baseschema import BaseSchema
+        if isinstance(schema, BaseSchema):
+            return schema._BaseSchema__journal
+
+        raise TypeError(f"schema must be a BaseSchema, not {type(schema)}")

siliconcompiler/schema/namedschema.py

@@ -15,7 +15,7 @@ class NamedSchema(BaseSchema):
         name (str): name of the schema
     '''
 
-    def __init__(self, name=None):
+    def __init__(self, name):
         super().__init__()
 
         self.__name = name
@@ -26,6 +26,35 @@ class NamedSchema(BaseSchema):
         '''
         return self.__name
 
+    def _reset(self) -> None:
+        """
+        Resets the state of the object
+        """
+        pass
+
+    @classmethod
+    def from_manifest(cls, name, filepath=None, cfg=None):
+        '''
+        Create a new schema based on the provided source files.
+
+        The two arguments to this method are mutually exclusive.
+
+        Args:
+            name (str): name of the schema
+            filepath (path): Initial manifest.
+            cfg (dict): Initial configuration dictionary.
+        '''
+
+        if not filepath and cfg is None:
+            raise RuntimeError("filepath or dictionary is required")
+
+        schema = cls(name)
+        if filepath:
+            schema.read_manifest(filepath)
+        if cfg:
+            schema._from_dict(cfg, [])
+        return schema
+
     def _from_dict(self, manifest, keypath, version=None):
         if keypath:
             self.__name = keypath[-1]
@@ -35,7 +64,7 @@ class NamedSchema(BaseSchema):
     def copy(self, key=None):
         copy = super().copy(key=key)
 
-        if not self.__name and key and key[-1] != "default":
+        if key and key[-1] != "default":
             copy.__name = key[-1]
 
         return copy

siliconcompiler/schema/parameter.py

@@ -409,12 +409,13 @@ class Parameter:
 
         return True
 
-    def getdict(self, include_default=True):
+    def getdict(self, include_default=True, values_only=False):
         """
         Returns a schema dictionary.
 
         Args:
             include_default (boolean): If true will include default values
+            values_only (boolean): If true will only return values
 
         Returns:
             A schema dictionary
@@ -424,6 +425,18 @@ class Parameter:
             Returns the complete dictionary for the parameter
         """
 
+        if values_only:
+            dictvals = {}
+            is_list = self.is_list()
+            for value, step, index in self.getvalues(return_defvalue=include_default):
+                if is_list:
+                    if value:
+                        dictvals.setdefault(step, {})[index] = value
+                else:
+                    if value is not None:
+                        dictvals.setdefault(step, {})[index] = value
+            return dictvals
+
         dictvals = {
             "type": NodeType.encode(self.__type),
             "require": self.__require,