siliconcompiler 0.34.0__py3-none-any.whl → 0.34.2__py3-none-any.whl

siliconcompiler/package/__init__.py

@@ -1,12 +1,15 @@
 import contextlib
 import functools
+import hashlib
 import importlib
 import json
 import logging
 import os
+import random
 import re
 import time
 import threading
+import uuid
 
 import os.path
 
@@ -50,13 +53,16 @@ class Resolver:
     _RESOLVERS_LOCK = threading.Lock()
     _RESOLVERS = {}
 
+    __CACHE_LOCK = threading.Lock()
+    __CACHE = {}
+
     def __init__(self, name, root, source, reference=None):
         self.__name = name
         self.__root = root
         self.__source = source
         self.__reference = reference
         self.__changed = False
-        self.__cache = {}
+        self.__cacheid = None
 
         if self.__root and hasattr(self.__root, "logger"):
             self.__logger = self.__root.logger.getChild(f"resolver-{self.name}")
@@ -126,34 +132,80 @@ class Resolver:
         return self.urlparse.netloc
 
     @property
-    def changed(self):
+    def changed(self) -> bool:
         change = self.__changed
         self.__changed = False
         return change
 
+    @property
+    def cache_id(self) -> str:
+        if self.__cacheid is None:
+            hash = hashlib.sha1()
+            hash.update(self.__source.encode())
+            if self.__reference:
+                hash.update(self.__reference.encode())
+            else:
+                hash.update("".encode())
+
+            self.__cacheid = hash.hexdigest()
+        return self.__cacheid
+
     def set_changed(self):
         self.__changed = True
 
-    def set_cache(self, cache):
-        self.__cache = cache
-
     def resolve(self):
         raise NotImplementedError("child class must implement this")
 
+    @staticmethod
+    def __get_root_id(root):
+        STORAGE = "__Resolver_cache_id"
+        if not getattr(root, STORAGE, None):
+            setattr(root, STORAGE, uuid.uuid4().hex)
+        return getattr(root, STORAGE)
+
+    @staticmethod
+    def get_cache(root, name: str = None):
+        with Resolver.__CACHE_LOCK:
+            root_id = Resolver.__get_root_id(root)
+            if root_id not in Resolver.__CACHE:
+                Resolver.__CACHE[root_id] = {}
+
+            if name:
+                return Resolver.__CACHE[root_id].get(name, None)
+
+            return Resolver.__CACHE[root_id].copy()
+
+    @staticmethod
+    def set_cache(root, name: str, path: str):
+        with Resolver.__CACHE_LOCK:
+            root_id = Resolver.__get_root_id(root)
+            if root_id not in Resolver.__CACHE:
+                Resolver.__CACHE[root_id] = {}
+            Resolver.__CACHE[root_id][name] = path
+
+    @staticmethod
+    def reset_cache(root):
+        with Resolver.__CACHE_LOCK:
+            root_id = Resolver.__get_root_id(root)
+            if root_id in Resolver.__CACHE:
+                del Resolver.__CACHE[root_id]
+
     def get_path(self):
-        if self.name in self.__cache:
-            return self.__cache[self.name]
+        cache_path = Resolver.get_cache(self.__root, self.cache_id)
+        if cache_path:
+            return cache_path
 
         path = self.resolve()
         if not os.path.exists(path):
             raise FileNotFoundError(f"Unable to locate {self.name} at {path}")
 
-        if self.changed and self.name not in self.__cache:
+        if self.changed:
             self.logger.info(f'Saved {self.name} data to {path}')
         else:
             self.logger.info(f'Found {self.name} data at {path}')
-        self.__cache[self.name] = path
-        return self.__cache[self.name]
+
+        Resolver.set_cache(self.__root, self.cache_id, path)
+        return path
 
     def __resolve_env(self, path):
         env_save = os.environ.copy()
@@ -198,6 +250,7 @@ class RemoteResolver(Resolver):
         if not root:
             return Path(default_path)
 
+        path = None
         if root.valid('option', 'cachedir'):
             path = root.get('option', 'cachedir')
             if path:
@@ -249,32 +302,51 @@ class RemoteResolver(Resolver):
             return RemoteResolver._CACHE_LOCKS[self.name]
 
     @contextlib.contextmanager
-    def lock(self):
+    def __thread_lock(self):
         lock = self.thread_lock()
         lock_acquired = False
         try:
-            if lock.acquire_lock(timeout=self.timeout):
-                data_path_lock = InterProcessLock(self.lock_file)
-                sc_data_path_lock = None
-                try:
-                    lock_acquired = data_path_lock.acquire(timeout=self.timeout)
-                except (OSError, RuntimeError):
-                    if not lock_acquired:
-                        sc_data_path_lock = Path(self.sc_lock_file)
-                        max_seconds = self.timeout
-                        while sc_data_path_lock.exists():
-                            if max_seconds == 0:
-                                raise RuntimeError(f'Failed to access {self.cache_path}. '
-                                                   f'Lock {sc_data_path_lock} still exists.')
-                            time.sleep(1)
-                            max_seconds -= 1
-                        sc_data_path_lock.touch()
-                        lock_acquired = True
-                if lock_acquired:
-                    yield
+            timeout = self.timeout
+            while timeout > 0:
+                if lock.acquire_lock(timeout=1):
+                    lock_acquired = True
+                    break
+                sleep_time = random.randint(1, max(1, int(timeout / 10)))
+                timeout -= sleep_time + 1
+                time.sleep(sleep_time)
+            if lock_acquired:
+                yield
         finally:
             if lock.locked():
                 lock.release()
+
+        if not lock_acquired:
+            raise RuntimeError(f'Failed to access {self.cache_path}. '
+                               f'Another thread is currently holding the lock.')
+
+    @contextlib.contextmanager
+    def __file_lock(self):
+        data_path_lock = InterProcessLock(self.lock_file)
+        lock_acquired = False
+        sc_data_path_lock = None
+        try:
+            try:
+                lock_acquired = data_path_lock.acquire(timeout=self.timeout)
+            except (OSError, RuntimeError):
+                if not lock_acquired:
+                    sc_data_path_lock = Path(self.sc_lock_file)
+                    max_seconds = self.timeout
+                    while sc_data_path_lock.exists():
+                        if max_seconds == 0:
+                            raise RuntimeError(f'Failed to access {self.cache_path}. '
+                                               f'Lock {sc_data_path_lock} still exists.')
+                        time.sleep(1)
+                        max_seconds -= 1
+                    sc_data_path_lock.touch()
+                    lock_acquired = True
+            if lock_acquired:
+                yield
+        finally:
             if lock_acquired:
                 if data_path_lock.acquired:
                     data_path_lock.release()
@@ -286,6 +358,12 @@ class RemoteResolver(Resolver):
                                f'{self.lock_file} is still locked, if this is a mistake, '
                                'please delete it.')
 
+    @contextlib.contextmanager
+    def lock(self):
+        with self.__thread_lock():
+            with self.__file_lock():
+                yield
+
     def resolve_remote(self):
         raise NotImplementedError("child class must implement this")
 
@@ -324,11 +402,8 @@ class FileResolver(Resolver):
 
     @property
     def urlpath(self):
-        parse = self.urlparse
-        if parse.netloc:
-            return parse.netloc
-        else:
-            return parse.path
+        # Rebuild URL and remove scheme prefix
+        return self.urlparse.geturl()[7:]
 
     def resolve(self):
         return os.path.abspath(self.urlpath)
@@ -412,6 +487,34 @@ class PythonPathResolver(Resolver):
                              path=path,
                              ref=ref)
 
+    @staticmethod
+    def set_dataroot(root,
+                     package_name,
+                     python_module,
+                     alternative_path,
+                     alternative_ref=None,
+                     python_module_path_append=None):
+        '''
+        Helper function to register a python module as data source with an alternative in case
+        the module is not installed in an editable state
+        '''
+        # check if installed in an editable state
+        if PythonPathResolver.is_python_module_editable(python_module):
+            if python_module_path_append:
+                path = PythonPathResolver(
+                    python_module, root, f"python://{python_module}").resolve()
+                path = os.path.abspath(os.path.join(path, python_module_path_append))
+            else:
+                path = f"python://{python_module}"
+            ref = None
+        else:
+            path = alternative_path
+            ref = alternative_ref
+
+        root.set_dataroot(name=package_name,
+                          path=path,
+                          tag=ref)
+
     def resolve(self):
         module = importlib.import_module(self.urlpath)
         python_path = os.path.dirname(module.__file__)

siliconcompiler/package/github.py

@@ -53,17 +53,13 @@ class GithubResolver(HTTPResolver):
         if not release:
             release = repo.get_latest_release().tag_name
 
-        url = None
-        for repo_release in repo.get_releases():
-            if repo_release.tag_name == release:
-                for asset in repo_release.assets:
-                    if asset.name == artifact:
-                        url = asset.url
+        repo_release = repo.get_release(release)
+        if repo_release:
+            for asset in repo_release.assets:
+                if asset.name == artifact:
+                    return asset.url
 
-        if not url:
-            raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
-
-        return url
+        raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
 
     def __get_gh_auth(self):
         token_name = self.name.upper()

siliconcompiler/packageschema.py

@@ -2,19 +2,270 @@ import os
 
 import os.path
 
-from siliconcompiler.schema import BaseSchema
-from siliconcompiler.schema import EditableSchema, Parameter, Scope
+from typing import Union, Dict, List
+
+from siliconcompiler.pathschema import PathSchema
+from siliconcompiler.schema import BaseSchema, EditableSchema, Parameter, Scope
 from siliconcompiler.schema.utils import trim
 from siliconcompiler.package import Resolver
 
 
-class PackageSchema(BaseSchema):
+class PackageSchema(PathSchema):
     def __init__(self):
         super().__init__()
 
         schema_package(self)
 
-        self.__cache = {}
+    def set_description(self, desc: str):
+        """
+        Set the description of the package.
+
+        Args:
+            desc (str): The description string.
+        """
+        return self.set("package", "description", trim(desc))
+
+    def get_description(self) -> str:
+        """Get the description of the package.
+
+        Returns:
+            str: The description string.
+        """
+        return self.get("package", "description")
+
+    def set_version(self, version: str):
+        """
+        Set the version of the package.
+
+        Args:
+            version (str): The version string.
+        """
+        return self.set("package", "version", version)
+
+    def get_version(self) -> str:
+        """
+        Get the version of the package.
+
+        Returns:
+            str: The version string.
+        """
+        return self.get("package", "version")
+
+    def add_license(self, name: str):
+        """
+        Add a license name to the package.
+
+        Args:
+            name (str): The name of the license.
+        """
+        return self.add("package", "license", name)
+
+    def add_licensefile(self, file: str, dataroot: str = None):
+        """
+        Add a license file to the package.
+
+        Args:
+            file (str): The path to the license file.
+            dataroot (str, optional): The data reference for the package. Defaults to None,
+                                    which uses the active package.
+        """
+        if not dataroot:
+            dataroot = self._get_active("package")
+        with self.active_dataroot(dataroot):
+            return self.add("package", "licensefile", file)
+
+    def get_license(self) -> List[str]:
+        """
+        Get a list of license names associated with the package.
+        """
+        return self.get("package", "license")
+
+    def get_licensefile(self) -> List[str]:
+        """
+        Get a list of license file paths associated with the package.
+        """
+        return self.find_files("package", "licensefile")
+
+    def add_author(self,
+                   identifier: str,
+                   name: str = None,
+                   email: str = None,
+                   organization: str = None):
+        """
+        Add or update author information for the package.
+
+        Args:
+            identifier (str): A unique identifier for the author.
+            name (str, optional): The author's name. Defaults to None.
+            email (str, optional): The author's email address. Defaults to None.
+            organization (str, optional): The author's organization. Defaults to None.
+        """
+        params = []
+        if name:
+            params.append(self.set("package", "author", identifier, "name", name))
+        if email:
+            params.append(self.set("package", "author", identifier, "email", email))
+        if organization:
+            params.append(self.set("package", "author", identifier, "organization", organization))
+        return [p for p in params if p]
+
+    def get_author(self, identifier: str = None):
+        """
+        Returns the author information for a specific author or all authors.
+
+        Args:
+            identifier (str): A unique identifier for the author, if None returns all
+        """
+        if identifier is None:
+            authors = []
+            for author in self.getkeys("package", "author"):
+                authors.append(self.get_author(author))
+            return authors
+        return {
+            "name": self.get("package", "author", identifier, "name"),
+            "email": self.get("package", "author", identifier, "email"),
+            "organization": self.get("package", "author", identifier, "organization")
+        }
+
+    def add_doc(self, type: str, path: str, dataroot: str = None):
+        """
+        Add documentation to the package.
+
+        Args:
+            type (str): The type of documentation (e.g., "manual", "api").
+            path (str): The path to the documentation file.
+            dataroot (str, optional): The data reference for the package. Defaults to None,
+                                    which uses the active package.
+
+        Returns:
+            The result of the `add` operation.
+        """
+        if not dataroot:
+            dataroot = self._get_active("package")
+        with self.active_dataroot(dataroot):
+            return self.add("package", "doc", type, path)
+
+    def get_doc(self, type: str = None) -> Union[List[str], Dict[str, List[str]]]:
+        """
+        Get documentation files for the package.
+
+        Args:
+            type (str, optional): The type of documentation to retrieve. If None,
+                                returns all documentation organized by type. Defaults to None.
+        """
+        if type:
+            return self.find_files("package", "doc", type)
+
+        docs = {}
+        for type in self.getkeys("package", "doc"):
+            doc_files = self.find_files("package", "doc", type)
+            if doc_files:
+                docs[type] = doc_files
+        return docs
+
+
+############################################
+# Package information
+############################################
+def schema_package(schema):
+    schema = EditableSchema(schema)
+
+    schema.insert(
+        'package', 'version',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: version",
+            switch="-package_version <str>",
+            example=[
+                "cli: -package_version 1.0",
+                "api: chip.set('package', 'version', '1.0')"],
+            help=trim("""Package version. Can be a branch, tag, commit hash,
+            or a semver compatible version.""")))
+
+    schema.insert(
+        'package', 'description',
+        Parameter(
+            'str',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: description",
+            switch="-package_description <str>",
+            example=[
+                "cli: -package_description 'Yet another cpu'",
+                "api: chip.set('package', 'description', 'Yet another cpu')"],
+            help=trim("""Package short one line description for package
+            managers and summary reports.""")))
+
+    for item in [
+            'datasheet',
+            'reference',
+            'userguide',
+            'quickstart',
+            'releasenotes',
+            'testplan',
+            'signoff',
+            'tutorial']:
+        schema.insert(
+            'package', 'doc', item,
+            Parameter(
+                '[file]',
+                scope=Scope.GLOBAL,
+                shorthelp=f"Package: {item} document",
+                switch=f"-package_doc_{item} <file>",
+                example=[
+                    f"cli: -package_doc_{item} {item}.pdf",
+                    f"api: chip.set('package', 'doc', '{item}', '{item}.pdf')"],
+                help=trim(f"""Package list of {item} documents.""")))
+
+    schema.insert(
+        'package', 'license',
+        Parameter(
+            '[str]',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: license identifiers",
+            switch="-package_license <str>",
+            example=[
+                "cli: -package_license 'Apache-2.0'",
+                "api: chip.set('package', 'license', 'Apache-2.0')"],
+            help=trim("""Package list of SPDX license identifiers.""")))
+
+    schema.insert(
+        'package', 'licensefile',
+        Parameter(
+            '[file]',
+            scope=Scope.GLOBAL,
+            shorthelp="Package: license files",
+            switch="-package_licensefile <file>",
+            example=[
+                "cli: -package_licensefile './LICENSE'",
+                "api: chip.set('package', 'licensefile', './LICENSE')"],
+            help=trim("""Package list of license files for to be
+            applied in cases when a SPDX identifier is not available.
+            (eg. proprietary licenses).""")))
+
+    for item in [
+            'name',
+            'email',
+            'organization']:
+        schema.insert(
+            'package', 'author', 'default', item,
+            Parameter(
+                'str',
+                scope=Scope.GLOBAL,
+                shorthelp=f"Package: author {item}",
+                switch=f"-package_author_{item} 'userid <str>'",
+                example=[
+                    f"cli: -package_author_{item} 'wiley wiley@acme.com'",
+                    f"api: chip.set('package', 'author', 'wiley', '{item}', 'wiley@acme.com')"],
+                help=trim(f"""Package author {item} provided with full name as key and
+                {item} as value.""")))
+
+
+class PackageSchemaTmp(BaseSchema):
+    def __init__(self):
+        super().__init__()
+
+        schema_package_tmp(self)
 
     def register(self, name, path, ref=None, clobber=True):
         """
@@ -60,7 +311,6 @@ class PackageSchema(BaseSchema):
         resolver = resolver_cls(package, self._parent(root=True),
                                 self.get("source", package, "path"),
                                 self.get("source", package, "ref"))
-        resolver.set_cache(self.__cache)
         return resolver
 
     def get_resolvers(self):
@@ -73,17 +323,11 @@ class PackageSchema(BaseSchema):
 
         return resolvers
 
-    def _set_cache(self, package, path):
-        self.__cache[package] = path
-
-    def get_path_cache(self):
-        return self.__cache.copy()
-
 
 ############################################
 # Package information
 ############################################
-def schema_package(schema):
+def schema_package_tmp(schema):
     schema = EditableSchema(schema)
 
     schema.insert(