siliconcompiler 0.34.1__py3-none-any.whl → 0.34.3__py3-none-any.whl

siliconcompiler/package/git.py

@@ -1,5 +1,11 @@
-import shutil
+"""
+This module provides a Git-based resolver for SiliconCompiler packages.
 
+It defines the `GitResolver` class, which is responsible for cloning remote
+Git repositories into a local cache, checking out specific references (like
+branches, tags, or commit hashes), and managing the cached repository's state.
+"""
+import shutil
 import os.path
 
 from git import Repo, GitCommandError
@@ -7,6 +13,15 @@ from siliconcompiler.package import RemoteResolver
 
 
 def get_resolver():
+    """
+    Returns a dictionary mapping Git-related URI schemes to the GitResolver class.
+
+    This function is used by the resolver system to discover and register this
+    resolver for handling git, git+https, git+ssh, and ssh protocols.
+
+    Returns:
+        dict: A dictionary mapping scheme names to the GitResolver class.
+    """
     return {
         "git": GitResolver,
         "git+https": GitResolver,
@@ -16,10 +31,32 @@ def get_resolver():
 
 
 class GitResolver(RemoteResolver):
+    """
+    A resolver for fetching data from remote Git repositories.
+
+    This class handles cloning repositories, checking out specific references,
+    and managing the local cache. It supports authentication via environment
+    tokens (e.g., GITHUB_TOKEN) for HTTPS and assumes SSH keys are configured
+    for SSH-based URLs.
+    """
+
     def __init__(self, name, root, source, reference=None):
+        """
+        Initializes the GitResolver.
+        """
         super().__init__(name, root, source, reference)
 
     def check_cache(self):
+        """
+        Checks if a valid, clean Git repository exists at the cache path.
+
+        This method verifies that the path points to a valid Git repository
+        and warns the user if the repository is "dirty" (has untracked files
+        or uncommitted changes). If the repository is corrupted, it is removed.
+
+        Returns:
+            bool: True if a valid repository exists, False otherwise.
+        """
         if os.path.exists(self.cache_path):
             try:
                 repo = Repo(self.cache_path)
@@ -33,9 +70,21 @@ class GitResolver(RemoteResolver):
         return False
 
     def __get_token_env(self):
+        """
+        Searches for a Git authentication token in predefined environment variables.
+
+        The search order is:
+        1. GITHUB_<PACKAGE_NAME>_TOKEN (e.g., GITHUB_SKYWATER130_TOKEN)
+        2. GITHUB_TOKEN
+        3. GIT_TOKEN
+
+        Returns:
+            str or None: The found token, or None if no token is set.
+        """
         token_name = self.name.upper()
-        for tok in ('#', '$', '&', '-', '=', '!', '/'):
-            token_name = token_name.replace(tok, '')
+        # Sanitize package name for environment variable compatibility
+        for char in ('#', '$', '&', '-', '=', '!', '/'):
+            token_name = token_name.replace(char, '')
 
         search_env = (
             f'GITHUB_{token_name}_TOKEN',
@@ -43,40 +92,67 @@ class GitResolver(RemoteResolver):
             'GIT_TOKEN'
         )
 
-        token = None
         for env in search_env:
-            token = os.environ.get(env, None)
-
+            token = os.environ.get(env)
             if token:
                 return token
         return None
 
     @property
     def git_path(self):
-        if self.urlscheme == "git+ssh":
-            return f"ssh://{self.urlpath}{self.urlparse.path}"
-        if self.urlscheme == "ssh":
-            return self.source
+        """
+        Constructs the final Git URL for cloning.
+
+        This method handles different URL schemes and automatically injects
+        an authentication token into HTTPS URLs if a token is found in the
+        environment.
+
+        Returns:
+            str: The fully-formed URL ready for `git clone`.
+        """
+        if self.urlscheme == "git+ssh" or self.urlscheme == "ssh":
+            # Reconstruct the original SSH URL
+            return self.source.replace('git+', '')
+
+        # For HTTPS, inject token if available
         url = self.urlparse
-        if not url.username and self.__get_token_env():
-            url = url._replace(netloc=f'{self.__get_token_env()}@{url.hostname}')
+        token = self.__get_token_env()
+        if not url.username and token:
+            url = url._replace(netloc=f'{token}@{url.hostname}')
+        # Ensure the scheme is HTTPS
         url = url._replace(scheme='https')
         return url.geturl()
 
     def resolve_remote(self):
+        """
+        Fetches the remote repository and checks out the specified reference.
+
+        This method performs the `git clone` operation, followed by `git checkout`
+        on the specified branch, tag, or commit. It also initializes all submodules.
+
+        Raises:
+            RuntimeError: If authentication fails.
+            GitCommandError: For other Git-related errors.
+        """
         try:
             path = self.git_path
             self.logger.info(f'Cloning {self.name} data from {path}')
             repo = Repo.clone_from(path, self.cache_path, recurse_submodules=True)
+
             self.logger.info(f'Checking out {self.reference}')
             repo.git.checkout(self.reference)
+
+            self.logger.info('Updating submodules')
             for submodule in repo.submodules:
                 submodule.update(recursive=True, init=True, force=True)
         except GitCommandError as e:
-            if 'Permission denied' in repr(e):
+            if 'Permission denied' in repr(e) or 'could not read Username' in repr(e):
                 if self.urlscheme in ('ssh', 'git+ssh'):
-                    raise RuntimeError('Failed to authenticate. Please setup your git ssh.')
-                elif self.urlscheme in ('git', 'git+https'):
-                    raise RuntimeError('Failed to authenticate. Please use a token or ssh.')
+                    raise RuntimeError('Failed to authenticate with Git. Please ensure your SSH '
+                                       'keys are set up correctly.')
+                else:  # 'git', 'git+https'
+                    raise RuntimeError('Failed to authenticate with Git. Please provide a token '
+                                       'via GITHUB_TOKEN or use an SSH URL.')
             else:
+                # Re-raise other Git errors
                 raise e

siliconcompiler/package/github.py

@@ -1,3 +1,9 @@
+"""
+This module provides a GitHub-based resolver for SiliconCompiler packages.
+
+It defines the `GithubResolver` class, which is responsible for downloading
+release assets from public or private GitHub repositories.
+"""
 import os
 
 from github import Github, Auth
@@ -7,6 +13,15 @@ from siliconcompiler.package.https import HTTPResolver
 
 
 def get_resolver():
+    """
+    Returns a dictionary mapping GitHub URI schemes to the GithubResolver class.
+
+    This function is used by the resolver system to discover and register this
+    resolver for handling `github` and `github+private` protocols.
+
+    Returns:
+        dict: A dictionary mapping scheme names to the GithubResolver class.
+    """
     return {
         "github": GithubResolver,
         "github+private": GithubResolver
@@ -14,61 +29,129 @@ def get_resolver():
 
 
 class GithubResolver(HTTPResolver):
+    """
+    A resolver for fetching release assets from GitHub repositories.
+
+    This class extends the `HTTPResolver` to interact with the GitHub API
+    for locating and downloading release assets. It supports both public
+    and private repositories.
+
+    The expected source URI format is:
+    `github://<owner>/<repository>/<release_tag>/<asset_name>`
+
+    For private repositories, the scheme should be `github+private://` and
+    a GitHub token must be provided via environment variables.
+    """
+
     def __init__(self, name, root, source, reference=None):
+        """
+        Initializes the GithubResolver.
+        """
         super().__init__(name, root, source, reference)
 
         if len(self.gh_path) != 4:
             raise ValueError(
-                f"{self.source} is not in the proper form: "
-                "<owner>/<repository>/<version>/<artifact>")
+                f"'{self.source}' is not in the proper form: "
+                "github://<owner>/<repository>/<version>/<artifact>")
 
     @property
     def gh_path(self):
+        """
+        Parses the source URL into its constituent GitHub parts.
+
+        Returns:
+            tuple: A tuple containing (owner, repository, release_tag, asset_name).
+        """
         return self.urlpath, *self.urlparse.path.split("/")[1:]
 
     @property
     def download_url(self):
-        url_parts = self.gh_path
+        """
+        Determines the direct download URL for the GitHub release asset.
 
+        This method first attempts to find the asset in a public repository.
+        If that fails (e.g., with an `UnknownObjectException`), it then tries
+        to find it in a private repository, which requires authentication.
+        The `github+private` scheme forces an authenticated private lookup directly.
+
+        Returns:
+            str: The direct URL to download the asset.
+        """
+        url_parts = self.gh_path
         repository = "/".join(url_parts[0:2])
         release = url_parts[2]
         artifact = url_parts[3]
 
         if self.urlscheme == "github+private":
-            return self.__get_release_url(repository, release, artifact, True)
+            return self.__get_release_url(repository, release, artifact, private=True)
 
         try:
-            return self.__get_release_url(repository, release, artifact, False)
+            # First, try as a public repository
+            return self.__get_release_url(repository, release, artifact, private=False)
         except UnknownObjectException:
-            return self.__get_release_url(repository, release, artifact, True)
+            # If public access fails, try as a private repository
+            self.logger.info("Could not find public release, trying private.")
+            return self.__get_release_url(repository, release, artifact, private=True)
 
     def __get_release_url(self, repository, release, artifact, private: bool):
+        """
+        Uses the GitHub API to find the download URL for a specific release asset.
+
+        Also handles special cases for downloading source code archives (`.zip`
+        or `.tar.gz`).
+
+        Args:
+            repository (str): The repository name in 'owner/repo' format.
+            release (str): The release tag (e.g., 'v1.0.0').
+            artifact (str): The filename of the asset to download.
+            private (bool): If True, use an authenticated API client.
+
+        Returns:
+            str: The direct download URL for the asset.
+
+        Raises:
+            ValueError: If the specified release or asset cannot be found.
+        """
+        # Handle standard source code archive names
         if artifact == f"{release}.zip":
             return f"https://github.com/{repository}/archive/refs/tags/{release}.zip"
         if artifact == f"{release}.tar.gz":
             return f"https://github.com/{repository}/archive/refs/tags/{release}.tar.gz"
 
+        # Use the GitHub API for other assets
         repo = self.__gh(private).get_repo(repository)
 
         if not release:
             release = repo.get_latest_release().tag_name
+            self.logger.info(f"No release specified, using latest: {release}")
 
-        url = None
-        for repo_release in repo.get_releases():
-            if repo_release.tag_name == release:
-                for asset in repo_release.assets:
-                    if asset.name == artifact:
-                        url = asset.url
-
-        if not url:
-            raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
+        repo_release = repo.get_release(release)
+        if repo_release:
+            for asset in repo_release.assets:
+                if asset.name == artifact:
+                    return asset.url
 
-        return url
+        raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
 
     def __get_gh_auth(self):
+        """
+        Searches for a GitHub authentication token in predefined environment variables.
+
+        The search order is:
+        1. GITHUB_<PACKAGE_NAME>_TOKEN
+        2. GITHUB_TOKEN
+        3. GIT_TOKEN
+
+        Returns:
+            str: The found token.
+
+        Raises:
+            ValueError: If no token can be found in the environment.
+        """
         token_name = self.name.upper()
-        for tok in ('#', '$', '&', '-', '=', '!', '/'):
-            token_name = token_name.replace(tok, '')
+        # Sanitize package name for environment variable compatibility
+        for char in ('#', '$', '&', '-', '=', '!', '/'):
+            token_name = token_name.replace(char, '')
 
         search_env = (
             f'GITHUB_{token_name}_TOKEN',
@@ -78,19 +161,28 @@ class GithubResolver(HTTPResolver):
 
         token = None
         for env in search_env:
-            token = os.environ.get(env, None)
-
+            token = os.environ.get(env)
             if token:
                 break
 
         if not token:
-            raise ValueError('Unable to determine authorization token for GitHub, '
-                             'please set one of the following environmental variables: '
+            raise ValueError('Unable to determine authorization token for GitHub. '
+                             'Please set one of the following environment variables: '
                              f'{", ".join(search_env)}')
 
         return token
 
     def __gh(self, private: bool) -> Github:
+        """
+        Initializes the PyGithub client.
+
+        Args:
+            private (bool): If True, initializes the client with an authentication
+                token. Otherwise, initializes an unauthenticated client.
+
+        Returns:
+            Github: An initialized PyGithub client instance.
+        """
         if private:
             return Github(auth=Auth.Token(self.__get_gh_auth()))
         else:

siliconcompiler/package/https.py

@@ -1,3 +1,9 @@
+"""
+This module provides a generic HTTP/HTTPS resolver for SiliconCompiler packages.
+
+It defines the `HTTPResolver` class, which is responsible for downloading
+and unpacking archives (TAR or ZIP) from a given URL.
+"""
 import requests
 import shutil
 import tarfile
@@ -12,6 +18,15 @@ from siliconcompiler.package import RemoteResolver
 
 
 def get_resolver():
+    """
+    Returns a dictionary mapping HTTP schemes to the HTTPResolver class.
+
+    This function is used by the resolver system to discover and register this
+    resolver for handling `http` and `https` protocols.
+
+    Returns:
+        dict: A dictionary mapping scheme names to the HTTPResolver class.
+    """
     return {
         "http": HTTPResolver,
         "https": HTTPResolver
@@ -19,23 +34,62 @@ def get_resolver():
 
 
 class HTTPResolver(RemoteResolver):
+    """
+    A resolver for fetching and unpacking data from HTTP/HTTPS URLs.
+
+    This class downloads a file from a URL, automatically determines if it's a
+    gzipped tarball or a zip file, and extracts its contents into the local
+    cache. It also includes special handling to flatten the directory structure
+    of archives downloaded from GitHub.
+    """
+
     def check_cache(self):
+        """
+        Checks if the data has already been cached.
+
+        For this resolver, the cache is considered valid if the target cache
+        directory simply exists.
+
+        Returns:
+            bool: True if the cache path exists, False otherwise.
+        """
         return os.path.exists(self.cache_path)
 
     @property
     def download_url(self):
+        """
+        Constructs the final download URL.
+
+        If the source URL ends with a '/', it appends the reference
+        (e.g., version) and a `.tar.gz` extension.
+
+        Returns:
+            str: The fully-formed URL to download from.
+        """
         data_url = self.source
         if data_url.endswith('/'):
             data_url = f"{data_url}{self.reference}.tar.gz"
         return data_url
 
     def resolve_remote(self):
+        """
+        Fetches the remote archive, unpacks it, and stores it in the cache.
+
+        This method downloads the file, detects the archive type (tar.gz or zip),
+        and extracts it. It includes special logic to handle the extra top-level
+        directory that GitHub often includes in its source archives.
+
+        Raises:
+            FileNotFoundError: If the download fails (e.g., 404 error).
+        """
         data_url = self.download_url
 
         headers = {}
+        # Use GIT_TOKEN for authentication if available, primarily for GitHub raw downloads.
         auth_token = os.environ.get('GIT_TOKEN', self.urlparse.username)
         if auth_token:
             headers['Authorization'] = f'token {auth_token}'
+        # GitHub release assets require a specific Accept header.
         if "github" in data_url:
             headers['Accept'] = 'application/octet-stream'
 
@@ -43,25 +97,33 @@ class HTTPResolver(RemoteResolver):
 
         response = requests.get(data_url, stream=True, headers=headers)
         if not response.ok:
-            raise FileNotFoundError(f'Failed to download {self.name} data source.')
+            raise FileNotFoundError(f'Failed to download {self.name} data source from {data_url}. '
+                                    f'Status code: {response.status_code}')
 
         os.makedirs(self.cache_path, exist_ok=True)
 
+        # Download content into an in-memory buffer
         fileobj = BytesIO(response.content)
+
+        # Attempt to extract as a tarball, fall back to zip
         try:
-            with tarfile.open(fileobj=fileobj, mode='r|gz') as tar_ref:
+            with tarfile.open(fileobj=fileobj, mode='r:gz') as tar_ref:
                 tar_ref.extractall(path=self.cache_path)
         except tarfile.ReadError:
             fileobj.seek(0)
-            # Try as zip
-            with zipfile.ZipFile(fileobj) as zip_ref:
-                zip_ref.extractall(path=self.cache_path)
-
+            try:
+                with zipfile.ZipFile(fileobj) as zip_ref:
+                    zip_ref.extractall(path=self.cache_path)
+            except zipfile.BadZipFile:
+                raise TypeError(f"Could not extract file from {data_url}. "
+                                "File is not a valid tar.gz or zip archive.")
+
+        # --- GitHub-specific directory flattening ---
+        # GitHub archives often have a single top-level directory like 'repo-v1.0'.
+        # This logic moves the contents of that directory up one level for a cleaner cache.
         if 'github' in data_url and len(os.listdir(self.cache_path)) == 1:
-            # Github inserts one folder at the highest level of the tar file
-            # this compensates for this behavior
+            # Heuristically determine the name of the top-level directory
             gh_url = urlparse(data_url)
-
             repo = gh_url.path.split('/')[2]
 
             gh_ref = gh_url.path.split('/')[-1]
@@ -78,10 +140,11 @@ class HTTPResolver(RemoteResolver):
                 gh_ref = gh_ref[1:]
 
             github_folder = f"{repo}-{gh_ref}"
-
-            if github_folder in os.listdir(self.cache_path):
-                # This moves all files one level up
-                git_path = os.path.join(self.cache_path, github_folder)
-                for data_file in os.listdir(git_path):
-                    shutil.move(os.path.join(git_path, data_file), self.cache_path)
-                os.removedirs(git_path)
+            potential_path = os.path.join(self.cache_path, github_folder)
+
+            if os.path.isdir(potential_path):
+                # Move all files from the subdirectory to the cache root
+                for data_file in os.listdir(potential_path):
+                    shutil.move(os.path.join(potential_path, data_file), self.cache_path)
+                # Clean up the now-empty directory
+                os.rmdir(potential_path)