siliconcompiler 0.33.1__py3-none-any.whl → 0.34.0__py3-none-any.whl

siliconcompiler/package/git.py

@@ -3,82 +3,80 @@ import shutil
 import os.path
 
 from git import Repo, GitCommandError
-from fasteners import InterProcessLock
-
-from siliconcompiler.package import get_download_cache_path
-from siliconcompiler.package import aquire_data_lock, release_data_lock
-
-
-def get_resolver(url):
-    if url.scheme in ("git", "git+https", "git+ssh", "ssh"):
-        return git_resolver
-    return None
-
-
-def git_resolver(chip, package, path, ref, url, fetch):
-    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
-
-    if not fetch:
-        return data_path, False
-
-    # Acquire lock
-    data_lock = InterProcessLock(data_path_lock)
-    aquire_data_lock(data_path, data_lock)
+from siliconcompiler.package import RemoteResolver
+
+
+def get_resolver():
+    return {
+        "git": GitResolver,
+        "git+https": GitResolver,
+        "git+ssh": GitResolver,
+        "ssh": GitResolver
+    }
+
+
+class GitResolver(RemoteResolver):
+    def __init__(self, name, root, source, reference=None):
+        super().__init__(name, root, source, reference)
+
+    def check_cache(self):
+        if os.path.exists(self.cache_path):
+            try:
+                repo = Repo(self.cache_path)
+                if repo.untracked_files or repo.index.diff("HEAD"):
+                    self.logger.warning('The repo of the cached data is dirty.')
+                return True
+            except GitCommandError:
+                self.logger.warning('Deleting corrupted cache data.')
+                shutil.rmtree(self.cache_path)
+                return False
+        return False
+
+    def __get_token_env(self):
+        token_name = self.name.upper()
+        for tok in ('#', '$', '&', '-', '=', '!', '/'):
+            token_name = token_name.replace(tok, '')
+
+        search_env = (
+            f'GITHUB_{token_name}_TOKEN',
+            'GITHUB_TOKEN',
+            'GIT_TOKEN'
+        )
+
+        token = None
+        for env in search_env:
+            token = os.environ.get(env, None)
+
+            if token:
+                return token
+        return None
+
+    @property
+    def git_path(self):
+        if self.urlscheme == "git+ssh":
+            return f"ssh://{self.urlpath}{self.urlparse.path}"
+        if self.urlscheme == "ssh":
+            return self.source
+        url = self.urlparse
+        if not url.username and self.__get_token_env():
+            url = url._replace(netloc=f'{self.__get_token_env()}@{url.hostname}')
+        url = url._replace(scheme='https')
+        return url.geturl()
 
-    if os.path.exists(data_path):
+    def resolve_remote(self):
         try:
-            repo = Repo(data_path)
-            if repo.untracked_files or repo.index.diff("HEAD"):
-                chip.logger.warning('The repo of the cached data is dirty.')
-            release_data_lock(data_lock)
-            return data_path, False
-        except GitCommandError:
-            chip.logger.warning('Deleting corrupted cache data.')
-            shutil.rmtree(data_path)
-
-    clone_synchronized(chip, package, path, ref, url, data_path)
-
-    release_data_lock(data_lock)
-
-    return data_path, True
-
-
-def clone_synchronized(chip, package, path, ref, url, data_path):
-    try:
-        clone_from_git(chip, package, path, ref, url, data_path)
-    except GitCommandError as e:
-        if 'Permission denied' in repr(e):
-            if url.scheme in ['ssh', 'git+ssh']:
-                chip.logger.error('Failed to authenticate. Please setup your git ssh.')
-            elif url.scheme in ['git', 'git+https']:
-                chip.logger.error('Failed to authenticate. Please use a token or ssh.')
-        else:
-            chip.logger.error(str(e))
-
-
-def clone_from_git(chip, package, path, ref, url, data_path):
-    if url.scheme in ['git', 'git+https'] and url.username:
-        chip.logger.warning('Your token is in the data source path and will be stored in the '
-                            'schema. If you do not want this set the env variable GIT_TOKEN '
-                            'or use ssh for authentication.')
-    if url.scheme in ['git+ssh']:
-        chip.logger.info(f'Cloning {package} data from {url.netloc}:{url.path[1:]}')
-        # Git requires the format git@github.com:org/repo instead of git@github.com/org/repo
-        repo = Repo.clone_from(f'{url.netloc}/{url.path[1:]}',
-                               data_path,
-                               recurse_submodules=True)
-    elif url.scheme in ['ssh']:
-        chip.logger.info(f'Cloning {package} data from {path}')
-        repo = Repo.clone_from(path,
-                               data_path,
-                               recurse_submodules=True)
-    else:
-        if os.environ.get('GIT_TOKEN') and not url.username:
-            url = url._replace(netloc=f'{os.environ.get("GIT_TOKEN")}@{url.hostname}')
-        url = url._replace(scheme='https')
-        chip.logger.info(f'Cloning {package} data from {url.geturl()}')
-        repo = Repo.clone_from(url.geturl(), data_path, recurse_submodules=True)
-    chip.logger.info(f'Checking out {ref}')
-    repo.git.checkout(ref)
-    for submodule in repo.submodules:
-        submodule.update(init=True)
+            path = self.git_path
+            self.logger.info(f'Cloning {self.name} data from {path}')
+            repo = Repo.clone_from(path, self.cache_path, recurse_submodules=True)
+            self.logger.info(f'Checking out {self.reference}')
+            repo.git.checkout(self.reference)
+            for submodule in repo.submodules:
+                submodule.update(recursive=True, init=True, force=True)
+        except GitCommandError as e:
+            if 'Permission denied' in repr(e):
+                if self.urlscheme in ('ssh', 'git+ssh'):
+                    raise RuntimeError('Failed to authenticate. Please setup your git ssh.')
+                elif self.urlscheme in ('git', 'git+https'):
+                    raise RuntimeError('Failed to authenticate. Please use a token or ssh.')
+            else:
+                raise e

siliconcompiler/package/github.py

@@ -1,124 +1,97 @@
 import os
-from fasteners import InterProcessLock
+
 from github import Github, Auth
 from github.GithubException import UnknownObjectException
-from urllib.parse import urlparse
-from siliconcompiler.package import get_download_cache_path
-from siliconcompiler.package import aquire_data_lock, release_data_lock
-from siliconcompiler.package.https import _http_resolver
-
-
-def get_resolver(url):
-    if url.scheme in ("github",):
-        return github_any_resolver
-    if url.scheme in ("github+private",):
-        return github_private_resolver
-    return None
-
-
-def github_any_resolver(chip, package, path, ref, url, fetch):
-    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
-
-    if not fetch:
-        return data_path, False
-
-    # Acquire lock
-    data_lock = InterProcessLock(data_path_lock)
-    aquire_data_lock(data_path, data_lock)
-
-    if os.path.exists(data_path):
-        release_data_lock(data_lock)
-        return data_path, False
-
-    try:
-        return _github_resolver(chip, package, path, ref, url, data_lock)
-    except UnknownObjectException:
-        return github_private_resolver(chip, package, path, ref, url, fetch, data_lock=data_lock)
-
-
-def github_private_resolver(chip, package, path, ref, url, fetch, data_lock=None):
-    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
-
-    if not fetch:
-        return data_path, False
-
-    if not data_lock:
-        # Acquire lock
-        data_lock = InterProcessLock(data_path_lock)
-        aquire_data_lock(data_path, data_lock)
 
-    if os.path.exists(data_path):
-        release_data_lock(data_lock)
-        return data_path, False
+from siliconcompiler.package.https import HTTPResolver
 
-    gh = Github(auth=Auth.Token(__get_github_auth_token(package)))
 
-    return _github_resolver(chip, package, path, ref, url, data_lock, gh=gh)
+def get_resolver():
+    return {
+        "github": GithubResolver,
+        "github+private": GithubResolver
+    }
 
 
-def _github_resolver(chip, package, path, ref, url, data_lock, gh=None):
-    if not gh:
-        gh = Github()
+class GithubResolver(HTTPResolver):
+    def __init__(self, name, root, source, reference=None):
+        super().__init__(name, root, source, reference)
 
-    url_parts = (url.netloc, *url.path.split("/")[1:])
+        if len(self.gh_path) != 4:
+            raise ValueError(
+                f"{self.source} is not in the proper form: "
+                "<owner>/<repository>/<version>/<artifact>")
 
-    if len(url_parts) != 4:
-        raise ValueError(
-            f"{path} is not in the proper form: <owner>/<repository>/<version>/<artifact>")
+    @property
+    def gh_path(self):
+        return self.urlpath, *self.urlparse.path.split("/")[1:]
 
-    repository = "/".join(url_parts[0:2])
-    release = url_parts[2]
-    artifact = url_parts[3]
+    @property
+    def download_url(self):
+        url_parts = self.gh_path
 
-    release_url = __get_release_url(gh, repository, release, artifact)
+        repository = "/".join(url_parts[0:2])
+        release = url_parts[2]
+        artifact = url_parts[3]
 
-    return _http_resolver(chip, package, release_url, ref, urlparse(release_url), data_lock)
+        if self.urlscheme == "github+private":
+            return self.__get_release_url(repository, release, artifact, True)
 
+        try:
+            return self.__get_release_url(repository, release, artifact, False)
+        except UnknownObjectException:
+            return self.__get_release_url(repository, release, artifact, True)
 
-def __get_release_url(gh, repository, release, artifact):
-    if artifact == f"{release}.zip":
-        return f"https://github.com/{repository}/archive/refs/tags/{release}.zip"
-    if artifact == f"{release}.tar.gz":
-        return f"https://github.com/{repository}/archive/refs/tags/{release}.tar.gz"
+    def __get_release_url(self, repository, release, artifact, private: bool):
+        if artifact == f"{release}.zip":
+            return f"https://github.com/{repository}/archive/refs/tags/{release}.zip"
+        if artifact == f"{release}.tar.gz":
+            return f"https://github.com/{repository}/archive/refs/tags/{release}.tar.gz"
 
-    repo = gh.get_repo(repository)
+        repo = self.__gh(private).get_repo(repository)
 
-    if not release:
-        release = repo.get_latest_release().tag_name
+        if not release:
+            release = repo.get_latest_release().tag_name
 
-    url = None
-    for repo_release in repo.get_releases():
-        if repo_release.tag_name == release:
-            for asset in repo_release.assets:
-                if asset.name == artifact:
-                    url = asset.url
+        url = None
+        for repo_release in repo.get_releases():
+            if repo_release.tag_name == release:
+                for asset in repo_release.assets:
+                    if asset.name == artifact:
+                        url = asset.url
 
-    if not url:
-        raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
+        if not url:
+            raise ValueError(f'Unable to find release asset: {repository}/{release}/{artifact}')
 
-    return url
+        return url
 
+    def __get_gh_auth(self):
+        token_name = self.name.upper()
+        for tok in ('#', '$', '&', '-', '=', '!', '/'):
+            token_name = token_name.replace(tok, '')
 
-def __get_github_auth_token(package_name):
-    token_name = package_name.upper()
-    for tok in ('#', '$', '&', '-', '=', '!', '/'):
-        token_name = token_name.replace(tok, '')
+        search_env = (
+            f'GITHUB_{token_name}_TOKEN',
+            'GITHUB_TOKEN',
+            'GIT_TOKEN'
+        )
 
-    search_env = (
-        f'GITHUB_{token_name}_TOKEN',
-        'GITHUB_TOKEN',
-        'GIT_TOKEN'
-    )
+        token = None
+        for env in search_env:
+            token = os.environ.get(env, None)
 
-    token = None
-    for env in search_env:
-        token = os.environ.get(env, None)
+            if token:
+                break
 
-        if token:
-            break
+        if not token:
+            raise ValueError('Unable to determine authorization token for GitHub, '
+                             'please set one of the following environmental variables: '
+                             f'{", ".join(search_env)}')
 
-    if not token:
-        raise ValueError('Unable to determine authorization token for GitHub, '
-                         f'please set one of the following environmental variables: {search_env}')
+        return token
 
-    return token
+    def __gh(self, private: bool) -> Github:
+        if private:
+            return Github(auth=Auth.Token(self.__get_gh_auth()))
+        else:
+            return Github()

siliconcompiler/package/https.py

@@ -5,99 +5,83 @@ import zipfile
 
 import os.path
 
-from fasteners import InterProcessLock
 from io import BytesIO
 from urllib.parse import urlparse
 
-from siliconcompiler import SiliconCompilerError
-from siliconcompiler.package import get_download_cache_path
-from siliconcompiler.package import aquire_data_lock, release_data_lock
-
-
-def get_resolver(url):
-    if url.scheme in ("http", "https"):
-        return http_resolver
-
-    return None
-
-
-def http_resolver(chip, package, path, ref, url, fetch):
-    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
-
-    if not fetch:
-        return data_path, False
-
-    # Acquire lock
-    data_lock = InterProcessLock(data_path_lock)
-    aquire_data_lock(data_path, data_lock)
-
-    if os.path.exists(data_path):
-        release_data_lock(data_lock)
-        return data_path, False
-
-    return _http_resolver(chip, package, path, ref, url, data_lock)
-
-
-def _http_resolver(chip, package, path, ref, url, data_lock):
-    data_path, _ = get_download_cache_path(chip, package, ref)
-
-    extract_from_url(chip, package, path, ref, url, data_path)
-
-    release_data_lock(data_lock)
-
-    return data_path, True
-
-
-def extract_from_url(chip, package, path, ref, url, data_path):
-    data_url = path
-    headers = {}
-    if os.environ.get('GIT_TOKEN') or url.username:
-        headers['Authorization'] = f'token {os.environ.get("GIT_TOKEN") or url.username}'
-    if "github" in data_url:
-        headers['Accept'] = 'application/octet-stream'
-    data_url = path
-    if data_url.endswith('/'):
-        data_url = f"{data_url}{ref}.tar.gz"
-    chip.logger.info(f'Downloading {package} data from {data_url}')
-    response = requests.get(data_url, stream=True, headers=headers)
-    if not response.ok:
-        raise SiliconCompilerError(f'Failed to download {package} data source.', chip=chip)
-
-    fileobj = BytesIO(response.content)
-    try:
-        with tarfile.open(fileobj=fileobj, mode='r|gz') as tar_ref:
-            tar_ref.extractall(path=data_path)
-    except tarfile.ReadError:
-        fileobj.seek(0)
-        # Try as zip
-        with zipfile.ZipFile(fileobj) as zip_ref:
-            zip_ref.extractall(path=data_path)
-
-    if 'github' in url.netloc and len(os.listdir(data_path)) == 1:
-        # Github inserts one folder at the highest level of the tar file
-        # this compensates for this behavior
-        gh_url = urlparse(data_url)
-
-        repo = gh_url.path.split('/')[2]
-
-        gh_ref = gh_url.path.split('/')[-1]
-        if repo.endswith('.git'):
-            gh_ref = ref
-        elif gh_ref.endswith('.tar.gz'):
-            gh_ref = gh_ref[0:-7]
-        elif gh_ref.endswith('.tgz'):
-            gh_ref = gh_ref[0:-4]
-        else:
-            gh_ref = gh_ref.split('.')[0]
-
-        if gh_ref.startswith('v'):
-            gh_ref = gh_ref[1:]
-
-        github_folder = f"{repo}-{gh_ref}"
-
-        if github_folder in os.listdir(data_path):
-            # This moves all files one level up
-            git_path = os.path.join(data_path, github_folder)
-            for data_file in os.listdir(git_path):
-                shutil.move(os.path.join(git_path, data_file), data_path)
-            os.removedirs(git_path)
+from siliconcompiler.package import RemoteResolver
+
+
+def get_resolver():
+    return {
+        "http": HTTPResolver,
+        "https": HTTPResolver
+    }
+
+
+class HTTPResolver(RemoteResolver):
+    def check_cache(self):
+        return os.path.exists(self.cache_path)
+
+    @property
+    def download_url(self):
+        data_url = self.source
+        if data_url.endswith('/'):
+            data_url = f"{data_url}{self.reference}.tar.gz"
+        return data_url
+
+    def resolve_remote(self):
+        data_url = self.download_url
+
+        headers = {}
+        auth_token = os.environ.get('GIT_TOKEN', self.urlparse.username)
+        if auth_token:
+            headers['Authorization'] = f'token {auth_token}'
+        if "github" in data_url:
+            headers['Accept'] = 'application/octet-stream'
+
+        self.logger.info(f'Downloading {self.name} data from {data_url}')
+
+        response = requests.get(data_url, stream=True, headers=headers)
+        if not response.ok:
+            raise FileNotFoundError(f'Failed to download {self.name} data source.')
+
+        os.makedirs(self.cache_path, exist_ok=True)
+
+        fileobj = BytesIO(response.content)
+        try:
+            with tarfile.open(fileobj=fileobj, mode='r|gz') as tar_ref:
+                tar_ref.extractall(path=self.cache_path)
+        except tarfile.ReadError:
+            fileobj.seek(0)
+            # Try as zip
+            with zipfile.ZipFile(fileobj) as zip_ref:
+                zip_ref.extractall(path=self.cache_path)
+
+        if 'github' in data_url and len(os.listdir(self.cache_path)) == 1:
+            # Github inserts one folder at the highest level of the tar file
+            # this compensates for this behavior
+            gh_url = urlparse(data_url)
+
+            repo = gh_url.path.split('/')[2]
+
+            gh_ref = gh_url.path.split('/')[-1]
+            if repo.endswith('.git'):
+                gh_ref = self.reference
+            elif gh_ref.endswith('.tar.gz'):
+                gh_ref = gh_ref[0:-7]
+            elif gh_ref.endswith('.tgz'):
+                gh_ref = gh_ref[0:-4]
+            else:
+                gh_ref = gh_ref.split('.')[0]
+
+            if gh_ref.startswith('v'):
+                gh_ref = gh_ref[1:]
+
+            github_folder = f"{repo}-{gh_ref}"
+
+            if github_folder in os.listdir(self.cache_path):
+                # This moves all files one level up
+                git_path = os.path.join(self.cache_path, github_folder)
+                for data_file in os.listdir(git_path):
+                    shutil.move(os.path.join(git_path, data_file), self.cache_path)
+                os.removedirs(git_path)