siliconcompiler 0.30.0__py3-none-any.whl → 0.31.1__py3-none-any.whl

siliconcompiler/_metadata.py

@@ -1,5 +1,5 @@
 # Version number following semver standard.
-version = '0.30.0'
+version = '0.31.1'
 
 # Default server address for remote runs, if unspecified.
 default_server = 'https://server.siliconcompiler.com'

siliconcompiler/apps/sc_install.py

@@ -17,7 +17,7 @@ class ChoiceOptional(Container):
     def __init__(self, choices):
         super().__init__()
 
-        self.__choices = set(choices)
+        self.__choices = sorted(set(choices))
 
     def __contains__(self, item):
         if not item:
@@ -30,7 +30,7 @@ class ChoiceOptional(Container):
 
     def get_items(self, choices):
         items = set(choices)
-        return sorted(list(items))
+        return sorted(items)
 
 
 def install_tool(tool, script, build_dir, prefix):
@@ -101,6 +101,19 @@ def print_machine_info():
     print("Scripts:  ", _get_tool_script_dir())
 
 
+def __print_summary(successful, failed):
+    max_len = 64
+    print("#"*max_len)
+    if successful:
+        msg = f"Installed: {', '.join(sorted(successful))}"
+        print(f"# {msg}")
+
+    if failed:
+        msg = f"Failed to install: {failed}"
+        print(f"# {msg}")
+    print("#"*max_len)
+
+
 def _get_tool_script_dir():
     return Path(siliconcompiler.__file__).parent / "toolscripts"
 
@@ -143,6 +156,10 @@ def _recommended_tool_groups(tools):
     return filter_groups
 
 
+class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
+    pass
+
+
 def main():
     progname = "sc-install"
     description = """
@@ -174,7 +191,7 @@ To system debugging information (this should only be used to debug):
     parser = argparse.ArgumentParser(
         prog=progname,
         description=description,
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        formatter_class=HelpFormatter)
 
     tools = _get_tools_list()
 
@@ -238,6 +255,7 @@ To system debugging information (this should only be used to debug):
                 args.tool.extend(tool_groups[group])
 
     tools_handled = set()
+    tools_completed = set()
     for tool in args.tool:
         if tool in tools_handled:
             continue
@@ -246,9 +264,14 @@ To system debugging information (this should only be used to debug):
             show_tool(tool, tools[tool])
         else:
             if not install_tool(tool, tools[tool], args.build_dir, args.prefix):
+                __print_summary(tools_completed, tool)
                 return 1
+            else:
+                tools_completed.add(tool)
 
     if not args.show:
+        __print_summary(tools_completed, None)
+
         msgs = []
         for env, path in (
                 ("PATH", "bin"),
@@ -258,7 +281,6 @@ To system debugging information (this should only be used to debug):
                 os.path.expandvars(os.path.expanduser(p))
                 for p in os.getenv(env, "").split(":")
             ]
-            print(envs)
             if check_path not in envs:
                 msgs.extend([
                     "",

siliconcompiler/apps/sc_remote.py

@@ -1,5 +1,4 @@
 # Copyright 2023 Silicon Compiler Authors. All Rights Reserved.
-import copy
 import os
 import sys
 
@@ -157,7 +156,6 @@ To delete a job, use:
     # in its "check_progress/ until job is done" loop.
     elif args['reconnect']:
         # Start from successors of entry nodes, so entry nodes are not fetched from remote.
-        environment = copy.deepcopy(os.environ)
         flow = chip.get('option', 'flow')
         entry_nodes = _get_flowgraph_entry_nodes(chip, flow)
         for entry_node in entry_nodes:
@@ -177,7 +175,7 @@ To delete a job, use:
                                     f'{chip.design}.pkg.json')
             if os.path.exists(manifest):
                 chip.schema.read_journal(manifest)
-        _finalize_run(chip, environment)
+        _finalize_run(chip)
 
         # Summarize the run.
         chip.summary()

siliconcompiler/core.py

@@ -392,7 +392,7 @@ class Chip:
                 for vals, step, index in self.schema._getvals(*key):
                     if not vals:
                         continue
-                    if self.get(*key, field='pernode') != 'never':
+                    if not self.get(*key, field='pernode').is_never():
                         if step is None:
                             step = Schema.GLOBAL_KEY
                         if index is None:
@@ -852,7 +852,7 @@ class Chip:
             strict = self.schema.get('option', 'strict')
             if field == 'value' and strict:
                 pernode = self.schema.get(*keypath, field='pernode')
-                if pernode == 'optional' and \
+                if pernode == schema_utils.PerNode.OPTIONAL and \
                    (step is None or index is None) and \
                    (Schema.GLOBAL_KEY not in (step, index)):  # allow explicit access to global
                     self.error(
@@ -1111,7 +1111,7 @@ class Chip:
         package_dir = os.path.dirname(os.path.abspath(filename))
 
         def __make_path(rel, path):
-            path = utils._resolve_env_vars(self, path)
+            path = utils._resolve_env_vars(self, path, None, None)
             if os.path.isabs(path):
                 if path.startswith(rel):
                     return os.path.relpath(path, rel), package_name
@@ -1282,7 +1282,7 @@ class Chip:
         """
         strict = self.get('option', 'strict')
         pernode = self.get(*keypath, field='pernode')
-        if strict and pernode == 'optional' and (step is None or index is None):
+        if strict and pernode == schema_utils.PerNode.OPTIONAL and (step is None or index is None):
             self.error(
                 f"Invalid args to find_files() of keypath {keypath}: step and "
                 "index are required for reading from this parameter while "
@@ -1394,7 +1394,9 @@ class Chip:
             result.append(utils.find_sc_file(self,
                                              path,
                                              missing_ok=missing_ok,
-                                             search_paths=search_paths))
+                                             search_paths=search_paths,
+                                             step=step,
+                                             index=index))
 
         if self._relative_path and not abs_path_only:
             rel_result = []
@@ -2828,10 +2830,10 @@ class Chip:
             set_step = None
             set_index = None
             pernode = self.get(*keypath, field='pernode')
-            if pernode == 'required':
+            if pernode == schema_utils.PerNode.REQUIRED:
                 set_step = step
                 set_index = index
-            elif pernode == 'optional':
+            elif pernode == schema_utils.PerNode.OPTIONAL:
                 for vals, key_step, key_index in self.schema._getvals(*keypath):
                     if key_step == step and key_index == index and vals:
                         set_step = step
@@ -2887,7 +2889,7 @@ class Chip:
         return self._dash
 
     ###########################################################################
-    def summary(self, show_all_indices=False, generate_image=True, generate_html=True):
+    def summary(self, show_all_indices=False, generate_image=True, generate_html=False):
         '''
         Prints a summary of the compilation manifest.
 
@@ -2927,6 +2929,10 @@ class Chip:
             results_img = os.path.join(work_dir, f'{self.design}.png')
             results_html = os.path.join(work_dir, 'report.html')
 
+            for path in (results_img, results_html):
+                if os.path.exists(path):
+                    os.remove(path)
+
             if generate_image:
                 _generate_summary_image(self, results_img)
 
@@ -2944,7 +2950,7 @@ class Chip:
                 elif os.path.isfile(results_html):
                     _open_html_report(self, results_html)
                 else:
-                    self._dashboard(wait=False)
+                    self.dashboard(wait=False)
 
     ###########################################################################
     def clock(self, pin, period, jitter=0, mode='global'):
@@ -3092,6 +3098,13 @@ class Chip:
             step (str): Step name
             index (int): Step index
         '''
+
+        if flow not in self.getkeys('flowgraph'):
+            raise ValueError(f'{flow} is not in the manifest')
+
+        if step not in self.getkeys('flowgraph', flow):
+            raise ValueError(f'{step} is not a valid step in {flow}')
+
         if index is None:
             # Iterate over all indexes
             for index in self.getkeys('flowgraph', flow, step):
@@ -3099,6 +3112,8 @@ class Chip:
             return
 
         index = str(index)
+        if index not in self.getkeys('flowgraph', flow, step):
+            raise ValueError(f'{index} is not a valid index for {step} in {flow}')
 
         # Save input edges
         node = (step, index)

siliconcompiler/flowgraph.py

@@ -4,23 +4,6 @@ from siliconcompiler import SiliconCompilerError, NodeStatus
 from siliconcompiler.tools._common import input_file_node_name, get_tool_task
 
 
-def _check_execution_nodes_inputs(chip, flow):
-    for node in nodes_to_execute(chip, flow):
-        if node in _get_execution_entry_nodes(chip, flow):
-            continue
-        pruned_node_inputs = set(_get_pruned_node_inputs(chip, flow, node))
-        node_inputs = set(_get_flowgraph_node_inputs(chip, flow, node))
-        tool, task = get_tool_task(chip, node[0], node[1], flow=flow)
-        if tool == 'builtin' and not pruned_node_inputs or \
-           tool != 'builtin' and pruned_node_inputs != node_inputs:
-            chip.logger.warning(
-                f'Flowgraph connection from {node_inputs.difference(pruned_node_inputs)} '
-                f'to {node} is missing. '
-                f'Double check your flowgraph and from/to/prune options.')
-            return False
-    return True
-
-
 def _nodes_to_execute(chip, flow, from_nodes, to_nodes, prune_nodes):
     '''
     Assumes a flowgraph with valid edges for the inputs
@@ -283,12 +266,12 @@ def nodes_to_execute(chip, flow=None):
     if flow is None:
         flow = chip.get('option', 'flow')
 
-    from_nodes = _get_execution_entry_nodes(chip, flow)
-    to_nodes = _get_execution_exit_nodes(chip, flow)
-    prune_nodes = chip.get('option', 'prune')
+    from_nodes = set(_get_execution_entry_nodes(chip, flow))
+    to_nodes = set(_get_execution_exit_nodes(chip, flow))
+    prune_nodes = set(chip.get('option', 'prune'))
     if from_nodes == to_nodes:
         return list(filter(lambda node: node not in prune_nodes, from_nodes))
-    return _nodes_to_execute(chip, flow, set(from_nodes), set(to_nodes), set(prune_nodes))
+    return _nodes_to_execute(chip, flow, from_nodes, to_nodes, prune_nodes)
 
 
 ###########################################################################
@@ -345,8 +328,13 @@ def _check_flowgraph(chip, flow=None):
             chip.logger.error(f'{step} is not defined in the {flow} flowgraph')
             error = True
 
-    if not _check_execution_nodes_inputs(chip, flow):
-        error = True
+    for step, index in chip.get('option', 'prune'):
+        if step not in chip.getkeys('flowgraph', flow):
+            chip.logger.error(f'{step} is not defined in the {flow} flowgraph')
+            error = True
+        elif str(index) not in chip.getkeys('flowgraph', flow, step):
+            chip.logger.error(f'{step}{index} is not defined in the {flow} flowgraph')
+            error = True
 
     unreachable_steps = _unreachable_steps_to_execute(chip, flow)
     if unreachable_steps:

siliconcompiler/{package.py → package/__init__.py}

@@ -1,25 +1,20 @@
 import os
-import requests
-import tarfile
-import zipfile
-from git import Repo, GitCommandError
 from urllib.parse import urlparse
 import importlib
-import shutil
 import re
 from siliconcompiler import SiliconCompilerError
 from siliconcompiler.utils import default_cache_dir, _resolve_env_vars
 import json
 from importlib.metadata import distributions, distribution
 import functools
-import fasteners
 import time
 from pathlib import Path
-from io import BytesIO
 
 from github import Github
 import github.Auth
 
+from siliconcompiler.utils import get_plugins
+
 
 def get_cache_path(chip):
     cache_path = chip.get('option', 'cachedir')
@@ -33,10 +28,45 @@ def get_cache_path(chip):
     return cache_path
 
 
-def _path(chip, package, download_handler):
-    if package in chip._packages:
-        return chip._packages[package]
+def get_download_cache_path(chip, package, ref):
+    cache_path = get_cache_path(chip)
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path, exist_ok=True)
+
+    if ref is None:
+        raise SiliconCompilerError(f'Reference is required for cached data: {package}', chip=chip)
+
+    return \
+        os.path.join(cache_path, f'{package}-{ref}'), \
+        os.path.join(cache_path, f'{package}-{ref}.lock')
+
+
+def _file_path_resolver(chip, package, path, ref, url):
+    return os.path.abspath(path.replace('file://', ''))
+
+
+def _python_path_resolver(chip, package, path, ref, url):
+    return path_from_python(chip, url.netloc)
 
+
+def _get_path_resolver(path):
+    url = urlparse(path)
+
+    for resolver in get_plugins("path_resolver"):
+        func = resolver(url)
+        if func:
+            return func, url
+
+    if url.scheme == "file":
+        return _file_path_resolver, url
+
+    if url.scheme == "python":
+        return _python_path_resolver, url
+
+    raise ValueError(f"{path} is not supported")
+
+
+def _path(chip, package):
     # Initially try retrieving data source from schema
     data = {}
     data['path'] = chip.get('package', 'source', package, 'path')
@@ -46,50 +76,15 @@ def _path(chip, package, download_handler):
             f'Could not find package source for {package} in schema. '
             'You can use register_source() to add it.', chip=chip)
 
-    data['path'] = _resolve_env_vars(chip, data['path'])
+    data['path'] = _resolve_env_vars(chip, data['path'], None, None)
 
-    url = urlparse(data['path'])
+    if os.path.exists(data['path']):
+        # Path is already a path
+        return os.path.abspath(data['path'])
 
-    # check network drive for package data source
-    if data['path'].startswith('file://') or os.path.exists(data['path']):
-        path = os.path.abspath(data['path'].replace('file://', ''))
-        chip.logger.info(f'Found {package} data at {path}')
-        chip._packages[package] = path
-        return path
-    elif data['path'].startswith('python://'):
-        path = path_from_python(chip, url.netloc)
-        chip.logger.info(f'Found {package} data at {path}')
-        chip._packages[package] = path
-        return path
-
-    # location of the python package
-    cache_path = get_cache_path(chip)
-    if not os.path.exists(cache_path):
-        os.makedirs(cache_path, exist_ok=True)
-    project_id = f'{package}-{data.get("ref")}'
-    if url.scheme not in ['git', 'git+https', 'https', 'git+ssh', 'ssh'] or not project_id:
-        raise SiliconCompilerError(
-            f'Could not find data path in package {package}: {data["path"]}',
-            chip=chip)
-
-    data_path = os.path.join(cache_path, project_id)
-
-    if download_handler:
-        download_handler(chip,
-                         package,
-                         data,
-                         url,
-                         data_path,
-                         os.path.join(cache_path, f'{project_id}.lock'))
-
-    if os.path.exists(data_path):
-        if package not in chip._packages:
-            chip.logger.info(f'Saved {package} data to {data_path}')
-            chip._packages[package] = data_path
-        return data_path
+    path_resolver, url = _get_path_resolver(data['path'])
 
-    raise SiliconCompilerError(f'Extracting {package} data to {data_path} failed',
-                               chip=chip)
+    return path_resolver(chip, package, data['path'], data['ref'], url)
 
 
 def path(chip, package):
@@ -102,40 +97,25 @@ def path(chip, package):
         path: Location of data source on the local system
     """
 
-    return _path(chip, package, __download_data)
-
+    if package not in chip._packages:
+        changed = False
+        data_path = _path(chip, package)
 
-def __download_data(chip, package, data, url, data_path, data_path_lock):
-    data_lock = fasteners.InterProcessLock(data_path_lock)
+        if isinstance(data_path, tuple) and len(data_path) == 2:
+            data_path, changed = data_path
 
-    _aquire_data_lock(data_path, data_lock)
+        if os.path.exists(data_path):
+            if package not in chip._packages and changed:
+                chip.logger.info(f'Saved {package} data to {data_path}')
+            else:
+                chip.logger.info(f'Found {package} data at {data_path}')
 
-    # check cached package data source
-    if os.path.exists(data_path):
-        chip.logger.info(f'Found cached {package} data at {data_path}')
-        if url.scheme in ['git', 'git+https', 'ssh', 'git+ssh']:
-            try:
-                repo = Repo(data_path)
-                if repo.untracked_files or repo.index.diff("HEAD"):
-                    chip.logger.warning('The repo of the cached data is dirty.')
-                _release_data_lock(data_lock)
-                chip._packages[package] = data_path
-                return
-            except GitCommandError:
-                chip.logger.warning('Deleting corrupted cache data.')
-                shutil.rmtree(path)
-        else:
-            _release_data_lock(data_lock)
             chip._packages[package] = data_path
-            return
-
-    # download package data source
-    if url.scheme in ['git', 'git+https', 'ssh', 'git+ssh']:
-        clone_synchronized(chip, package, data, data_path)
-    elif url.scheme == 'https':
-        extract_from_url(chip, package, data, data_path)
+        else:
+            raise SiliconCompilerError(f'Unable to locate {package} data in {data_path}',
+                                       chip=chip)
 
-    _release_data_lock(data_lock)
+    return chip._packages[package]
 
 
 def __get_filebased_lock(data_lock):
@@ -143,7 +123,7 @@ def __get_filebased_lock(data_lock):
     return Path(f'{base}.sc_lock')
 
 
-def _aquire_data_lock(data_path, data_lock):
+def aquire_data_lock(data_path, data_lock):
     # Wait a maximum of 10 minutes for other processes to finish
     max_seconds = 10 * 60
     try:
@@ -168,7 +148,7 @@ def _aquire_data_lock(data_path, data_lock):
                                'please delete it.')
 
 
-def _release_data_lock(data_lock):
+def release_data_lock(data_lock):
     # Check if file based locking method was used
     lock_file = __get_filebased_lock(data_lock)
     if lock_file.exists():
@@ -178,100 +158,6 @@ def _release_data_lock(data_lock):
     data_lock.release()
 
 
-def clone_synchronized(chip, package, data, data_path):
-    url = urlparse(data['path'])
-    try:
-        clone_from_git(chip, package, data, data_path)
-    except GitCommandError as e:
-        if 'Permission denied' in repr(e):
-            if url.scheme in ['ssh', 'git+ssh']:
-                chip.logger.error('Failed to authenticate. Please setup your git ssh.')
-            elif url.scheme in ['git', 'git+https']:
-                chip.logger.error('Failed to authenticate. Please use a token or ssh.')
-        else:
-            chip.logger.error(str(e))
-
-
-def clone_from_git(chip, package, data, repo_path):
-    url = urlparse(data['path'])
-    if url.scheme in ['git', 'git+https'] and url.username:
-        chip.logger.warning('Your token is in the data source path and will be stored in the '
-                            'schema. If you do not want this set the env variable GIT_TOKEN '
-                            'or use ssh for authentication.')
-    if url.scheme in ['git+ssh', 'ssh']:
-        chip.logger.info(f'Cloning {package} data from {url.netloc}:{url.path[1:]}')
-        # Git requires the format git@github.com:org/repo instead of git@github.com/org/repo
-        repo = Repo.clone_from(f'{url.netloc}:{url.path[1:]}',
-                               repo_path,
-                               recurse_submodules=True)
-    else:
-        if os.environ.get('GIT_TOKEN') and not url.username:
-            url = url._replace(netloc=f'{os.environ.get("GIT_TOKEN")}@{url.hostname}')
-        url = url._replace(scheme='https')
-        chip.logger.info(f'Cloning {package} data from {url.geturl()}')
-        repo = Repo.clone_from(url.geturl(), repo_path, recurse_submodules=True)
-    chip.logger.info(f'Checking out {data["ref"]}')
-    repo.git.checkout(data["ref"])
-    for submodule in repo.submodules:
-        submodule.update(init=True)
-
-
-def extract_from_url(chip, package, data, data_path):
-    url = urlparse(data['path'])
-    data_url = data.get('path')
-    headers = {}
-    if os.environ.get('GIT_TOKEN') or url.username:
-        headers['Authorization'] = f'token {os.environ.get("GIT_TOKEN") or url.username}'
-    if "github" in data_url:
-        headers['Accept'] = 'application/octet-stream'
-    data_url = data['path']
-    if data_url.endswith('/'):
-        data_url = f"{data_url}{data['ref']}.tar.gz"
-    chip.logger.info(f'Downloading {package} data from {data_url}')
-    response = requests.get(data_url, stream=True, headers=headers)
-    if not response.ok:
-        raise SiliconCompilerError(f'Failed to download {package} data source.', chip=chip)
-
-    fileobj = BytesIO(response.content)
-    try:
-        with tarfile.open(fileobj=fileobj, mode='r|gz') as tar_ref:
-            tar_ref.extractall(path=data_path)
-    except tarfile.ReadError:
-        fileobj.seek(0)
-        # Try as zip
-        with zipfile.ZipFile(fileobj) as zip_ref:
-            zip_ref.extractall(path=data_path)
-
-    if 'github' in url.netloc and len(os.listdir(data_path)) == 1:
-        # Github inserts one folder at the highest level of the tar file
-        # this compensates for this behavior
-        gh_url = urlparse(data_url)
-
-        repo = gh_url.path.split('/')[2]
-
-        ref = gh_url.path.split('/')[-1]
-        if repo.endswith('.git'):
-            ref = data['ref']
-        elif ref.endswith('.tar.gz'):
-            ref = ref[0:-7]
-        elif ref.endswith('.tgz'):
-            ref = ref[0:-4]
-        else:
-            ref = ref.split('.')[0]
-
-        if ref.startswith('v'):
-            ref = ref[1:]
-
-        github_folder = f"{repo}-{ref}"
-
-        if github_folder in os.listdir(data_path):
-            # This moves all files one level up
-            git_path = os.path.join(data_path, github_folder)
-            for data_file in os.listdir(git_path):
-                shutil.move(os.path.join(git_path, data_file), data_path)
-            os.removedirs(git_path)
-
-
 def path_from_python(chip, python_package, append_path=None):
     try:
         module = importlib.import_module(python_package)

siliconcompiler/package/git.py

@@ -0,0 +1,81 @@
+import shutil
+
+import os.path
+
+from git import Repo, GitCommandError
+from fasteners import InterProcessLock
+
+from siliconcompiler.package import get_download_cache_path
+from siliconcompiler.package import aquire_data_lock, release_data_lock
+
+
+def get_resolver(url):
+    if url.scheme in ("git", "git+https", "git+ssh", "ssh"):
+        return git_resolver
+    return None
+
+
+def git_resolver(chip, package, path, ref, url):
+    data_path, data_path_lock = get_download_cache_path(chip, package, ref)
+
+    # Acquire lock
+    data_lock = InterProcessLock(data_path_lock)
+    aquire_data_lock(data_path, data_lock)
+
+    if os.path.exists(data_path):
+        try:
+            repo = Repo(data_path)
+            if repo.untracked_files or repo.index.diff("HEAD"):
+                chip.logger.warning('The repo of the cached data is dirty.')
+            release_data_lock(data_lock)
+            return data_path, False
+        except GitCommandError:
+            chip.logger.warning('Deleting corrupted cache data.')
+            shutil.rmtree(data_path)
+
+    clone_synchronized(chip, package, path, ref, url, data_path)
+
+    release_data_lock(data_lock)
+
+    return data_path, True
+
+
+def clone_synchronized(chip, package, path, ref, url, data_path):
+    try:
+        clone_from_git(chip, package, path, ref, url, data_path)
+    except GitCommandError as e:
+        if 'Permission denied' in repr(e):
+            if url.scheme in ['ssh', 'git+ssh']:
+                chip.logger.error('Failed to authenticate. Please setup your git ssh.')
+            elif url.scheme in ['git', 'git+https']:
+                chip.logger.error('Failed to authenticate. Please use a token or ssh.')
+        else:
+            chip.logger.error(str(e))
+
+
+def clone_from_git(chip, package, path, ref, url, data_path):
+    if url.scheme in ['git', 'git+https'] and url.username:
+        chip.logger.warning('Your token is in the data source path and will be stored in the '
+                            'schema. If you do not want this set the env variable GIT_TOKEN '
+                            'or use ssh for authentication.')
+    if url.scheme in ['git+ssh']:
+        chip.logger.info(f'Cloning {package} data from {url.netloc}:{url.path[1:]}')
+        # Git requires the format git@github.com:org/repo instead of git@github.com/org/repo
+        repo = Repo.clone_from(f'{url.netloc}/{url.path[1:]}',
+                               data_path,
+                               recurse_submodules=True)
+    elif url.scheme in ['ssh']:
+        chip.logger.info(f'Cloning {package} data from {path}')
+        repo = Repo.clone_from(path,
+                               data_path,
+                               recurse_submodules=True)
+    else:
+        if os.environ.get('GIT_TOKEN') and not url.username:
+            url = url._replace(netloc=f'{os.environ.get("GIT_TOKEN")}@{url.hostname}')
+        url = url._replace(scheme='https')
+        chip.logger.info(f'Cloning {package} data from {url.geturl()}')
+        repo = Repo.clone_from(url.geturl(), data_path, recurse_submodules=True)
+    chip.logger.info(f'Checking out {ref}')
+    repo.git.checkout(ref)
+    for submodule in repo.submodules:
+        submodule.update(init=True)