jolt 0.9.342__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/plugins/git.py

@@ -1,9 +1,10 @@
 import os
 import pygit2
 import re
+from threading import RLock
 import urllib.parse
 
-from jolt.tasks import BooleanParameter, Export, Parameter, TaskRegistry, WorkspaceResource
+from jolt.tasks import BooleanParameter, Export, IntParameter, Parameter, TaskRegistry, WorkspaceResource
 from jolt.influence import FileInfluence, HashInfluenceRegistry
 from jolt.tools import Tools
 from jolt.loader import JoltLoader, workspace_locked
@@ -20,6 +21,13 @@ from jolt.error import raise_task_error_if
 log.verbose("[Git] Loaded")
 
 
+def locked(func):
+    def _f(self, *args, **kwargs):
+        with self._lock:
+            return func(self, *args, **kwargs)
+    return _f
+
+
 class GitRepository(object):
     def __init__(self, url, path, relpath, refspecs=None):
         self.path = path
@@ -29,7 +37,7 @@ class GitRepository(object):
         self.url = url
         self.default_refspecs = [
             '+refs/heads/*:refs/remotes/origin/*',
-            '+refs/tags/*:refs/remotes/origin/*',
+            '+refs/tags/*:refs/tags/*',
         ]
         self.refspecs = refspecs or []
         self._tree_hash = {}
@@ -68,7 +76,7 @@ class GitRepository(object):
     def is_indexed(self):
         return self.is_cloned() and fs.path.exists(self._git_index())
 
-    def clone(self):
+    def clone(self, submodules=False):
         log.info("Cloning into {0}", self.path)
 
         # Get reference repository path
@@ -103,19 +111,30 @@ class GitRepository(object):
 
                 utils.call_and_catch(self.tools.run, "git remote remove origin", output=False)
                 self.tools.run("git remote add origin {}", self.url, output_on_error=True)
-                self.tools.run("git fetch origin", output_on_error=True)
+                self._fetch_origin(submodules=submodules)
                 self.tools.run("git checkout -f FETCH_HEAD", output_on_error=True)
         else:
+            # Get configurable extra clone options
+            extra_clone_options = config.get("git", "clone_options", "")
+
             if refpath and os.path.isdir(refpath):
-                self.tools.run("git clone --reference-if-able {0} {1} {2}", refpath, self.url, self.path, output_on_error=True)
+                self.tools.run("git clone --reference-if-able  {0} {1} {2} {3}", refpath, extra_clone_options, self.url, self.path, output_on_error=True)
             else:
-                self.tools.run("git clone  {0} {1}", self.url, self.path, output_on_error=True)
+                self.tools.run("git clone {0} {1} {2}", extra_clone_options, self.url, self.path, output_on_error=True)
 
         self._init_repo()
         raise_error_if(
             self.repository is None,
             "Failed to clone repository '{0}'", self.relpath)
 
+    @utils.retried.on_exception(JoltCommandError, pattern="Command failed: git fetch", count=6, backoff=[2, 5, 10, 15, 20, 30])
+    def _fetch_origin(self, submodules=False):
+        # Get configurable extra clone options
+        extra_fetch_options = config.get("git", "fetch_options", "")
+
+        with self.tools.cwd(self.path):
+            self.tools.run("git fetch {0} {1} origin", "", extra_fetch_options, output_on_error=True)
+
     @utils.cached.instance
     def diff_unchecked(self):
         if not self.is_indexed():
@@ -239,20 +258,25 @@ class GitRepository(object):
         with self.tools.cwd(self.path):
             return self.tools.run("git reset --hard", output_on_error=True)
 
+    @utils.retried.on_exception(JoltCommandError, pattern="Command failed: git fetch", count=6, backoff=[2, 5, 10, 15, 20, 30])
     def fetch(self, commit=None):
         if commit and not self.is_valid_sha(commit):
             commit = None
 
+        # Get configurable extra clone options
+        extra_fetch_options = config.get("git", "fetch_options", "")
+
         refspec = " ".join(self.default_refspecs + self.refspecs)
         with self.tools.cwd(self.path):
             log.info("Fetching {0} from {1}", commit or refspec or 'commits', self.url)
             self.tools.run(
-                "git fetch --prune {url} {what}",
+                "git fetch --force --prune {extra_fetch_options} {url} {what}",
+                extra_fetch_options=extra_fetch_options,
                 url=self.url,
                 what=commit or refspec or '',
                 output_on_error=True)
 
-    def checkout(self, rev, commit=None):
+    def checkout(self, rev, commit=None, submodules=False):
         if rev == self._last_rev:
             log.debug("Checkout skipped, already @ {}", rev)
             return False
@@ -260,16 +284,24 @@ class GitRepository(object):
         with self.tools.cwd(self.path):
             try:
                 self.tools.run("git checkout -f {rev}", rev=rev, output=False)
+                if submodules:
+                    self._update_submodules()
             except Exception:
                 self.fetch(commit=commit)
                 try:
                     self.tools.run("git checkout -f {rev}", rev=rev, output_on_error=True)
+                    if submodules:
+                        self._update_submodules()
                 except Exception:
                     raise_error("Commit does not exist in remote for '{}': {}", self.relpath, rev)
         self._original_head = False
         self._last_rev = rev
         return True
 
+    def _update_submodules(self):
+        with self.tools.cwd(self.path):
+            self.tools.run("git submodule update --init --recursive", output_on_error=True)
+
 
 _gits = {}
 
@@ -420,6 +452,12 @@ class Git(WorkspaceResource, FileInfluence):
     path = Parameter(required=False, help="Local path where the repository should be cloned.")
     """ Alternative path where the repository should be cloned. Relative to ``joltdir``. Optional. """
 
+    submodules = BooleanParameter(default=False, help="Initialize and update git submodules after cloning.")
+    """ Initialize and update git submodules after cloning. Default ``False``. Optional. """
+
+    clean = BooleanParameter(default=False, help="Clean repository after it has been used")
+    """ Removes untracked files from the repository. """
+
     _revision = Export(value=lambda t: t._export_revision())
     """ To worker exported value of the revision to be checked out. """
 
@@ -428,6 +466,7 @@ class Git(WorkspaceResource, FileInfluence):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self._lock = RLock()
         self.joltdir = JoltLoader.get().joltdir
 
         # Set the path to the repo
@@ -438,6 +477,9 @@ class Git(WorkspaceResource, FileInfluence):
             self.abspath = fs.path.join(self.joltdir, str(self.path) or self._get_name())
             self.relpath = fs.path.relpath(self.abspath, self.tools.wsroot)
 
+        self.abspath = fs.path.normpath(self.abspath)
+        self.relpath = fs.path.normpath(self.relpath)
+
         # Create the git repository
         self.refspecs = kwargs.get("refspecs", [])
         self.git = new_git(self.url, self.abspath, self.relpath, self.refspecs)
@@ -482,6 +524,11 @@ class Git(WorkspaceResource, FileInfluence):
         self._assign_git(owner)
         artifact.worktree = fs.path.relpath(self.abspath, owner.joltdir)
 
+    def release(self, artifact, deps, tools, owner):
+        if self.clean:
+            self.git.clean()
+            self.git.reset()
+
     def prepare_ws_for(self, task):
         """ Prepare the workspace for the task.
 
@@ -502,10 +549,11 @@ class Git(WorkspaceResource, FileInfluence):
         if force or self._must_influence() or self._revision.is_imported:
             self._acquire_ws()
 
+    @locked
     def _acquire_ws(self):
         commit = None
         if not self.git.is_cloned():
-            self.git.clone()
+            self.git.clone(submodules=bool(self.submodules))
         if not self._revision.is_imported:
             self.git.diff_unchecked()
         else:
@@ -518,7 +566,7 @@ class Git(WorkspaceResource, FileInfluence):
             # Should be safe to do this now
             rev = self.git.rev_parse(rev)
             if not self.git.is_head(rev) or self._revision.is_imported:
-                if self.git.checkout(rev, commit=commit):
+                if self.git.checkout(rev, commit=commit, submodules=bool(self.submodules)):
                     self.git.clean()
                     self.git.patch(self._diff.value)
 
@@ -545,10 +593,11 @@ class Git(WorkspaceResource, FileInfluence):
         influence = super()._influence()
         return influence + [self] if self._must_influence() else influence
 
+    @locked
     @utils.cached.instance
     def get_influence(self, task):
         if not self.git.is_cloned():
-            self.git.clone()
+            self.git.clone(submodules=bool(self.submodules))
         if not self._revision.is_imported:
             self.git.diff_unchecked()
         rev = self._get_revision()

jolt/plugins/libtool.py

@@ -0,0 +1,63 @@
+from jolt import attributes
+
+
+class Libtool(object):
+    def __init__(self, tools):
+        self.tools = tools
+
+    def relocate(self, artifact, dirs=["lib"], prefix=None):
+        prefix = str(artifact.strings.install_prefix) if prefix is None else prefix
+        for dir in dirs:
+            with self.tools.cwd(artifact.path, dir):
+                for file in self.tools.glob("*.la"):
+                    self.tools.replace_in_file(file, prefix, artifact.final_path)
+
+
+def relocate(dirs=["lib"]):
+    """
+    Relocate libtool archive files (.la) published by task.
+
+    When an artifact is published, all .la files found in the specified
+    directories will have their install prefix updated to the artifact's final path.
+    The original install prefix is stored in the artifact's strings metadata
+    under the key 'libtool_prefix' for use during unpacking.
+
+      :param dirs: List of directories inside the artifact to relocate .la files in.
+
+    """
+
+    def decorate(cls):
+        original_publish = cls.publish
+        original_unpack = cls.unpack
+
+        def publish(self, artifact, tools):
+            original_publish(self, artifact, tools)
+            lt = Libtool(tools)
+            lt.relocate(artifact, dirs, prefix=artifact.strings.install_prefix)
+            artifact.libtool_prefix = artifact.final_path
+
+        def unpack(self, artifact, tools):
+            original_unpack(self, artifact, tools)
+            lt = Libtool(tools)
+            lt.relocate(artifact, dirs, prefix=artifact.strings.libtool_prefix)
+            artifact.strings.libtool_prefix = None
+
+        cls.publish = publish
+        cls.unpack = unpack
+
+        return cls
+
+    return decorate
+
+
+def requires():
+    """ Decorator to add Libtool requirement to a task. """
+
+    import jolt.pkgs.libtool
+
+    def decorate(cls):
+        cls = attributes.requires("requires_libtool")(cls)
+        cls.requires_libtool = ["libtool"]
+        return cls
+
+    return decorate