jolt 0.9.328__tar.gz → 0.9.332__tar.gz

{jolt-0.9.328 → jolt-0.9.332}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jolt
-Version: 0.9.328
+Version: 0.9.332
 Summary: A task executor
 Home-page: https://github.com/srand/jolt
 Author: Robert Andersson
@@ -27,33 +27,33 @@ Requires-Dist: backports.tarfile==1.2.0
 Requires-Dist: bz2file==0.98
 Requires-Dist: certifi==2024.8.30
 Requires-Dist: cffi==1.17.1
-Requires-Dist: charset-normalizer==3.3.2
+Requires-Dist: charset-normalizer==3.4.0
 Requires-Dist: click==8.1.7
 Requires-Dist: colorama==0.4.6
 Requires-Dist: fasteners==0.19
-Requires-Dist: grpcio==1.66.1
+Requires-Dist: grpcio==1.68.0
 Requires-Dist: idna==3.10
 Requires-Dist: importlib-metadata==8.5.0
 Requires-Dist: importlib_metadata
 Requires-Dist: jaraco.classes==3.4.0
 Requires-Dist: jaraco.context==6.0.1
-Requires-Dist: jaraco.functools==4.0.2
+Requires-Dist: jaraco.functools==4.1.0
 Requires-Dist: jeepney==0.8.0
 Requires-Dist: jinja2==3.1.4
-Requires-Dist: keyring==25.4.1
+Requires-Dist: keyring==25.5.0
 Requires-Dist: keyrings.alt==5.0.2
 Requires-Dist: lxml==5.3.0
 Requires-Dist: more-itertools==10.5.0
 Requires-Dist: multi_key_dict
-Requires-Dist: ninja==1.11.1.1
-Requires-Dist: protobuf==5.28.2
-Requires-Dist: psutil==6.0.0
+Requires-Dist: ninja==1.11.1.2
+Requires-Dist: protobuf==5.29.0
+Requires-Dist: psutil==6.1.0
 Requires-Dist: pycparser==2.22
 Requires-Dist: pygit2==1.15.1
 Requires-Dist: requests==2.32.3
-Requires-Dist: tqdm==4.66.5
+Requires-Dist: tqdm==4.67.1
 Requires-Dist: urllib3==1.26.20
-Requires-Dist: zipp==3.20.2
+Requires-Dist: zipp==3.21.0
 Requires-Dist: zstandard==0.23.0
 Provides-Extra: allure
 Requires-Dist: allure-python-commons; extra == "allure"

{jolt-0.9.328 → jolt-0.9.332}/jolt/cache.py

@@ -879,6 +879,28 @@ class Artifact(object):
         return self._node.task
 
 
+class ArtifactToolsProxy(object):
+    def __init__(self, artifact, tools):
+        self._artifact = artifact
+        self._tools = tools
+
+    def __getattr__(self, name):
+        if name == "tools":
+            return self._tools
+        if name == "_artifact":
+            return self._artifact
+        attr = getattr(self._artifact.__class__, name, None)
+        if attr is not None and callable(attr):
+            return attr.__get__(self, ArtifactToolsProxy)
+        return getattr(self._artifact, name)
+
+    def __setattr__(self, name, value):
+        if name == "_artifact" or name == "_tools":
+            super(ArtifactToolsProxy, self).__setattr__(name, value)
+        else:
+            setattr(self._artifact, name, value)
+
+
 class Context(object):
     """
     Execution context and dependency wrapper.
@@ -904,18 +926,14 @@ class Context(object):
             for dep in reversed(self._node.children):
                 for artifact in dep.artifacts:
                     # Create clone with tools from this task
-                    artifact = self._cache.get_artifact(
-                        dep,
-                        name=artifact.name,
-                        session=artifact.is_session(),
-                        tools=self._node.tools,
-                    )
+                    artifact = ArtifactToolsProxy(artifact, self._node.tools)
 
                     # Don't include session artifacts that don't exist,
                     # i.e. where no build has taken place due to presence
                     # of the persistent artifacts.
-                    if artifact.is_session() and not self._cache.is_available_locally(artifact):
-                        continue
+                    if not dep.is_resource():
+                        if artifact.is_session() and not self._cache.is_available_locally(artifact):
+                            continue
 
                     self._cache.unpack(artifact)
 

{jolt-0.9.328 → jolt-0.9.332}/jolt/graph.py

@@ -1,4 +1,4 @@
-from contextlib import contextmanager, ExitStack
+from contextlib import contextmanager, ExitStack, nullcontext
 import copy
 import hashlib
 from os import getenv
@@ -144,7 +144,11 @@ class TaskProxy(object):
 
     @property
     def instance(self):
-        return self.task._instance.value
+        return self.task.instance
+
+    @instance.setter
+    def instance(self, value):
+        self.task.instance = value
 
     @property
     def is_unstable(self):
@@ -183,7 +187,7 @@ class TaskProxy(object):
         return len(self.ancestors) > 0
 
     def has_artifact(self):
-        return self.is_cacheable() and not self.is_resource() and not self.is_alias()
+        return self.is_cacheable() and not self.is_alias()
 
     def has_extensions(self):
         return len(self.extensions) > 0
@@ -212,12 +216,14 @@ class TaskProxy(object):
     def is_alias(self):
         return isinstance(self.task, Alias)
 
-    def is_available_locally(self, extensions=True):
+    def is_available_locally(self, extensions=True, persistent_only=True):
         dep_artifacts = []
         if extensions:
             for dep in self.extensions:
                 dep_artifacts += dep.artifacts
-        artifacts = filter(lambda a: not a.is_session(), self._artifacts + dep_artifacts)
+        artifacts = self._artifacts + dep_artifacts
+        if persistent_only:
+            artifacts = filter(lambda a: not a.is_session(), artifacts)
         return all(map(self.cache.is_available_locally, artifacts))
 
     def is_available_remotely(self, extensions=True, cache=True):
@@ -250,6 +256,12 @@ class TaskProxy(object):
     def is_goal(self, with_extensions=True):
         return self._goal or (with_extensions and any([e.is_goal() for e in self.extensions]))
 
+    def is_local(self):
+        if self.is_extension():
+            return self.get_extended_task().is_local()
+        tasks = [self.task] + [e.task for e in self.extensions]
+        return any([task.local for task in tasks])
+
     def in_progress(self):
         return self._in_progress
 
@@ -316,7 +328,7 @@ class TaskProxy(object):
         self._download = False
 
     def download(self, force=False, session_only=False, persistent_only=False):
-        if not self.is_downloadable():
+        if not force and not self.is_downloadable():
             return True
         artifacts = self._artifacts
         if session_only:
@@ -420,12 +432,13 @@ class TaskProxy(object):
         self._finalized = True
         self.identity
 
-        self._artifacts.extend(self.task._artifacts(self.cache, self))
-
         hooks.task_created(self)
 
         return self.identity
 
+    def finalize_artifacts(self):
+        self._artifacts.extend(self.task._artifacts(self.cache, self))
+
     def taint(self, salt=None):
         self.task.taint = self.task.taint or salt or uuid.uuid4()
         if salt is None:
@@ -570,21 +583,20 @@ class TaskProxy(object):
         for child in self.children:
             if not child.has_artifact():
                 continue
+
+            if child.is_resource() and child.is_local() and child.options.worker:
+                return raise_task_error_if(
+                    not child.download(force=True),
+                    child, "Failed to download task artifact")
+
             raise_task_error_if(
                 not child.is_completed() and child.is_unstable,
                 self, "Task depends on failed task '{}'", child.short_qualified_name)
             if not child.is_available_locally(extensions=False):
                 raise_task_error_if(
-                    not child.download(persistent_only=True),
+                    not child.download(force=True),
                     child, "Failed to download task artifact")
 
-    def _run_prepare_resources(self, cache, force_upload=False, force_build=False):
-        from jolt.scheduler import ExecutorRegistry, JoltEnvironment
-
-        for child in filter(lambda task: task.is_resource() and not task.is_completed(), reversed(self.children)):
-            executor = ExecutorRegistry.get().create_local(child)
-            executor.run(JoltEnvironment(cache=cache))
-
     def partially_available_locally(self):
         availability = map(lambda a: self.cache.is_available_locally(a), self.artifacts)
         return any(availability) and not all(availability)
@@ -607,16 +619,126 @@ class TaskProxy(object):
                 arch != platform_arch,
                 self, f"Task is not runnable on current platform (wants node.arch={arch})")
 
-    def run(self, cache, force_upload=False, force_build=False):
+    def run_acquire(self, artifact, owner, log_prefix=False):
+        """
+        Acquires a resource and publishes its artifact.
+
+        The artifact is published to the cache even if the acquisition fails.
+        """
+        try:
+            if not isinstance(self.task, WorkspaceResource):
+                ts = utils.duration()
+                log.info(colors.blue("Resource acquisition started ({} for {})"), self.short_qualified_name, owner.short_qualified_name)
+
+            try:
+                with log.thread_prefix(owner.identity[:8]) if log_prefix else nullcontext():
+                    acquire = getattr(self.task, "acquire_" + artifact.name) if artifact.name != "main" else self.task.acquire
+                    acquire(artifact, self.deps, self.tools, owner.task)
+            finally:
+                # Always commit the resource session artifact to the cache, even if the acquisition failed.
+                self.cache.commit(artifact)
+
+            if not isinstance(self.task, WorkspaceResource):
+                log.info(colors.green("Resource acquisition finished after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+
+        except (KeyboardInterrupt, Exception) as e:
+            if not isinstance(self.task, WorkspaceResource):
+                log.error(colors.red("Resource acquisition failed after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+            if self.task.release_on_error:
+                with utils.ignore_exception():
+                    self.run_release(artifact, owner)
+            raise e
+
+    def run_release(self, artifact, owner, log_prefix=False):
+        """
+        Releases a resource.
+        """
+        try:
+            if not isinstance(self.task, WorkspaceResource):
+                ts = utils.duration()
+                log.info(colors.blue("Resource release started ({} for {})"), self.short_qualified_name, owner.short_qualified_name)
+
+            with log.thread_prefix(owner.identity[:8]) if log_prefix else nullcontext():
+                release = getattr(self.task, "release_" + artifact.name) if artifact.name != "main" else self.task.release
+                release(artifact, self.deps, self.tools, owner.task)
+
+            if not isinstance(self.task, WorkspaceResource):
+                log.info(colors.green("Resource release finished after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+
+        except (KeyboardInterrupt, Exception) as e:
+            if not isinstance(self.task, WorkspaceResource):
+                log.error(colors.red("Resource release failed after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+            raise e
+
+    @contextmanager
+    def run_resources(self):
+        """
+        Acquires and releases resources for the task.
+
+        The method is called by executors before invoking the task proxy's run() method.
+        Resource dependencies are acquired and released in reverse order. If an acquisition fails,
+        already acquired resources are released in reverse order and the exception is propagated
+        to the caller.
+
+        Resource artifacts are always published and uploaded if the acquisition has been started,
+        even if the acquisition fails. That way, a failed acquisition can be debugged.
+        """
+
+        # Log messages are prefixed with task identity if resources are acquired in parallel
+        log_prefix = False
+
+        # Collect list of resource dependencies
+        resource_deps = [child for child in self.children if child.is_resource()]
+
+        if self.options.worker:
+            # Exclude local resources when running as worker. They are already acquired by the client.
+            resource_deps = [child for child in resource_deps if not child.is_local()]
+        elif self.options.network and not self.is_local():
+            # Exclude non-local resources in the client when running a network build.
+            # They are acquired by the remote worker.
+            resource_deps = [child for child in resource_deps if child.is_local()]
+            log_prefix = True
+
+        exitstack = ExitStack()
+        acquired = []
+        try:
+            # Acquire resource dependencies in reverse order.
+            for resource in reversed(resource_deps):
+                # Download resource dependencies if not already done.
+                resource._run_download_dependencies(self.cache, force_upload=False, force_build=False)
+
+                with resource.lock_artifacts(discard=False):
+                    resource.deps = self.cache.get_context(resource)
+                    exitstack.enter_context(resource.deps)
+
+                    # Just like tasks, a resource may have multiple artifacts. Run acquire for each artifact.
+                    for artifact in resource.artifacts:
+                        try:
+                            resource.run_acquire(artifact, self, log_prefix=log_prefix)
+                            acquired.append(resource)
+                        finally:
+                            # Always upload the artifact session artifact to the cache, even if the acquisition failed.
+                            if not resource.is_workspace_resource():
+                                resource.upload(locked=False, session_only=True, artifacts=[artifact])
+
+            yield
+
+        finally:
+            for resource in reversed(acquired):
+                for artifact in resource.artifacts:
+                    resource.run_release(artifact, self, log_prefix=log_prefix)
+            exitstack.close()
+
+    def run(self, env, force_upload=False, force_build=False):
+        cache = env.cache
+        queue = env.queue
+
         with self.tools:
             available_locally = available_remotely = False
 
             # Download dependency artifacts if not already done
             self._run_download_dependencies(cache, force_upload, force_build)
 
-            # Prepare resources if not already done. They are not acquired yet.
-            self._run_prepare_resources(cache, force_upload, force_build)
-
             # Check if task artifact is available locally or remotely,
             # either skip execution or download it if necessary.
             if not force_build:
@@ -689,10 +811,14 @@ class TaskProxy(object):
                             raise e
 
                         except Exception as e:
-                            self.failed_execution()
+                            self.failed_execution(interrupt=queue.is_aborted() if queue else False)
+
                             with utils.ignore_exception():
                                 exitstack.close()
 
+                            if queue is not None and queue.is_aborted():
+                                raise KeyboardInterrupt()
+
                             if cli.debug_enabled:
                                 import pdb
                                 extype, value, tb = sys.exc_info()
@@ -734,7 +860,7 @@ class TaskProxy(object):
 
             for extension in self.extensions:
                 with hooks.task_run(extension):
-                    extension.run(cache, force_upload, force_build)
+                    extension.run(env, force_upload, force_build)
 
     def publish(self, context, artifact, buildlog=None):
         hooks.task_prepublish(self, artifact, self.tools)
@@ -929,7 +1055,7 @@ class GraphBuilder(object):
         self.progress = progress
         self.options = options or JoltOptions()
 
-    def _get_node(self, progress, name):
+    def _get_node(self, progress, name, parent=None):
         name = utils.stable_task_name(name)
         node = self.nodes.get(name)
         if not node:
@@ -938,8 +1064,12 @@ class GraphBuilder(object):
             if node is not None:
                 return node
             node = TaskProxy(task, self.graph, self.cache, self.options)
-            self.nodes[node.short_qualified_name] = node
-            self.nodes[node.qualified_name] = node
+            if not node.is_resource():
+                self.nodes[node.short_qualified_name] = node
+                self.nodes[node.qualified_name] = node
+            elif parent:
+                # A resource inherits its instance uuid from the consuming task
+                node.instance = parent.instance
             if self.options.salt:
                 node.taint(self.options.salt)
             self._build_node(progress, node)
@@ -950,7 +1080,7 @@ class GraphBuilder(object):
         self.graph.add_node(node)
 
         if node.task.extends:
-            extended_node = self._get_node(progress, node.task.extends)
+            extended_node = self._get_node(progress, node.task.extends, parent=node)
             self.graph.add_edges_from([(node, extended_node)])
             node.set_extended_task(extended_node)
             extended_node.add_extension(node)
@@ -961,7 +1091,8 @@ class GraphBuilder(object):
 
         for requirement in node.task.requires:
             alias, artifact, task, name = utils.parse_aliased_task_name(requirement)
-            child = self._get_node(progress, utils.format_task_name(task, name))
+            child = self._get_node(progress, utils.format_task_name(task, name), parent=node)
+
             # Create direct edges from alias parents to alias children
             if child.is_alias():
                 for child_child in child.children:
@@ -997,6 +1128,11 @@ class GraphBuilder(object):
                     node.finalize(self.graph, self.manifest)
                     p.update(1)
 
+                # Create artifacts in forward order so that parent identities are available
+                # when creating resource artifacts that depend on them.
+                for node in topological_nodes:
+                    node.finalize_artifacts()
+
             max_time = 0
             min_time = 0
             for node in topological_nodes:

{jolt-0.9.328 → jolt-0.9.332}/jolt/log.py

@@ -143,6 +143,25 @@ class TqdmStream(object):
         getattr(self.stream, 'flush', lambda: None)()
 
 
+class ThreadPrefix(logging.LoggerAdapter):
+    thread_prefix = {}
+
+    def process(self, msg, kwargs):
+        tid = threading.current_thread()
+        if tid in self.thread_prefix:
+            msg = f"[{self.thread_prefix[tid]}] {msg}"
+        return msg, kwargs
+
+    def set_thread_prefix(self, prefix):
+        tid = threading.current_thread()
+        self.thread_prefix[tid] = prefix
+
+    def clear_thread_prefix(self):
+        tid = threading.current_thread()
+        if tid in self.thread_prefix:
+            del self.thread_prefix[tid]
+
+
 # silence root logger
 _root = logging.getLogger()
 _root.setLevel(logging.CRITICAL)
@@ -175,6 +194,7 @@ _stderr.addFilter(Filter(lambda r: r.levelno >= ERROR or r.levelno == EXCEPTION)
 
 _logger.addHandler(_stdout)
 _logger.addHandler(_stderr)
+_logger_frontend = ThreadPrefix(_logger, {})
 
 _file_formatter = Formatter('{asctime} [{levelname:>7}] {message}')
 
@@ -212,35 +232,35 @@ def log(level, message, created=None, context=None, prefix=False):
 
 
 def info(fmt, *args, **kwargs):
-    _logger.log(INFO, fmt, *args, **kwargs)
+    _logger_frontend.log(INFO, fmt, *args, **kwargs)
 
 
 def warning(fmt, *args, **kwargs):
-    _logger.log(WARNING, fmt, *args, **kwargs)
+    _logger_frontend.log(WARNING, fmt, *args, **kwargs)
 
 
 def verbose(fmt, *args, **kwargs):
-    _logger.log(VERBOSE, fmt, *args, **kwargs)
+    _logger_frontend.log(VERBOSE, fmt, *args, **kwargs)
 
 
 def debug(fmt, *args, **kwargs):
-    _logger.log(DEBUG, fmt, *args, **kwargs)
+    _logger_frontend.log(DEBUG, fmt, *args, **kwargs)
 
 
 def error(fmt, *args, **kwargs):
-    _logger.log(ERROR, fmt, *args, **kwargs)
+    _logger_frontend.log(ERROR, fmt, *args, **kwargs)
 
 
 def stdout(line, **kwargs):
     line = line.replace("{", "{{")
     line = line.replace("}", "}}")
-    _logger.log(STDOUT, line, extra=kwargs)
+    _logger_frontend.log(STDOUT, line, extra=kwargs)
 
 
 def stderr(line, **kwargs):
     line = line.replace("{", "{{")
     line = line.replace("}", "}}")
-    _logger.log(STDERR, line, extra=kwargs)
+    _logger_frontend.log(STDERR, line, extra=kwargs)
 
 
 def format_exception_msg(exc):
@@ -273,7 +293,7 @@ def format_exception_msg(exc):
 def exception(exc=None, error=True):
     if exc:
         if error:
-            _logger.log(ERROR, format_exception_msg(exc))
+            _logger_frontend.log(ERROR, format_exception_msg(exc))
 
         tb = traceback.format_exception(type(exc), value=exc, tb=exc.__traceback__)
         installdir = fs.path.dirname(__file__)
@@ -288,7 +308,7 @@ def exception(exc=None, error=True):
         line = line.replace("{", "{{")
         line = line.replace("}", "}}")
         line = line.strip()
-        _logger.log(EXCEPTION, line)
+        _logger_frontend.log(EXCEPTION, line)
 
 
 def transfer(line, context):
@@ -308,7 +328,7 @@ def transfer(line, context):
     elif line.startswith("[ EXCEPT]"):
         outline1 = outline1.replace("{", "{{")
         outline1 = outline1.replace("}", "}}")
-        _logger.log(EXCEPTION, outline1)
+        _logger_frontend.log(EXCEPTION, outline1)
     elif line.startswith("[ STDERR]"):
         stderr(outline1, prefix=True)
     elif line.startswith("[ STDOUT]"):
@@ -491,6 +511,15 @@ def map_thread(thread_from, thread_to):
         _thread_map.unmap(tid)
 
 
+@contextmanager
+def thread_prefix(prefix):
+    try:
+        _logger_frontend.set_thread_prefix(prefix)
+        yield
+    finally:
+        _logger_frontend.clear_thread_prefix()
+
+
 class _LogStream(object):
     def __init__(self):
         self.buf = ""

{jolt-0.9.328 → jolt-0.9.332}/jolt/plugins/docker.py

@@ -25,8 +25,6 @@ class DockerListVariable(ArtifactListAttribute):
 
 class DockerLoadListVariable(DockerListVariable):
     def apply(self, task, artifact):
-        if isinstance(task, Resource):
-            return
         for image in self.items():
             task.tools.run(
                 "docker load -i {}",
@@ -35,16 +33,12 @@ class DockerLoadListVariable(DockerListVariable):
 
 class DockerPullListVariable(DockerListVariable):
     def apply(self, task, artifact):
-        if isinstance(task, Resource):
-            return
         for image in self.items():
             task.tools.run("docker pull {}", image, output_on_error=True)
 
 
 class DockerRmiListVariable(DockerListVariable):
     def unapply(self, task, artifact):
-        if isinstance(task, Resource):
-            return
         for image in self.items():
             task.tools.run("docker rmi -f {}", image, output_on_error=True)
 
@@ -499,6 +493,9 @@ class DockerImage(Task):
     """
     abstract = True
 
+    annotations = []
+    """ A list of image annotations """
+
     autoload = True
     """
     Automatically load image file into local registry when the artifact is
@@ -585,6 +582,10 @@ class DockerImage(Task):
         with _Tarfile.open(layerpath, 'r') as tar:
             tar.extractall(targetpath)
 
+    @property
+    def _annotations(self):
+        return " ".join([utils.option("--annotation ", self.tools.expand(an)) for an in self.annotations])
+
     @property
     def _buildargs(self):
         return " ".join([utils.option("--build-arg ", self.tools.expand(ba)) for ba in self.buildargs])
@@ -626,7 +627,7 @@ class DockerImage(Task):
                   tools.expand_relpath(context))
 
         with tools.cwd(context):
-            tools.run("docker build {_platform} . -f {} {_buildargs} {_labels} {_tags} {pull}{squash}",
+            tools.run("docker build {_platform} . -f {} {_annotations} {_buildargs} {_labels} {_tags} {pull}{squash}",
                       utils.quote(dockerfile), pull=pull, squash=squash)
 
         try: