jolt 0.9.172__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/graph.py

@@ -1,12 +1,16 @@
-from contextlib import contextmanager, ExitStack
+from contextlib import contextmanager, ExitStack, nullcontext
 import copy
 import hashlib
 from os import getenv
 from threading import RLock
 from collections import OrderedDict
 import uuid
+import socket
+import sys
 
+from jolt import cli
 from jolt import common_pb2 as common_pb
+from jolt import config
 from jolt import log
 from jolt import utils
 from jolt import colors
@@ -16,7 +20,7 @@ from jolt.error import raise_error_if
 from jolt.error import raise_task_error_if
 from jolt.influence import HashInfluenceRegistry, TaskRequirementInfluence
 from jolt.options import JoltOptions
-from jolt.tasks import Alias, Resource, WorkspaceResource, Task
+from jolt.tasks import Alias, Resource, WorkspaceResource
 
 
 class TaskProxy(object):
@@ -56,8 +60,13 @@ class TaskProxy(object):
         self._network = False
         self._artifacts = []
         self._status = None
+        self._finalized = False
 
-        hooks.task_created(self)
+        # Consumer task if this is a resource
+        self._owner = None
+
+        # List of all artifacts that are produced by this task
+        self._artifacts = []
 
     def __hash__(self):
         return id(self)
@@ -66,6 +75,15 @@ class TaskProxy(object):
     def artifacts(self):
         return self._artifacts
 
+    def add_artifact(self, artifact):
+        self._artifacts.append(artifact)
+
+    def get_artifact(self, name):
+        for artifact in self.artifacts:
+            if artifact.name == name:
+                return artifact
+        return None
+
     @property
     def tools(self):
         return self.task.tools
@@ -86,12 +104,33 @@ class TaskProxy(object):
     def short_qualified_name(self):
         return self.task.short_qualified_name
 
+    @property
+    def exported_name(self):
+        return self.task.exported_name
+
     @property
     def log_name(self):
         return "({0} {1})".format(self.short_qualified_name, self.identity[:8])
 
+    def log_running_time(self):
+        """ Emits an information log line every 5 mins a task has been running. """
+        if not self.is_running():
+            return
+        minutes = int(self.duration_running.seconds / 60)
+        if (minutes % 5) == 0 and minutes > 0:
+            if minutes >= 60:
+                fmt = f"{int(minutes / 60)}h {int(minutes % 60)}min"
+            else:
+                fmt = f"{int(minutes % 60)}min"
+            if self.is_remotely_executed():
+                self.info("Remote execution still in progress after {}", fmt)
+            else:
+                self.info("Execution still in progress after {}", fmt)
+
     @property
     def identity(self):
+        raise_task_error_if(not self._finalized, self, "Task identity read prematurely")
+
         if self.task.identity is not None:
             return self.task.identity
 
@@ -112,7 +151,11 @@ class TaskProxy(object):
 
     @property
     def instance(self):
-        return self.task._instance.value
+        return self.task.instance
+
+    @instance.setter
+    def instance(self, value):
+        self.task.instance = value
 
     @property
     def is_unstable(self):
@@ -132,6 +175,9 @@ class TaskProxy(object):
     def info(self, fmt, *args, **kwargs):
         self.task.info(fmt + " " + self.log_name, *args, **kwargs)
 
+    def debug(self, fmt, *args, **kwargs):
+        log.debug(fmt + " " + self.log_name, *args, **kwargs)
+
     def verbose(self, fmt, *args, **kwargs):
         log.verbose(fmt + " " + self.log_name, *args, **kwargs)
 
@@ -148,7 +194,7 @@ class TaskProxy(object):
         return len(self.ancestors) > 0
 
     def has_artifact(self):
-        return self.is_cacheable() and not self.is_resource() and not self.is_alias()
+        return self.is_cacheable() and not self.is_alias()
 
     def has_extensions(self):
         return len(self.extensions) > 0
@@ -170,19 +216,21 @@ class TaskProxy(object):
         for c in self.children:
             if c.is_resource() or c.is_alias():
                 continue
-            if not c.is_available_locally():
+            if not c.is_available_locally(persistent_only=True):
                 return False
         return True
 
     def is_alias(self):
         return isinstance(self.task, Alias)
 
-    def is_available_locally(self, extensions=True, skip_session=False):
+    def is_available_locally(self, extensions=True, persistent_only=True):
         dep_artifacts = []
         if extensions:
             for dep in self.extensions:
                 dep_artifacts += dep.artifacts
-        artifacts = filter(lambda a: not a.is_session(), self._artifacts + dep_artifacts)
+        artifacts = self._artifacts + dep_artifacts
+        if persistent_only:
+            artifacts = filter(lambda a: not a.is_session(), artifacts)
         return all(map(self.cache.is_available_locally, artifacts))
 
     def is_available_remotely(self, extensions=True, cache=True):
@@ -215,6 +263,12 @@ class TaskProxy(object):
     def is_goal(self, with_extensions=True):
         return self._goal or (with_extensions and any([e.is_goal() for e in self.extensions]))
 
+    def is_local(self):
+        if self.is_extension():
+            return self.get_extended_task().is_local()
+        tasks = [self.task] + [e.task for e in self.extensions]
+        return any([task.local for task in tasks])
+
     def in_progress(self):
         return self._in_progress
 
@@ -236,8 +290,18 @@ class TaskProxy(object):
     def is_resource(self):
         return isinstance(self.task, Resource)
 
+    def is_workspace_resource(self):
+        return isinstance(self.task, WorkspaceResource)
+
+    def is_running(self):
+        return self.status() == common_pb.TaskStatus.TASK_RUNNING
+
     def is_unpackable(self):
-        return self.task.unpack.__func__ is not Task.unpack
+        tasks = [self] + self.extensions
+        artifacts = []
+        for task in tasks:
+            artifacts.extend(task._artifacts)
+        return any(map(lambda artifact: artifact.is_unpackable(), artifacts))
 
     def is_unpacked(self):
         tasks = [self] + self.extensions
@@ -254,17 +318,14 @@ class TaskProxy(object):
                 artifacts.extend(task._artifacts)
         return all(map(lambda artifact: artifact.is_uploadable(), artifacts))
 
-    def is_workspace_resource(self):
-        return isinstance(self.task, WorkspaceResource)
-
     @contextmanager
     def lock_artifacts(self, discard=False):
         artifacts = []
         stack = ExitStack()
-        for artifact in self.artifacts:
-            lock = self.cache.lock_artifact(artifact, discard=discard)
-            artifacts.append(stack.enter_context(lock))
         try:
+            for artifact in self.artifacts:
+                lock = self.cache.lock_artifact(artifact, discard=discard)
+                artifacts.append(stack.enter_context(lock))
             self._artifacts = artifacts
             yield artifacts
         finally:
@@ -274,23 +335,39 @@ class TaskProxy(object):
         self._download = False
 
     def download(self, force=False, session_only=False, persistent_only=False):
-        if not self.is_downloadable():
-            return True
-        artifacts = self._artifacts
-        if session_only:
-            artifacts = list(filter(lambda a: a.is_session(), self._artifacts))
-        if persistent_only:
-            artifacts = list(filter(lambda a: not a.is_session(), self._artifacts))
-        if not artifacts:
-            return True
-        return all([self.cache.download(artifact, force=force) for artifact in artifacts])
+        """
+        Downloads all artifacts of this task.
+
+        If the task is not downloadable, the method returns True. Failure to
+        download persistent artifacts is considered a failure, and the method
+        returns False. Session artifacts are not required to be downloaded.
 
-    def upload(self, force=False, locked=False, session_only=False, persistent_only=False):
+        :param force: Force download even if the artifacts are already available.
+        :param session_only: Download only session artifacts.
+        :param persistent_only: Download only persistent artifacts.
+
+        """
+        if not force and not self.is_downloadable():
+            return True
+        success = True
         artifacts = self._artifacts
+        artifacts_session = list(filter(lambda a: a.is_session(), artifacts))
+        artifacts_persistent = list(filter(lambda a: not a.is_session(), artifacts))
+        download_all = not session_only and not persistent_only
+        if session_only or download_all:
+            for artifact in artifacts_session:
+                if not self.cache.download(artifact, force=force):
+                    self.warning("Failed to download session artifact: {}", artifact.identity)
+        if persistent_only or download_all:
+            success = all([self.cache.download(artifact, force=force) for artifact in artifacts_persistent])
+        return success
+
+    def upload(self, force=False, locked=False, session_only=False, persistent_only=False, artifacts=None):
+        artifacts = artifacts or self._artifacts
         if session_only:
-            artifacts = list(filter(lambda a: a.is_session(), self._artifacts))
+            artifacts = list(filter(lambda a: a.is_session(), artifacts))
         if persistent_only:
-            artifacts = list(filter(lambda a: not a.is_session(), self._artifacts))
+            artifacts = list(filter(lambda a: not a.is_session(), artifacts))
         if not artifacts:
             return True
         if not self.is_uploadable(artifacts):
@@ -300,6 +377,9 @@ class TaskProxy(object):
     def resolve_requirement_alias(self, name):
         return self.requirement_aliases.get(name)
 
+    def set_cancelled(self):
+        self.set_status(common_pb.TaskStatus.TASK_CANCELLED)
+
     def set_passed(self):
         self.set_status(common_pb.TaskStatus.TASK_PASSED)
 
@@ -340,9 +420,12 @@ class TaskProxy(object):
     def set_goal(self):
         self._goal = True
 
-    def finalize(self, dag, manifest):
+    def set_owner(self, owner):
+        self._owner = owner
+        self.task.exported_name = f"{self.short_qualified_name}@@{owner.short_qualified_name}"
+
+    def finalize(self, dag):
         log.debug("Finalizing: " + self.short_qualified_name)
-        self.manifest = manifest
 
         # Find all direct and transitive dependencies
         self.ancestors = set()
@@ -358,25 +441,49 @@ class TaskProxy(object):
                 self.children.extend(n.children)
             n.ancestors.add(self)
 
-        # Exclude transitive alias and resources dependencies
+        # Exclude transitive alias and resources dependencies.
+        # Workspace resources are included as they may be required by its dependencies.
         self.children = list(
-            filter(lambda n: not n.is_alias() and (not n.is_resource() or dag.are_neighbors(self, n)),
+            filter(lambda n: dag.are_neighbors(self, n) or (not n.is_alias() and not n.is_resource()),
                    utils.unique_list(self.children)))
 
+        # Prepare workspace resources for this task so that influence can be calculated
+        for child in self.children:
+            if not child.is_workspace_resource():
+                continue
+            child.task.prepare_ws_for(self.task)
+
         self.descendants = list(self.descendants)
 
         self.task.influence += [TaskRequirementInfluence(n) for n in self.neighbors]
+        self._finalized = True
         self.identity
-        self._artifacts = self.task._artifacts(self.cache, self)
+
+        hooks.task_created(self)
 
         return self.identity
 
+    def finalize_artifacts(self):
+        self._artifacts.extend(self.task._artifacts(self.cache, self))
+
     def taint(self, salt=None):
-        self.task.taint = self.task.taint or salt or uuid.uuid4()
+        self.task.taint = salt or uuid.uuid4()
         if salt is None:
             # Only recalculate identity when build is forced, not when salted
             self.identity = None
             self.identity
+            # Recreate artifacts
+            self._artifacts = []
+            self.finalize_artifacts()
+
+        # If this is an alias, taint all children
+        if self.is_alias():
+            for child in self.children:
+                child.taint()
+
+        # Taint all extensions
+        for extension in self.extensions:
+            extension.taint()
 
     def queued(self, remote=True):
         self.task.verbose("Task queued " + self.log_name)
@@ -392,9 +499,15 @@ class TaskProxy(object):
         self.duration_running = utils.duration() if not when else when
 
     def running_execution(self, remote=False):
+        self.worker = socket.gethostname()
         hooks.task_started_execution(self)
         self.running(what="Remote execution" if remote else "Execution")
 
+    def restarted_execution(self, remote=False):
+        hooks.task_finished_execution(self)
+        self.task.warning("Remote execution interrupted {}" if remote else "Execution interrupted {}", self.log_name)
+        self.queued()
+
     def started_execution(self, remote=False):
         self.queued()
         self.running_execution(remote=remote)
@@ -409,16 +522,24 @@ class TaskProxy(object):
         self.running(what="Upload")
         hooks.task_started_upload(self)
 
-    def _failed(self, what="Execution"):
-        self.set_failed()
+    def _failed(self, what="Execution", interrupt=False):
+        if interrupt:
+            how = "interrupted"
+            logfn = self.warning
+            self.set_cancelled()
+        else:
+            how = "failed"
+            logfn = self.error
+            self.set_failed()
+
         if self.duration_queued and self.duration_running:
-            self.error("{0} failed after {1} {2}", what,
-                       self.duration_running,
-                       self.duration_queued.diff(self.duration_running))
+            logfn("{0} {1} after {2} {3}", what, how,
+                  self.duration_running,
+                  self.duration_queued.diff(self.duration_running))
         elif self.duration_queued:
-            self.error("{0} failed after {1}", what, self.duration_queued or utils.duration())
+            logfn("{0} {1} after {2}", what, how, self.duration_queued or utils.duration())
         else:
-            self.error("{0} failed immediately", what)
+            logfn("{0} {1} immediately", what, how)
 
         if self.is_unstable:
             try:
@@ -437,8 +558,8 @@ class TaskProxy(object):
     def failed_upload(self):
         self._failed("Upload")
 
-    def failed_execution(self, remote=False):
-        self._failed(what="Remote execution" if remote else "Execution")
+    def failed_execution(self, remote=False, interrupt=False):
+        self._failed(what="Remote execution" if remote else "Execution", interrupt=interrupt)
 
     def _finished(self, what="Execution"):
         raise_task_error_if(
@@ -450,7 +571,7 @@ class TaskProxy(object):
         except KeyError:
             self.warning("Pruned task was executed")
         self.task.info(colors.green(what + " finished after {0} {1}" + self.log_name),
-                       self.duration_running,
+                       self.duration_running or "00s",
                        self.duration_queued.diff(self.duration_running))
         hooks.task_finished(self)
 
@@ -497,61 +618,220 @@ class TaskProxy(object):
                 else:
                     log.debug(" Retained: {} ({})", self.short_qualified_name, artifact.identity)
 
-    def _run_download_dependencies(self, cache, force_upload=False, force_build=False):
+    def _run_download_dependencies(self, resource_only=False):
         for child in self.children:
             if not child.has_artifact():
                 continue
+
+            if child.is_resource() and child.is_local():
+                if child.options.worker:
+                    # Resource already acquired by the client when running as worker
+                    raise_task_error_if(
+                        not child.download(force=True),
+                        child, "Failed to download task artifact")
+                else:
+                    # Resource about to be acquired by the client
+                    child._run_download_dependencies()
+                continue
+
+            if resource_only and not child.is_resource():
+                continue
+
             raise_task_error_if(
                 not child.is_completed() and child.is_unstable,
                 self, "Task depends on failed task '{}'", child.short_qualified_name)
             if not child.is_available_locally(extensions=False):
                 raise_task_error_if(
-                    not child.download(persistent_only=True),
+                    not child.download(force=True),
                     child, "Failed to download task artifact")
 
-    def _run_prepare_resources(self, cache, force_upload=False, force_build=False):
-        from jolt.scheduler import ExecutorRegistry, JoltEnvironment
+    def partially_available_locally(self):
+        availability = map(lambda a: self.cache.is_available_locally(a), self.artifacts)
+        return any(availability) and not all(availability)
 
-        for child in filter(lambda task: task.is_resource() and not task.is_completed(), reversed(self.children)):
-            executor = ExecutorRegistry.get().create_local(child)
-            executor.run(JoltEnvironment(cache=cache))
+    def _validate_platform(self):
+        """ Validates that the task is runnable on the current platform. """
+        platform_os, platform_arch = utils.platform_os_arch()
 
-    def run(self, cache, force_upload=False, force_build=False):
-        with self.tools:
-            available_locally = available_remotely = False
+        os = self.task.platform.get("node.os")
+        if os:
+            os = self.tools.expand(os)
+            raise_task_error_if(
+                os != platform_os,
+                self, f"Task is not runnable on current platform (wants node.os={os})")
+
+        arch = self.task.platform.get("node.arch")
+        if arch:
+            arch = self.tools.expand(arch)
+            raise_task_error_if(
+                arch != platform_arch,
+                self, f"Task is not runnable on current platform (wants node.arch={arch})")
 
-            # Download dependency artifacts if not already done
-            self._run_download_dependencies(cache, force_upload, force_build)
+    def run_acquire(self, artifact, owner, log_prefix=False):
+        """
+        Acquires a resource and publishes its artifact.
 
-            # Prepare resources if not already done. They are not acquired yet.
-            self._run_prepare_resources(cache, force_upload, force_build)
+        The artifact is published to the cache even if the acquisition fails.
+        """
+
+        try:
+            if not self.is_workspace_resource():
+                ts = utils.duration()
+                log.info(colors.blue("Resource acquisition started ({} for {})"), self.short_qualified_name, owner.short_qualified_name)
+
+            try:
+                with log.thread_prefix(owner.identity[:8]) if log_prefix else nullcontext():
+                    acquire = getattr(self.task, "acquire_" + artifact.name) if artifact.name != "main" else self.task.acquire
+                    acquire(artifact, self.deps, self.tools, owner.task)
+            finally:
+                # Always commit the resource session artifact to the cache, even if the acquisition failed.
+                if not self.is_workspace_resource():
+                    self.cache.commit(artifact)
+
+            if not self.is_workspace_resource():
+                log.info(colors.green("Resource acquisition finished after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+
+        except (KeyboardInterrupt, Exception) as e:
+            if not self.is_workspace_resource():
+                log.error(colors.red("Resource acquisition failed after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+            if self.task.release_on_error:
+                with utils.ignore_exception():
+                    self.run_release(artifact, owner)
+            raise e
+
+    def run_release(self, artifact, owner, log_prefix=False):
+        """
+        Releases a resource.
+        """
+        try:
+            if not self.is_workspace_resource():
+                ts = utils.duration()
+                log.info(colors.blue("Resource release started ({} for {})"), self.short_qualified_name, owner.short_qualified_name)
+
+            with log.thread_prefix(owner.identity[:8]) if log_prefix else nullcontext():
+                release = getattr(self.task, "release_" + artifact.name) if artifact.name != "main" else self.task.release
+                release(artifact, self.deps, self.tools, owner.task)
+
+            if not self.is_workspace_resource():
+                log.info(colors.green("Resource release finished after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+
+        except (KeyboardInterrupt, Exception) as e:
+            if not self.is_workspace_resource():
+                log.error(colors.red("Resource release failed after {} ({} for {})"), ts, self.short_qualified_name, owner.short_qualified_name)
+            raise e
+
+    @contextmanager
+    def run_resources(self):
+        """
+        Acquires and releases resources for the task.
+
+        The method is called by executors before invoking the task proxy's run() method.
+        Resource dependencies are acquired and released in reverse order. If an acquisition fails,
+        already acquired resources are released in reverse order and the exception is propagated
+        to the caller.
+
+        Resource artifacts are always published and uploaded if the acquisition has been started,
+        even if the acquisition fails. That way, a failed acquisition can be debugged.
+        """
+        self._run_download_dependencies(resource_only=True)
+
+        with self._run_resources_no_dep_download():
+            yield
+
+    @contextmanager
+    def _run_resources_no_dep_download(self):
+        # Log messages are prefixed with task identity if resources are acquired in parallel
+        log_prefix = False
+
+        # Collect list of resource dependencies
+        resource_deps = [child for child in self.children if child.is_resource()]
+
+        if self.options.worker:
+            # Exclude local resources when running as worker. They are already acquired by the client.
+            resource_deps = [child for child in resource_deps if not child.is_local()]
+        elif self.options.network and not self.is_local():
+            # Exclude non-local resources in the client when running a network build.
+            # They are acquired by the remote worker.
+            resource_deps = [child for child in resource_deps if child.is_local()]
+            log_prefix = True
+
+        exitstack = ExitStack()
+        acquired = []
+        try:
+            # Acquire resource dependencies in reverse order.
+            for resource in reversed(resource_deps):
+                # Always discard resource artifacts before acquiring the resource.
+                # They should not exist in the cache when the resource is acquired,
+                # but may exist if the resource was previously acquired by an interrupted build.
+                with resource.lock_artifacts(discard=True) if not resource.is_workspace_resource() else nullcontext():
+                    resource.deps = self.cache.get_context(resource)
+                    exitstack.enter_context(resource.deps)
+
+                    # Just like tasks, a resource may have multiple artifacts. Run acquire for each artifact.
+                    for artifact in resource.artifacts:
+                        try:
+                            resource.run_acquire(artifact, self, log_prefix=log_prefix)
+                            acquired.append(resource)
+                        finally:
+                            # Always upload the artifact session artifact to the cache, even if the acquisition failed.
+                            if not resource.is_workspace_resource():
+                                resource.upload(locked=False, session_only=True, artifacts=[artifact])
+
+            yield
+
+        finally:
+            for resource in reversed(acquired):
+                for artifact in resource.artifacts:
+                    resource.run_release(artifact, self, log_prefix=log_prefix)
+            exitstack.close()
+
+    def run(self, env, force_upload=False, force_build=False):
+        # Download dependency artifacts if not already done
+        self._run_download_dependencies()
+
+        with self._run_resources_no_dep_download():
+            self._run_task(env, force_upload, force_build)
+
+    def _run_task(self, env, force_upload=False, force_build=False):
+        queue = env.queue
+
+        with self.tools:
+            available_locally = available_remotely = False
 
             # Check if task artifact is available locally or remotely,
             # either skip execution or download it if necessary.
             if not force_build:
                 available_locally = self.is_available_locally()
                 if available_locally and not force_upload:
+                    self.skipped()
                     return
-                available_remotely = cache.download_enabled() and self.is_available_remotely()
+
+                available_remotely = self.cache.download_enabled() and self.is_available_remotely()
                 if not available_locally and available_remotely:
                     available_locally = self.download()
 
+                if not available_locally and self.partially_available_locally():
+                    force_build = True
+
             if force_build or not available_locally:
                 with log.threadsink() as buildlog:
                     if self.task.is_runnable():
                         log.verbose("Host: {0}", getenv("HOSTNAME", "localhost"))
 
-                    with self.lock_artifacts(discard=force_build) as artifacts:
+                    with self.lock_artifacts(discard=not self.is_resource()) as artifacts:
                         exitstack = ExitStack()
 
                         # Indicates whether session artifacts have been published
                         upload_session_artifacts = False
 
                         try:
-                            context = cache.get_context(self)
+                            context = self.cache.get_context(self)
                             exitstack.enter_context(context)
 
                             self.running_execution()
+
+                            self._validate_platform()
+
                             with self.tools.cwd(self.task.joltdir):
                                 if self.is_goal() and self.options.debug:
                                     log.info("Entering debug shell")
@@ -561,7 +841,8 @@ class TaskProxy(object):
                                     # Run task
                                     try:
                                         hooks.task_prerun(self, context, self.tools)
-                                        self.task.run(context, self.tools)
+                                        with self.tools.timeout(seconds=config.getint("jolt", "task_timeout")):
+                                            self.task.run(context, self.tools)
                                     finally:
                                         hooks.task_postrun(self, context, self.tools)
 
@@ -574,16 +855,34 @@ class TaskProxy(object):
 
                                 finally:
                                     # Session artifacts should be uploaded
-                                    upload_session_artifacts = True
+                                    upload_session_artifacts = []
 
                                     # Publish session artifacts to local cache
                                     for artifact in filter(lambda a: a.is_session(), artifacts):
                                         self.publish(context, artifact)
+                                        upload_session_artifacts.append(artifact)
+
+                        except KeyboardInterrupt as e:
+                            self.failed_execution(interrupt=True)
+                            with utils.ignore_exception():
+                                exitstack.close()
+
+                            raise e
 
                         except Exception as e:
-                            self.failed_execution()
+                            self.failed_execution(interrupt=queue.is_aborted() if queue else False)
+
                             with utils.ignore_exception():
                                 exitstack.close()
+
+                            if queue is not None and queue.is_aborted():
+                                raise KeyboardInterrupt()
+
+                            if cli.debug_enabled:
+                                import pdb
+                                extype, value, tb = sys.exc_info()
+                                pdb.post_mortem(tb)
+
                             raise e
 
                         else:
@@ -595,22 +894,22 @@ class TaskProxy(object):
                             if force_upload or force_build or not available_remotely:
                                 raise_task_error_if(
                                     not self.upload(force=force_upload, locked=False, persistent_only=True) \
-                                    and cache.upload_enabled(),
+                                    and self.cache.upload_enabled(),
                                     self, "Failed to upload task artifact")
 
                         finally:
-                            # Upload session artifacts to remote cache
+                            # Upload published session artifacts to remote cache
                             raise_task_error_if(
                                 upload_session_artifacts \
-                                and not self.upload(force=force_upload, locked=False, session_only=True) \
-                                and cache.upload_enabled(),
+                                and not self.upload(force=force_upload, locked=False, session_only=True, artifacts=upload_session_artifacts) \
+                                and self.cache.upload_enabled(),
                                 self, "Failed to upload session artifact")
 
             elif force_upload or not available_remotely:
                 self.started_upload()
                 raise_task_error_if(
                     not self.upload(force=force_upload, persistent_only=True) \
-                    and cache.upload_enabled(),
+                    and self.cache.upload_enabled(),
                     self, "Failed to upload task artifact")
                 self.finished_upload()
 
@@ -620,7 +919,7 @@ class TaskProxy(object):
 
             for extension in self.extensions:
                 with hooks.task_run(extension):
-                    extension.run(cache, force_upload, force_build)
+                    extension.run(env, force_upload, force_build)
 
     def publish(self, context, artifact, buildlog=None):
         hooks.task_prepublish(self, artifact, self.tools)
@@ -634,9 +933,18 @@ class TaskProxy(object):
         hooks.task_postpublish(self, artifact, self.tools)
         artifact.get_cache().commit(artifact)
 
+    def raise_for_status(self, log_details=False, log_error=False):
+        with self.report() as report:
+            report.raise_for_status(log_details, log_error)
+
     def report(self):
         return self.task.report()
 
+    def unpack(self):
+        """ Unpacks all artifacts produced by this task. """
+        for artifact in self.artifacts:
+            self.cache.unpack(artifact)
+
 
 class Graph(object):
     def __init__(self):
@@ -746,7 +1054,7 @@ class Graph(object):
 
     @property
     def pruned(self):
-        return self._pruned
+        return [p for p in self._pruned]
 
     @property
     def roots(self):
@@ -776,7 +1084,7 @@ class Graph(object):
         with self._mutex:
             log.debug("[GRAPH] Listing all nodes")
             for node in self.topological_nodes:
-                log.debug("[GRAPH]   " + node.qualified_name + " ({})", len(self._children[node].keys()))
+                log.debug("[GRAPH]   " + node.short_qualified_name + " ({})", len(self._children[node].keys()))
 
     def is_leaf(self, node):
         with self._mutex:
@@ -796,27 +1104,31 @@ class Graph(object):
 
 
 class GraphBuilder(object):
-    def __init__(self, registry, cache, manifest=None, options=None, progress=False, buildenv=None):
+    def __init__(self, registry, cache, options=None, progress=False, buildenv=None):
         self.cache = cache
         self.graph = Graph()
         self.nodes = {}
         self.registry = registry
-        self.manifest = manifest
         self.buildenv = buildenv
         self.progress = progress
         self.options = options or JoltOptions()
 
-    def _get_node(self, progress, name):
+    def _get_node(self, progress, name, parent=None):
         name = utils.stable_task_name(name)
         node = self.nodes.get(name)
         if not node:
-            task = self.registry.get_task(name, manifest=self.manifest, buildenv=self.buildenv)
+            task = self.registry.get_task(name, buildenv=self.buildenv)
             node = self.nodes.get(task.qualified_name, None)
             if node is not None:
                 return node
             node = TaskProxy(task, self.graph, self.cache, self.options)
-            self.nodes[node.short_qualified_name] = node
-            self.nodes[node.qualified_name] = node
+            if not node.is_resource() or node.is_workspace_resource():
+                self.nodes[node.short_qualified_name] = node
+                self.nodes[node.qualified_name] = node
+            elif parent:
+                node.set_owner(parent)
+                if self.buildenv:
+                    task._apply_protobuf(self.buildenv)
             if self.options.salt:
                 node.taint(self.options.salt)
             self._build_node(progress, node)
@@ -827,7 +1139,7 @@ class GraphBuilder(object):
         self.graph.add_node(node)
 
         if node.task.extends:
-            extended_node = self._get_node(progress, node.task.extends)
+            extended_node = self._get_node(progress, node.task.extends, parent=node)
             self.graph.add_edges_from([(node, extended_node)])
             node.set_extended_task(extended_node)
             extended_node.add_extension(node)
@@ -837,8 +1149,9 @@ class GraphBuilder(object):
             parent = node
 
         for requirement in node.task.requires:
-            alias, artifact, task, name = utils.parse_aliased_task_name(requirement)
-            child = self._get_node(progress, utils.format_task_name(task, name))
+            alias, _, task, name = utils.parse_aliased_task_name(requirement)
+            child = self._get_node(progress, utils.format_task_name(task, name), parent=node)
+
             # Create direct edges from alias parents to alias children
             if child.is_alias():
                 for child_child in child.children:
@@ -871,9 +1184,14 @@ class GraphBuilder(object):
             topological_nodes = self.graph.topological_nodes
             with self._progress("Collecting task influence", len(self.graph.tasks), "tasks") as p:
                 for node in reversed(topological_nodes):
-                    node.finalize(self.graph, self.manifest)
+                    node.finalize(self.graph)
                     p.update(1)
 
+                # Create artifacts in forward order so that parent identities are available
+                # when creating resource artifacts that depend on them.
+                for node in topological_nodes:
+                    node.finalize_artifacts()
+
             max_time = 0
             min_time = 0
             for node in topological_nodes: