jolt 0.9.76__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/cache.py

@@ -1,6 +1,6 @@
 import atexit
 import contextlib
-from collections import OrderedDict
+from collections import namedtuple, OrderedDict
 from datetime import datetime
 import fasteners
 import json
@@ -10,18 +10,20 @@ from threading import RLock
 import uuid
 
 from jolt import config
+from jolt import expires
 from jolt import filesystem as fs
 from jolt import influence
 from jolt import log
 from jolt import tools
 from jolt import utils
+from jolt import tasks
 from jolt.options import JoltOptions
 from jolt.error import raise_error, raise_error_if
 from jolt.error import raise_task_error, raise_task_error_if
 from jolt.expires import ArtifactEvictionStrategyRegister
 
 
-DEFAULT_ARCHIVE_TYPE = ".tar.gz"
+DEFAULT_ARCHIVE_TYPE = ".tar.zst"
 
 
 def locked(func):
@@ -31,179 +33,122 @@ def locked(func):
     return _f
 
 
-class StorageProvider(object):
-    def download(self, node, force=False):
-        return False
-
-    def download_enabled(self):
-        return True
-
-    def upload(self, node, force=False):
-        return False
-
-    def upload_enabled(self):
-        return True
-
-    def location(self, node):
-        return ''  # URL
-
-
-class StorageProviderFactory(StorageProvider):
-    def create(self):
-        pass
-
-
-def RegisterStorage(cls):
-    ArtifactCache.storage_provider_factories.append(cls)
-
-
-class ArtifactAttributeSet(object):
-    def __init__(self):
-        super(ArtifactAttributeSet, self).__setattr__("_attributes", {})
-
-    def _get_attributes(self):
-        return self._attributes
-
-    def __getattr__(self, name):
-        attributes = self._get_attributes()
-        if name not in attributes:
-            attributes[name] = self.create(name)
-        return attributes[name]
-
-    def __setattr__(self, name, value):
-        attributes = self._get_attributes()
-        if name not in attributes:
-            attributes[name] = self.create(name)
-        attributes[name].set_value(value)
-        return attributes[name]
-
-    def __dict__(self):
-        return {key: str(value) for key, value in self.items()}
-
-    def items(self):
-        return self._get_attributes().items()
-
-    def apply(self, task, artifact):
-        for _, value in self.items():
-            value.apply(task, artifact)
-
-    def apply_deps(self, task, deps):
-        pass
-
-    def unapply(self, task, artifact):
-        for _, value in self.items():
-            value.unapply(task, artifact)
-
-    def unapply_deps(self, task, deps):
-        pass
-
-    def visit(self, task, artifact, visitor):
-        for _, value in self.items():
-            value.visit(task, artifact, visitor)
-
-
 class ArtifactAttributeSetRegistry(object):
-    providers = []
+    """
+    Registry for providers of artifact attribute sets.
+    """
+
+    providers = []  # List of objects that implement ArtifactAttributeSetProvider
 
     @staticmethod
     def create_all(artifact):
+        """ Create all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().create(artifact)
 
     @staticmethod
     def parse_all(artifact, content):
+        """ Parse all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().parse(artifact, content)
 
     @staticmethod
     def format_all(artifact, content):
+        """ Format all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().format(artifact, content)
 
     @staticmethod
     def apply_all(task, artifact):
+        """ Apply all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().apply(task, artifact)
 
-    @staticmethod
-    def apply_all_deps(task, deps):
-        for provider in ArtifactAttributeSetRegistry.providers:
-            provider().apply_deps(task, deps)
-
     @staticmethod
     def unapply_all(task, artifact):
+        """ Unapply all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().unapply(task, artifact)
 
-    @staticmethod
-    def unapply_all_deps(task, deps):
-        for provider in ArtifactAttributeSetRegistry.providers:
-            provider().unapply_deps(task, deps)
-
     @staticmethod
     def visit_all(task, artifact, visitor):
+        """ Visit all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().visit(task, artifact, visitor)
 
 
-def visit_artifact(task, artifact, visitor):
-    ArtifactAttributeSetRegistry.visit_all(task, artifact, visitor)
-
-
-class ArtifactAttributeSetProvider(object):
-    @staticmethod
-    def Register(cls):
-        ArtifactAttributeSetRegistry.providers.append(cls)
-
-    def create(self, artifact):
-        raise NotImplementedError()
-
-    def parse(self, artifact, content):
-        raise NotImplementedError()
-
-    def format(self, artifact, content):
-        raise NotImplementedError()
-
-    def apply(self, task, artifact):
-        pass
-
-    def apply_deps(self, task, deps):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
-
-    def unapply_deps(self, task, deps):
-        pass
+class ArtifactAttribute(object):
+    """
+    An artifact attribute.
 
-    def visit(self, task, artifact, visitor):
-        pass
+    An artifact attribute is a key-value pair that can be set and retrieved
+    from an artifact attribute set. Attributes are used to store metadata and other
+    information that is associated with an artifact. They communicate information
+    between tasks and store information that is used by tasks when they consume an artifact.
 
+    Artifact attributes can also perform actions when the artifact is consumed.
 
-class ArtifactAttribute(object):
+    """
     def __init__(self, name):
         self._name = name
 
     def get_name(self):
+        """ Get the name of the attribute. """
         return self._name
 
     def set_value(self, value, expand=True):
+        """
+        Set the value of the attribute.
+
+        Must be implemented by subclasses.
+
+        Args:
+            value: The value to set.
+            expand: If True, the value is macro expanded using the tools.expand() method.
+        """
         raise NotImplementedError()
 
     def get_value(self):
+        """
+        Get the value of the attribute.
+
+        Must be implemented by subclasses.
+        """
         raise NotImplementedError()
 
     def apply(self, task, artifact):
+        """
+        Perform an action when the artifact is being used.
+
+        Args:
+            task (Task): The task that is using the artifact.
+            artifact (Artifact): The artifact that is being used.
+
+        """
         pass
 
     def unapply(self, task, artifact):
+        """
+        Undo an action when the artifact is no longer being used.
+
+        Args:
+            task (Task): The task that is no longer using the artifact.
+            artifact (Artifact): The artifact that is no longer being used.
+        """
         pass
 
-    def __str__(self):
+    def __str__(self) -> str:
+        """
+        Get a string representation of the attribute.
+
+        Must be implemented by subclasses.
+        """
         raise NotImplementedError()
 
 
 class ArtifactStringAttribute(ArtifactAttribute):
+    """ An artifact attribute that stores a string value. """
+
     def __init__(self, artifact, name):
         self._artifact = artifact
         self._name = name
@@ -213,50 +158,55 @@ class ArtifactStringAttribute(ArtifactAttribute):
         return self._name
 
     def set_value(self, value, expand=True):
-        self._value = self._artifact.get_task().expand(str(value)) if expand else str(value)
+        self._value = self._artifact.tools.expand(str(value)) if expand else str(value)
 
     def get_value(self):
         return self._value
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
-
-    def __str__(self):
+    def __str__(self) -> str:
         return str(self._value)
 
 
 class ArtifactListAttribute(ArtifactAttribute):
+    """ An artifact attribute that stores a list of values. """
+
     def __init__(self, artifact, name):
         self._artifact = artifact
         self._name = name
         self._value = []
 
+    def __getitem__(self, key):
+        return self._value[key]
+
+    def __getslice__(self, i, j):
+        return self._value[i:j]
+
+    def __len__(self):
+        return len(self._value)
+
     def get_name(self):
         return self._name
 
     def set_value(self, value, expand=True):
-        if type(value) == str:
+        if type(value) is str:
             value = value.split(":")
-        raise_error_if(type(value) != list, "Illegal value assigned to artifact list attribute")
-        self._value = self._artifact.get_task().expand(value) if expand else value
+        raise_error_if(type(value) is not list, "Illegal value assigned to artifact list attribute")
+        self._value = self._artifact.tools.expand(value) if expand else value
 
     def get_value(self):
         return self._value
 
     def append(self, value):
-        if type(value) == list:
-            self._value.extend(self._artifact.get_task().expand(value))
+        if type(value) is list:
+            self._value.extend(self._artifact.tools.expand(value))
         else:
-            self._value.append(self._artifact.get_task().expand(value))
+            self._value.append(self._artifact.tools.expand(value))
 
     def extend(self, value):
         raise_error_if(
-            type(value) != list,
+            type(value) is not list,
             "Illegal type passed to {}.extend() - list expected".format(self._name))
-        self._value.extend(self._artifact.get_task().expand(value))
+        self._value.extend(self._artifact.tools.expand(value))
 
     def items(self):
         return list(self._value)
@@ -264,23 +214,16 @@ class ArtifactListAttribute(ArtifactAttribute):
     def count(self):
         return len(self.items())
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
+    def __str__(self) -> str:
+        return fs.pathsep.join(str(v) for v in self._value)
 
 
 class ArtifactFileAttribute(object):
+    """ An attribute that stores a list of source and destination path tuples for files collected into the artifact. """
+
     def __init__(self):
         self._files = []
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
-
     def append(self, src, dst):
         self._files.append((fs.as_posix(src), fs.as_posix(dst)))
 
@@ -291,8 +234,145 @@ class ArtifactFileAttribute(object):
         return self._files
 
 
+class ArtifactAttributeSet(object):
+    """
+    A set of artifact attributes.
+
+    An attribute set is a collection of attributes. Each attribute is
+    accessed using the attribute name as an attribute of the set. For
+    example, to access an attribute named 'version' in an attribute set
+    named 'strings', you would write:
+
+    .. code-block:: python
+
+        artifact.strings.version = "1.0"
+
+    """
+
+    def __init__(self):
+        super(ArtifactAttributeSet, self).__setattr__("_attributes", {})
+
+    def _get_attributes(self):
+        return self._attributes
+
+    def __getattr__(self, name) -> ArtifactAttribute:
+        """
+        Get or create an attribute by name.
+
+        Args:
+            name (str): The name of the attribute.
+
+        Returns:
+            An attribute object.
+        """
+        attributes = self._get_attributes()
+        if name not in attributes:
+            attributes[name] = self.create(name)
+        return attributes[name]
+
+    def __setattr__(self, name, value):
+        """
+        Set an attribute by name.
+
+        Args:
+            name (str): The name of the attribute.
+            value: The value to set.
+        """
+        attributes = self._get_attributes()
+        if name not in attributes:
+            attributes[name] = self.create(name)
+        attributes[name].set_value(value)
+        return attributes[name]
+
+    def __dict__(self):
+        """ Get a dictionary representation of the attribute set. """
+        return {key: str(value) for key, value in self.items()}
+
+    def __iter__(self):
+        """ Iterate over the attribute set. """
+        return iter(self.items())
+
+    def get(self, name, default=None):
+        """ Get an attribute by name.
+
+        Args:
+            name (str): The name of the attribute.
+
+        Returns:
+            The attribute object, or None if it does not exist.
+        """
+        attributes = self._get_attributes()
+        return attributes.get(name, default)
+
+    def items(self):
+        """ Get a list of tuples containing the attribute name and value. """
+        return self._get_attributes().items()
+
+    def apply(self, task, artifact):
+        """ Perform attribute actions when the artifact is being used. """
+        for _, value in self.items():
+            value.apply(task, artifact)
+
+    def unapply(self, task, artifact):
+        """ Undo attribute actions when the artifact is no longer being used. """
+        for _, value in self.items():
+            value.unapply(task, artifact)
+
+    def visit(self, task, artifact, visitor):
+        """ Visit all attributes in the set. """
+        for _, value in self.items():
+            value.visit(task, artifact, visitor)
+
+
+class ArtifactAttributeSetProvider(object):
+    """ Base class for artifact attribute set providers.
+
+    An artifact attribute set provider is a factory for creating and managing
+    attribute sets in an artifact.
+    """
+
+    @staticmethod
+    def Register(cls):
+        """ Decorator for registering a provider class. """
+        ArtifactAttributeSetRegistry.providers.append(cls)
+
+    def create(self, artifact):
+        """ Create an attribute set for an artifact. """
+        raise NotImplementedError()
+
+    def parse(self, artifact, content):
+        """
+        Parse an attribute set from a dictionary.
+
+        The dictionary is loaded from a JSON file embedded in the artifact.
+        """
+        raise NotImplementedError()
+
+    def format(self, artifact, content):
+        """
+        Format an attribute set to a dictionary.
+
+        The dictionary is saved to a JSON file embedded in the artifact.
+        """
+        raise NotImplementedError()
+
+    def apply(self, task, artifact):
+        """ Perform actions when the artifact is being used. """
+        pass
+
+    def unapply(self, task, artifact):
+        """ Undo actions when the artifact is no longer being used. """
+        pass
+
+    def visit(self, task, artifact, visitor):
+        """ Visit all attributes in the set. """
+        pass
+
+
 @ArtifactAttributeSetProvider.Register
 class ArtifactFileAttributeProvider(ArtifactAttributeSetProvider):
+    """ Provider for the artifact 'files' attribute set. """
+
     def create(self, artifact):
         setattr(artifact, "files", ArtifactFileAttribute())
 
@@ -306,22 +386,19 @@ class ArtifactFileAttributeProvider(ArtifactAttributeSetProvider):
     def format(self, artifact, content):
         content["files"] = [{"src": src, "dst": dst} for src, dst in artifact.files.items()]
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
 
-    def visit(self, task, artifact, visitor):
-        pass
+def visit_artifact(task, artifact, visitor):
+    ArtifactAttributeSetRegistry.visit_all(task, artifact, visitor)
 
 
 def json_serializer(obj):
+    """ JSON serializer for datetime objects. """
     if isinstance(obj, datetime):
         return dict(type="datetime", value=obj.strftime("%Y-%m-%d %H:%M:%S.%f"))
 
 
 def json_deserializer(dct):
+    """ JSON deserializer for datetime objects. """
     if dct.get("type") == "datetime":
         return datetime.strptime(dct["value"], "%Y-%m-%d %H:%M:%S.%f")
     return dct
@@ -453,26 +530,40 @@ class Artifact(object):
                 artifact.strings.version = "1.2"
     """
 
-    def __init__(self, cache, node, tools=None):
+    def __init__(self, cache, node, name=None, identity=None, tools=None, session=False):
         self._cache = cache
+        if identity:
+            self._identity = identity
+        else:
+            self._identity = node.identity if not session else node.instance
+        if name:
+            self._identity = name + "@" + self._identity
+        self._main = name == "main"
+        self._name = name or "main"
+        self._full_name = f"{self._name}@{node.short_qualified_name}" if node else self._name
+        self._log_name = f"{self._full_name} {node.identity[:8]}" if node else self._full_name
         self._node = node
+        self._session = session
+        self._task = node.task if node else None
         self._tools = tools or self._node.tools
-        self._path = cache._fs_get_artifact_path(node.identity, node.canonical_name)
-        self._temp = cache._fs_get_artifact_tmppath(node.identity, node.canonical_name)
-        self._archive = cache._fs_get_artifact_archivepath(node.identity, node.canonical_name)
-        self._lock_path = cache._fs_get_artifact_lockpath(node.identity)
-        self._unpacked = False
-        self._uploadable = True
-        self._created = datetime.now()
-        self._modified = datetime.now()
-        self._expires = node.task.expires
-        self._size = 0
-        self._influence = None
+        self._path = cache._fs_get_artifact_path(self._identity, node.canonical_name if node else name)
+        self._temp = cache._fs_get_artifact_tmppath(self._identity, node.canonical_name if node else name)
+        self._archive = cache._fs_get_artifact_archivepath(self._identity, node.canonical_name if node else name)
+        self._lock_path = cache._fs_get_artifact_lockpath(self._identity)
         ArtifactAttributeSetRegistry.create_all(self)
-        self._valid = False
-        self._temporary = False
-        self._read_manifest()
-        self._temporary = not self._valid
+        self.reload()
+
+    def _info(self, fmt, *args, **kwargs):
+        log.info(fmt + f" ({self._log_name})", *args, **kwargs)
+
+    def _debug(self, fmt, *args, **kwargs):
+        log.debug(fmt + f" ({self._log_name})", *args, **kwargs)
+
+    def _warning(self, fmt, *args, **kwargs):
+        log.warning(fmt + f" ({self._log_name})", *args, **kwargs)
+
+    def _error(self, fmt, *args, **kwargs):
+        log.error(fmt + f" ({self._log_name})", *args, **kwargs)
 
     def __enter__(self):
         return self
@@ -483,19 +574,23 @@ class Artifact(object):
     def __getattr__(self, name):
         raise_task_error(self._node, "Attempt to access invalid artifact attribute '{0}'", name)
 
-    def _write_manifest(self):
+    def _write_manifest(self, temporary=False):
         content = {}
-        content["task"] = self._node.name
         content["size"] = self._get_size()
         content["unpacked"] = self._unpacked
         content["uploadable"] = self._uploadable
-        content["identity"] = self._node.identity
-        content["requires"] = self._node.task.requires
-        content["parameters"] = self._node.task._get_parameters()
+        if self._node:
+            content["task"] = self._node.name
+            content["identity"] = self._node.identity
+            content["requires"] = self._node.task.requires
+            content["parameters"] = self._node.task._get_parameters()
+
         if self._influence is not None:
             content["influence"] = self._influence
-        else:
+        elif self._node:
             content["influence"] = influence.HashInfluenceRegistry.get().get_strings(self._node.task)
+        else:
+            content["influence"] = []
         content["created"] = self._created
         content["modified"] = datetime.now()
         content["expires"] = self._expires.value
@@ -504,13 +599,19 @@ class Artifact(object):
 
         ArtifactAttributeSetRegistry.format_all(self, content)
 
-        manifest = fs.path.join(self.path, ".manifest.json")
+        if temporary:
+            manifest = fs.path.join(self.temporary_path, ".manifest.json")
+        else:
+            manifest = fs.path.join(self.final_path, ".manifest.json")
         with open(manifest, "wb") as f:
             f.write(json.dumps(content, indent=2, default=json_serializer).encode())
 
-    def _read_manifest(self):
+    def _read_manifest(self, temporary=False):
         try:
-            manifest_path = fs.path.join(self.path, ".manifest.json")
+            if temporary:
+                manifest_path = fs.path.join(self.temporary_path, ".manifest.json")
+            else:
+                manifest_path = fs.path.join(self.final_path, ".manifest.json")
             with open(manifest_path) as manifest_file:
                 content = json.load(manifest_file, object_hook=json_deserializer)
             self._valid = True
@@ -568,9 +669,44 @@ class Artifact(object):
     def unapply(self):
         pass
 
+    def is_main(self):
+        return self._main
+
+    def is_session(self):
+        return self._session
+
     def is_valid(self):
         return self._valid
 
+    def reload(self):
+        self._unpacked = False
+        self._uploadable = True
+        self._created = datetime.now()
+        self._modified = datetime.now()
+        self._expires = self._task.expires if not self._session else expires.Immediately()
+        self._size = 0
+        self._influence = None
+        self._valid = False
+        self._temporary = False
+        self._read_manifest()
+        self._temporary = not self._valid
+
+    def reset(self):
+        self._unpacked = False
+        self._uploadable = True
+        self._created = datetime.now()
+        self._modified = datetime.now()
+        self._expires = self._task.expires if not self._session else expires.Immediately()
+        self._size = 0
+        self._influence = None
+        self._valid = False
+        self._temporary = True
+
+    @property
+    def name(self):
+        """ str: The name of the artifact. Default: 'main'. """
+        return self._name
+
     @property
     def path(self):
         """ str: The current location of the artifact in the local cache. """
@@ -625,7 +761,7 @@ class Artifact(object):
         raise_task_error_if(
             not self.is_temporary(),
             self._node,
-            "Can't collect files into an already published task artifact")
+            "Can't collect files into an already published task artifact ({})", self._log_name)
 
         files = self.tools.expand_path(files)
         files = self.tools.glob(files)
@@ -638,7 +774,7 @@ class Artifact(object):
             self.files.append(self.tools.expand_relpath(src), dest)
             self.tools.copy(src, fs.path.join(self._temp, dest), symlinks=symlinks)
             log.verbose("Collected {0} -> {2}/{1}", src, dest, self._temp)
-            return
+            return [dest]
 
         # Expand directories into full file list if flatting a tree
         # Determine relative artifact destination paths
@@ -662,6 +798,8 @@ class Artifact(object):
                 self.tools.copy(srcpath, dstpath, symlinks=symlinks)
                 log.verbose("Collected {0} -> {1}", relsrcpath, reldstpath)
 
+        return reldestfiles
+
     def copy(self, files, dest, flatten=False, symlinks=False, cwd=None):
         """ Copy files from the artifact.
 
@@ -694,7 +832,7 @@ class Artifact(object):
         raise_task_error_if(
             self.is_temporary(),
             self._node,
-            "Can't copy files from an unpublished task artifact")
+            "Can't copy files from an unpublished task artifact ({})", self._log_name)
 
         files = fs.path.join(self._path, files)
         files = self.tools.expand_path(files)
@@ -751,23 +889,24 @@ class Artifact(object):
     def get_size(self):
         return self._size
 
-    def get_task(self):
-        return self._node.task
-
-    def get_name(self):
-        return self._node.qualified_name
-
     def get_cache(self):
         return self._cache
 
-    def get_identity(self):
-        return self._node.identity
+    def get_task(self):
+        return self._node.task
+
+    def get_node(self):
+        return self._node
 
-    def is_temporary(self):
+    def is_temporary(self) -> bool:
         return self._temporary
 
-    def is_unpackable(self):
-        return self._node.is_unpackable()
+    def is_unpackable(self) -> bool:
+        if not self._node:
+            return True
+        if self.name == "main":
+            return self._task.unpack.__func__ is not tasks.Task.unpack
+        return getattr(self._task, "unpack_" + self.name, tasks.Task.unpack) is not tasks.Task.unpack
 
     def is_unpacked(self):
         return self._unpacked
@@ -775,9 +914,55 @@ class Artifact(object):
     def is_uploadable(self):
         return self._uploadable
 
+    def is_cacheable(self):
+        if not self._node:
+            return True
+        if self.is_session():
+            return True
+        return self.task.is_cacheable()
+
     @property
     def identity(self):
-        return self._node.identity
+        return self._identity
+
+    @property
+    def task(self):
+        if not self._node:
+            Task = namedtuple('Point', ['name'])
+            return Task(name=self.name)
+        return self._node.task
+
+
+class ArtifactToolsProxy(object):
+    """
+    An artifact proxy that uses a specific tools object.
+
+    Used when artifacts are consumed by tasks. The proxy allows the
+    task to access the artifact's methods and attributes using the
+    task's own tools object. This is useful when the consumer task
+    wishes to copy files, read files, etc, using the current working
+    directory and environment of the task.
+    """
+
+    def __init__(self, artifact, tools):
+        self._artifact = artifact
+        self._tools = tools
+
+    def __getattr__(self, name):
+        if name == "tools":
+            return self._tools
+        if name == "_artifact":
+            return self._artifact
+        attr = getattr(self._artifact.__class__, name, None)
+        if attr is not None and callable(attr):
+            return attr.__get__(self, ArtifactToolsProxy)
+        return getattr(self._artifact, name)
+
+    def __setattr__(self, name, value):
+        if name == "_artifact" or name == "_tools":
+            super(ArtifactToolsProxy, self).__setattr__(name, value)
+        else:
+            setattr(self._artifact, name, value)
 
 
 class Context(object):
@@ -803,17 +988,29 @@ class Context(object):
     def __enter__(self):
         try:
             for dep in reversed(self._node.children):
-                self._cache.unpack(dep)
-                with self._cache.get_artifact(dep, self._node.tools) as artifact:
-                    self._artifacts[dep.qualified_name] = artifact
-                    self._artifacts_index[dep.qualified_name] = artifact
-                    self._artifacts_index[dep.short_qualified_name] = artifact
+                for artifact in dep.artifacts:
+                    # Create clone with tools from this task
+                    artifact = ArtifactToolsProxy(artifact, self._node.tools)
+
+                    # Don't include session artifacts that don't exist,
+                    # i.e. where no build has taken place due to presence
+                    # of the persistent artifacts.
+                    if not dep.is_resource():
+                        if artifact.is_session() and not self._cache.is_available_locally(artifact):
+                            continue
+
+                    self._cache.unpack(artifact)
+
+                    if artifact.name == "main":
+                        self._artifacts_index[dep.qualified_name] = artifact
+                        self._artifacts_index[dep.short_qualified_name] = artifact
+                    self._artifacts[artifact.name + "@" + dep.qualified_name] = artifact
+                    self._artifacts_index[artifact.name + "@" + dep.qualified_name] = artifact
+                    self._artifacts_index[artifact.name + "@" + dep.short_qualified_name] = artifact
                     artifact.apply()
                     ArtifactAttributeSetRegistry.apply_all(self._node.task, artifact)
-            ArtifactAttributeSetRegistry.apply_all_deps(self._node.task, self)
-        except Exception as e:
+        except (Exception, KeyboardInterrupt) as e:
             # Rollback all attributes/resources except the last failing one
-            ArtifactAttributeSetRegistry.unapply_all_deps(self._node.task, self)
             for name, artifact in reversed(list(self._artifacts.items())[:-1]):
                 with utils.ignore_exception():
                     ArtifactAttributeSetRegistry.unapply_all(self._node.task, artifact)
@@ -822,7 +1019,6 @@ class Context(object):
         return self
 
     def __exit__(self, type, value, tb):
-        ArtifactAttributeSetRegistry.unapply_all_deps(self._node.task, self)
         for name, artifact in reversed(self._artifacts.items()):
             ArtifactAttributeSetRegistry.unapply_all(self._node.task, artifact)
             artifact.unapply()
@@ -846,25 +1042,32 @@ class Context(object):
 
         """
 
-        key = self._node.task.expand(key)
+        key = self._node.tools.expand(key)
 
-        if key not in self._artifacts_index:
-            key = self._node.resolve_requirement_alias(key) or key
+        alias, artifact, task, params = utils.parse_aliased_task_name(key)
+        raise_task_error_if(alias, self._node, "Cannot define alias when indexing dependencies: {}", alias)
+        task_name = utils.format_task_name(task, params)
+        task_artifact_name = utils.format_task_name(task, params, artifact)
 
-        # Parameters may be unordered, sort them
-        key = utils.stable_task_name(key)
+        if task_name not in self._artifacts_index and \
+           task_artifact_name not in self._artifacts_index and not params:
+            key = self._node.resolve_requirement_alias(task_name)
+            if key:
+                _, _, task, params = utils.parse_aliased_task_name(key)
+                task_name = utils.format_task_name(task, params)
+                task_artifact_name = utils.format_task_name(task, params, artifact)
 
         # Parameters may be overspecified, resolve task
-        if key not in self._artifacts_index:
+        if task_artifact_name not in self._artifacts_index:
             from jolt.tasks import TaskRegistry
-            task = TaskRegistry.get().get_task(key)
-            key = task.short_qualified_name if task is not None else key
+            task_obj = TaskRegistry.get().get_task(task_name)
+            task_name = task_obj.short_qualified_name if task_obj is not None else task
+            task_artifact_name = task_name if not artifact else f"{artifact}@{task_name}"
 
-        raise_task_error_if(
-            key not in self._artifacts_index,
-            self._node,
-            "No such dependency '{0}'", key)
-        return self._artifacts_index[key]
+        if task_artifact_name not in self._artifacts_index:
+            raise KeyError("No such artifact dependency '{0}' ({1})".format(
+                task_artifact_name, self._node.short_qualified_name))
+        return self._artifacts_index[task_artifact_name]
 
     def items(self):
         """ List all requirements and their artifacts.
@@ -877,11 +1080,148 @@ class Context(object):
 
 
 class PidProvider(object):
-    def __init__(self):
-        self._uuid = uuid.uuid4()
-
     def __call__(self):
-        return str(self._uuid)
+        pid = str(uuid.uuid4())
+        log.debug("New cache lock file: {0}", pid)
+        return pid
+
+
+class StorageProvider(object):
+    """
+    Base class for remote artifact storage providers.
+
+    A storage provider is responsible for uploading and downloading
+    artifacts to and from a remote storage location. The storage
+    location can be a file system path, a cloud storage service, or
+    any other type of storage.
+
+    """
+
+    def download(self, artifact: Artifact, force: bool = False) -> bool:
+        """
+        Download an artifact from the storage location.
+
+        The should be downloaded to the path returned by the artifact's
+        :func:`~jolt.Artifact.get_archive_path` method. The downloaded artifact
+        must be in the format specified by DEFAULT_ARCHIVE_TYPE.
+
+        The download should be retried if it fails due to network issues.
+        The method may raise an exception on errors.
+
+        Args:
+            artifact (Artifact): The artifact to download.
+            force (bool, optional): If True, the download should be forced,
+                even if the artifact is already present locally, or if the
+                download is disabled. The default is False.
+
+        Returns:
+            bool: True if the download was successful, False otherwise.
+
+        """
+        return False
+
+    def download_enabled(self) -> bool:
+        """ Return True if downloading is enabled. Default is True. """
+        return True
+
+    def upload(self, artifact: Artifact, force: bool = False) -> bool:
+        """
+        Upload an artifact to the storage location.
+
+        The artifact to be uploaded is located at the path returned by
+        the artifact's :func:`~jolt.Artifact.get_archive_path` method. The
+        uploaded artifact is in the format specified by DEFAULT_ARCHIVE_TYPE.
+        The provider may choose to upload the artifact using a different
+        format, but it must be able to download the artifact in the
+        DEFAULT_ARCHIVE_TYPE format.
+
+        The upload should be retried if it fails due to network issues.
+        The method may raise an exception on errors.
+
+        Args:
+            artifact (Artifact): The artifact to upload.
+            force (bool, optional): If True, the upload should be forced,
+                even if the artifact is already present remotely, or if the
+                upload is disabled. The default is False.
+
+        Returns:
+            bool: True if the upload was successful, False otherwise.
+
+        """
+        return False
+
+    def upload_enabled(self) -> bool:
+        """ Return True if uploading is enabled. Default is True. """
+        return True
+
+    def location(self, artifact) -> str:
+        """
+        Return the URL of the artifact in the storage location.
+
+        This method is sometimes used to identify if an artifact is
+        present in the storage location. The URL should point to the
+        artifact if present, or an empty string if the artifact is
+        absent.
+
+        Args:
+            artifact (Artifact): The artifact to locate.
+        """
+        return ''  # URL
+
+    def availability(self, artifacts: list) -> tuple:
+        """
+        Check the availability of a list of artifacts.
+
+        This method is used to determine which artifacts are present in the
+        storage location. The method should return a tuple of two lists:
+        the first list contains the artifacts that are present, and the
+        second list contains the artifacts that are missing.
+
+        The default implementation of this method calls the :func:`~jolt.StorageProvider.location`
+        method for each artifact in the list. Subclasses may override this
+        method to provide a more efficient implementation.
+
+        Args:
+            artifacts (list): A list of artifacts to check.
+
+        Returns:
+            tuple: A tuple of two lists: the first list contains the artifacts
+                that are present, and the second list contains the artifacts
+                that are missing.
+
+        """
+        # Ensure artifacts is a list
+        artifacts = utils.as_list(artifacts)
+
+        present = set()
+        missing = set()
+
+        for artifact in artifacts:
+            if self.location(artifact):
+                present.add(artifact)
+            else:
+                missing.add(artifact)
+
+        return list(present), list(missing)
+
+
+class StorageProviderFactory(StorageProvider):
+    """ A factory for store providers. """
+
+    def create(self) -> StorageProvider:
+        """
+        Create a new storage provider.
+
+        This method should return a new instance of a storage provider,
+        which must be a subclass of :class:`~jolt.StorageProvider`.
+
+        """
+        pass
+
+
+def RegisterStorage(cls):
+    """ Decorator used to register a storage provider factory. """
+    ArtifactCache.storage_provider_factories.append(cls)
 
 
 @utils.Singleton
@@ -943,14 +1283,18 @@ class ArtifactCache(StorageProvider):
 
     def __init__(self, options=None, pidprovider=None):
         self._options = options or JoltOptions()
-        self._remote_identity_cache = set()
         self._storage_providers = [
             factory.create(self)
             for factory in ArtifactCache.storage_provider_factories]
 
+        # If no storage providers supports the availability method,
+        # we will not only use the local presence cache.
+        self._remote_presence_cache = set()
+        self._presence_cache_only = self.has_availability()
+
         # Read configuration
         self._max_size = config.getsize(
-            "jolt", "cachesize", os.environ.get("JOLT_CACHESIZE", 1 * 1024 ** 3))
+            "jolt", "cachesize", os.environ.get("JOLT_CACHE_SIZE", 1 * 1024 ** 3))
 
         # Create cache directory
         self._fs_create_cachedir()
@@ -959,10 +1303,12 @@ class ArtifactCache(StorageProvider):
         self._cache_locked = False
         self._lock_file = fasteners.InterProcessLock(self._fs_get_lock_file())
         self._thread_lock = RLock()
+        self._artifact_thread_lock = utils.IdLock()
 
         # Create process lock file
         with self._cache_lock():
-            self._pid = pidprovider() if pidprovider else PidProvider()()
+            self._pid_provider = pidprovider or PidProvider()
+            self._pid = self._pid_provider()
             self._pid_file = fasteners.InterProcessLock(self._fs_get_pid_file(self._pid))
             self._pid_file.acquire()
 
@@ -994,6 +1340,7 @@ class ArtifactCache(StorageProvider):
         db = sqlite3.connect(self._db_path, detect_types=sqlite3.PARSE_DECLTYPES)
         try:
             db.execute("PRAGMA journal_mode=OFF")
+            # db.set_trace_callback(log.warning)
             yield db
         finally:
             db.close()
@@ -1015,9 +1362,9 @@ class ArtifactCache(StorageProvider):
         cur.execute("CREATE TABLE IF NOT EXISTS artifact_lockrefs (identity text, pid text)")
         db.commit()
 
-    def _db_insert_artifact(self, db, identity, name, size):
+    def _db_insert_artifact(self, db, identity, task_name, size):
         cur = db.cursor()
-        cur.execute("INSERT INTO artifacts VALUES (?,?,?,?)", (identity, name, size, datetime.now()))
+        cur.execute("INSERT INTO artifacts VALUES (?,?,?,?)", (identity, task_name, size, datetime.now()))
         db.commit()
 
     def _db_update_artifact_size(self, db, identity, size):
@@ -1180,6 +1527,7 @@ class ArtifactCache(StorageProvider):
 
     def _fs_create_cachedir(self):
         self.root = config.get_cachedir()
+        log.verbose("Jolt cache path: {}", self.root)
         try:
             fs.makedirs(self.root)
         except KeyboardInterrupt as e:
@@ -1187,73 +1535,103 @@ class ArtifactCache(StorageProvider):
         except Exception:
             raise_error("Failed to create cache directory '{0}'", self.root)
 
-    def _fs_get_artifact(self, node, tools=None):
-        return Artifact(self, node, tools)
+    def _fs_get_artifact(self, node, name, tools=None, session=False):
+        return Artifact(self, node, name=name, tools=tools, session=session)
 
-    def _fs_commit_artifact(self, artifact, uploadable):
+    def _fs_commit_artifact(self, artifact: Artifact, uploadable: bool, temporary: bool):
         artifact._set_uploadable(uploadable)
         if not artifact.is_unpackable():
             artifact._set_unpacked()
-        artifact._write_manifest()
-        if artifact.is_temporary():
+        if temporary:
+            artifact._write_manifest(temporary=True)
             fs.rmtree(artifact.final_path, ignore_errors=True)
-            fs.rename(artifact.path, artifact.final_path)
+            fs.rename(artifact.temporary_path, artifact.final_path)
+        else:
+            artifact._write_manifest(temporary=False)
 
     @contextlib.contextmanager
     def _fs_compress_artifact(self, artifact):
-        task = artifact.get_task()
+        task = artifact.task
         archive = artifact.get_archive_path()
 
         raise_task_error_if(
             artifact.is_temporary(), task,
-            "Can't compress an unpublished task artifact")
+            "Can't compress an unpublished task artifact ({})", artifact._log_name)
 
         try:
-            task.tools.archive(artifact.path, archive)
+            artifact.tools.archive(artifact.path, archive)
         except KeyboardInterrupt as e:
             raise e
         except Exception:
-            raise_task_error(task, "Failed to compress task artifact")
+            raise_task_error(task, "Failed to compress task artifact ({})", artifact._log_name)
         try:
             yield
         finally:
             fs.unlink(archive, ignore_errors=True)
 
     def _fs_decompress_artifact(self, artifact):
-        task = artifact.get_task()
+        task = artifact.task
         archive = artifact.get_archive_path()
         try:
             task.tools.extract(archive, artifact.temporary_path, ignore_owner=True)
+            artifact._read_manifest(temporary=True)
         except KeyboardInterrupt as e:
+            fs.rmtree(artifact.temporary_path, ignore_errors=True)
             raise e
         except Exception:
-            raise_task_error(task, "Failed to extract task artifact archive")
+            fs.rmtree(artifact.temporary_path, ignore_errors=True)
+            raise_task_error(task, "Failed to extract task artifact archive ({})", artifact._log_name)
         finally:
             fs.unlink(archive, ignore_errors=True)
-        artifact._read_manifest()
-
-    def _fs_delete_artifact(self, identity, name, onerror=None):
-        fs.rmtree(self._fs_get_artifact_path(identity, name), ignore_errors=True, onerror=onerror)
-        fs.rmtree(self._fs_get_artifact_tmppath(identity, name), ignore_errors=True, onerror=onerror)
-        fs.unlink(fs.path.join(self.root, name), ignore_errors=True)
 
-    def _fs_get_artifact_archivepath(self, identity, name):
-        return fs.get_archive(fs.path.join(self.root, name, identity))
+    def _fs_delete_artifact(self, identity, task_name, onerror=None):
+        fs.rmtree(self._fs_get_artifact_path(identity, task_name), ignore_errors=True, onerror=onerror)
+        fs.rmtree(self._fs_get_artifact_tmppath(identity, task_name), ignore_errors=True, onerror=onerror)
+        fs.rmtree(self._fs_get_artifact_path_legacy(identity, task_name), ignore_errors=True, onerror=onerror)
+        fs.rmtree(self._fs_get_artifact_tmppath_legacy(identity, task_name), ignore_errors=True, onerror=onerror)
+        fs.unlink(fs.path.join(self.root, task_name), ignore_errors=True)
+
+    def _fs_identity(self, identity):
+        parts = identity.split("@", 1)
+        if len(parts) <= 1:
+            parts = ["main"] + parts
+        return parts[1] + "-" + utils.canonical(parts[0])
+
+    def _fs_identity_legacy(self, identity):
+        parts = identity.split("@", 1)
+        if len(parts) <= 1:
+            parts = ["main"] + parts
+        return parts[0] + "@" + utils.canonical(parts[1])
+
+    def _fs_get_artifact_archivepath(self, identity, task_name):
+        identity = self._fs_identity(identity)
+        return fs.path.join(self.root, task_name, identity) + DEFAULT_ARCHIVE_TYPE
 
     def _fs_get_artifact_lockpath(self, identity):
+        identity = self._fs_identity(identity)
         return fs.path.join(self.root, "locks", identity + ".lock")
 
-    def _fs_get_artifact_tmppath(self, identity, name):
-        return fs.path.join(self.root, name, "." + identity)
+    def _fs_get_artifact_tmppath(self, identity, task_name):
+        identity = self._fs_identity(identity)
+        return fs.path.join(self.root, task_name, "." + identity)
 
-    def _fs_get_artifact_path(self, identity, name):
-        return fs.path.join(self.root, name, identity)
+    def _fs_get_artifact_path(self, identity, task_name):
+        identity = self._fs_identity(identity)
+        return fs.path.join(self.root, task_name, identity)
 
-    def _fs_get_artifact_manifest_path(self, identity, name):
-        return fs.path.join(self._fs_get_artifact_path(identity, name), ".manifest.json")
+    def _fs_get_artifact_tmppath_legacy(self, identity, task_name):
+        identity = self._fs_identity_legacy(identity)
+        return fs.path.join(self.root, task_name, "." + identity)
 
-    def _fs_get_artifact_manifest(self, identity, name):
-        path = self._fs_get_artifact_manifest_path(identity, name)
+    def _fs_get_artifact_path_legacy(self, identity, task_name):
+        identity = self._fs_identity_legacy(identity)
+        return fs.path.join(self.root, task_name, identity)
+
+    def _fs_get_artifact_manifest_path(self, identity, task_name):
+        return fs.path.join(self._fs_get_artifact_path(identity, task_name), ".manifest.json")
+
+    def _fs_get_artifact_manifest(self, identity, task_name):
+        path = self._fs_get_artifact_manifest_path(identity, task_name)
         with open(path) as manifest_file:
             return json.load(manifest_file, object_hook=json_deserializer)
 
@@ -1266,9 +1644,9 @@ class ArtifactCache(StorageProvider):
     def _fs_get_pid_file(self, pid):
         return fs.path.join(self.root, "pids", pid)
 
-    def _fs_is_artifact_expired(self, identity, name, last_used):
+    def _fs_is_artifact_expired(self, identity, task_name, last_used):
         try:
-            manifest = self._fs_get_artifact_manifest(identity, name)
+            manifest = self._fs_get_artifact_manifest(identity, task_name)
             manifest["used"] = last_used
             strategy = ArtifactEvictionStrategyRegister.get().find(
                 manifest.get("expires", "immediately"))
@@ -1318,19 +1696,39 @@ class ArtifactCache(StorageProvider):
         """ Discard list of artifacts. Cache lock must be held. """
         self._assert_cache_locked()
         evicted = 0
-        for identity, name, _, used in artifacts:
-            if not if_expired or self._fs_is_artifact_expired(identity, name, used):
-                self._db_delete_artifact(db, identity)
-                self._fs_delete_artifact(identity, name, onerror=onerror)
-                evicted += 1
-                log.debug("Evicted {}: {}", identity, name)
+        for identity, task_name, _, used in artifacts:
+            if not if_expired or self._fs_is_artifact_expired(identity, task_name, used):
+                with utils.delayed_interrupt():
+                    self._db_delete_artifact(db, identity)
+                    self._fs_delete_artifact(identity, task_name, onerror=onerror)
+                    evicted += 1
+                    log.debug("Evicted {}: {}", identity, task_name)
         return evicted == len(artifacts)
 
     ############################################################################
     # Public API
     ############################################################################
 
-    def is_available_locally(self, node):
+    def release(self):
+        """
+        Release references to artifacts held by the current process.
+
+        Effectively, a new pid lock file is created and the old one is deleted. This
+        allows other processes to detect termination of the current process and
+        garbage collect any references owned by the process.
+        """
+        with self._cache_lock(), self._db() as db:
+            self._db_invalidate_locks(db, try_all=True)
+            self._db_invalidate_references(db, try_all=True)
+            self._fs_invalidate_pids(db, try_all=True)
+            self._pid_file.release()
+
+            self._pid = self._pid_provider()
+            self._pid_file = fasteners.InterProcessLock(self._fs_get_pid_file(self._pid))
+            self._pid_file.acquire()
+
+    @utils.delay_interrupt
+    def is_available_locally(self, artifact):
         """
         Check presence of task artifact in cache.
 
@@ -1338,84 +1736,115 @@ class ArtifactCache(StorageProvider):
         recorded for the running process to prevent eviction by other
         processes.
         """
-        if not node.task.is_cacheable():
+        if not artifact.is_cacheable():
             return False
 
-        # Cache availability in node
-        try:
-            return node.__available
-        except AttributeError:
-            pass
-
         with self._cache_lock(), self._db() as db:
-            if self._db_select_artifact(db, node.identity) or self._db_select_reference(db, node.identity):
-                with self._fs_get_artifact(node) as a:
-                    if a.is_temporary():
-                        self._db_delete_artifact(db, node.identity)
-                        return False
-                    self._db_insert_reference(db, node.identity)
-                node.__available = True
+            if self._db_select_artifact(db, artifact.identity) or self._db_select_reference(db, artifact.identity):
+                artifact.reload()
+                if artifact.is_temporary():
+                    self._db_delete_artifact(db, artifact.identity, and_refs=False)
+                    return False
+                self._db_insert_reference(db, artifact.identity)
                 return True
         return False
 
-    def is_available_remotely(self, node):
+    def is_available_remotely(self, artifact, cache=True):
         """
         Check presence of task artifact in external remote caches.
         """
-        if not node.task.is_cacheable():
-            return False
-        if node.identity in self._remote_identity_cache:
-            return True
+        if cache:
+            if artifact.identity in self._remote_presence_cache:
+                return True
+            if self._presence_cache_only:
+                return False
         for provider in self._storage_providers:
-            if provider.location(node):
-                self._remote_identity_cache.add(node.identity)
+            present, _ = provider.availability([artifact])
+            if present:
+                self._remote_presence_cache.add(artifact.identity)
                 return True
         return False
 
-    def is_available(self, node):
+    def is_available(self, artifact):
         """ Check presence of task artifact in any cache, local or remote """
-        return self.is_available_locally(node) or self.is_available_remotely(node)
+        return self.is_available_locally(artifact) or self.is_available_remotely(artifact)
 
-    def is_unpacked(self, node):
-        with self.get_artifact(node) as artifact:
-            return artifact.is_unpacked()
+    def has_availability(self):
+        # Returns true if all storage providers implement the availability method
+        return all([provider.availability.__func__ != StorageProvider.availability for provider in self._storage_providers])
+
+    def availability(self, artifacts, remote=True):
+        """ Check presence of task artifacts in any cache, local or remote """
+        present = set()
+        missing = set()
+
+        # Make sure artifacts is a list
+        artifacts = utils.as_list(artifacts)
+
+        # Check presence of all artifacts in the local cache
+        for artifact in artifacts:
+            if self.is_available_locally(artifact):
+                present.add(artifact)
+            else:
+                missing.add(artifact)
+
+        if not remote:
+            return list(present), list(missing)
+
+        # Check presence of all artifacts in the remote caches
+        missing_remotely = artifacts
+
+        for provider in self._storage_providers:
+            present_in_provider, missing_in_provider = provider.availability(missing_remotely)
+            for artifact in present_in_provider:
+                self._remote_presence_cache.add(artifact.identity)
+            present.update(present_in_provider)
+            missing_remotely = missing_in_provider
+            if not missing_in_provider:
+                break
 
-    def is_uploadable(self, node):
-        with self.get_artifact(node) as artifact:
-            return artifact.is_uploadable()
+        missing.update(missing_remotely)
+        missing = missing - present
+
+        return list(present), list(missing)
 
     def download_enabled(self):
         return self._options.download and \
             any([provider.download_enabled() for provider in self._storage_providers])
 
+    def download_session_enabled(self):
+        return self._options.download_session and \
+            any([provider.download_enabled() for provider in self._storage_providers])
+
     def upload_enabled(self):
         return self._options.upload and \
             any([provider.upload_enabled() for provider in self._storage_providers])
 
-    def download(self, node, force=False):
+    def download(self, artifact, force=False):
         """
         Downloads an artifact from a remote cache to the local cache.
 
         The artifact is interprocess locked during the operation.
         """
-        if not force and not self.download_enabled():
-            return False
-        if not node.task.is_cacheable():
+        if not force:
+            if not artifact.is_session() and not self.download_enabled():
+                return False
+            if artifact.is_session() and not self.download_session_enabled():
+                return False
+        if not artifact.is_cacheable():
             return False
-        if not node.is_downloadable():
-            return True
-        with self.get_locked_artifact(node) as artifact:
-            if self.is_available_locally(node):
-                node.info("Download skipped, already in local cache")
+        with self.lock_artifact(artifact, why="download") as artifact:
+            if self.is_available_locally(artifact):
+                artifact._info("Download skipped, already in local cache")
                 return True
             for provider in self._storage_providers:
-                if provider.download(node, force):
+                if provider.download(artifact, force):
                     self._fs_decompress_artifact(artifact)
-                    self.commit(artifact)
+                    self.commit(artifact, temporary=True)
                     return True
         return len(self._storage_providers) == 0
 
-    def upload(self, node, force=False, locked=True):
+    def upload(self, artifact, force=False, locked=True):
         """
         Uploads an artifact from the local cache to all configured remote caches.
 
@@ -1423,30 +1852,28 @@ class ArtifactCache(StorageProvider):
         """
         if not force and not self.upload_enabled():
             return False
-        if not node.task.is_cacheable():
+        if not artifact.is_cacheable():
             return True
         raise_task_error_if(
-            not self.is_available_locally(node), node,
-            "Can't upload task artifact, no artifact present in the local cache")
-        with self.get_locked_artifact(node) if locked else self.get_artifact(node) as artifact:
+            not self.is_available_locally(artifact), artifact.task,
+            "Can't upload task artifact, no artifact present in the local cache ({})", artifact._log_name)
+        with self.lock_artifact(artifact, why="upload") if locked else artifact as artifact:
             raise_task_error_if(
-                not artifact.is_uploadable(), node,
-                "Artifact was modified locally by another process and can no longer be uploaded, try again")
+                not artifact.is_uploadable(), artifact.task,
+                "Artifact was modified locally by another process and can no longer be uploaded, try again ({})", artifact._log_name)
             if self._storage_providers:
                 with self._fs_compress_artifact(artifact):
-                    return all([provider.upload(node, force) for provider in self._storage_providers])
+                    return all([provider.upload(artifact, force) for provider in self._storage_providers])
         return len(self._storage_providers) == 0
 
-    def location(self, node):
-        if not node.task.is_cacheable():
-            return ''
+    def location(self, artifact):
         for provider in self._storage_providers:
-            url = provider.location(node)
+            url = provider.location(artifact)
             if url:
                 return url
         return ''
 
-    def unpack(self, node):
+    def unpack(self, artifact):
         """
         Unpacks/relocates the task artifact to the local cache.
 
@@ -1458,40 +1885,59 @@ class ArtifactCache(StorageProvider):
 
         The artifact is interprocess locked during the operation.
         """
-        if not node.task.is_cacheable():
-            return False
-        if not node.is_unpackable():
+        if not artifact.is_unpackable():
             return True
-        with self._thread_lock, self.get_locked_artifact(node) as artifact:
-            if not self.is_available_locally(node):
-                raise_task_error(node, "Locked artifact is missing in cache (forcibly removed?)")
+        with self._thread_lock, self.lock_artifact(artifact, why="unpack") as artifact:
+            raise_task_error_if(
+                not self.is_available_locally(artifact),
+                artifact.task,
+                "Locked artifact is missing in cache (forcibly removed?) ({})", artifact._log_name)
+
+            raise_task_error_if(
+                artifact.is_temporary(),
+                artifact.task,
+                "Can't unpack an unpublished task artifact ({})", artifact._log_name)
+
             if artifact.is_unpacked():
                 return True
 
             # Keep a temporary copy of the artifact if the task
             # unpack() method fails. The copy is removed in
             # get_locked_artifact() if left unused.
-            fs.copy(artifact.path, artifact.temporary_path, symlinks=True)
+            fs.copy(artifact.final_path, artifact.temporary_path, symlinks=True)
 
-            task = artifact.get_task()
+            task = artifact.task
             with tools.Tools(task) as t:
                 try:
                     # Note: unpack() will run on the original
                     # artifact, not in the temporary copy.
-                    node.verbose("Unpacking")
+                    if task.unpack.__func__ is not tasks.Task.unpack:
+                        artifact._info("Unpack started")
                     artifact._set_unpacked()
-                    task.unpack(artifact, t)
-                    self.commit(artifact, uploadable=False)
+                    if artifact.name == "main":
+                        task.unpack(artifact, t)
+                    else:
+                        unpack = getattr(task, "unpack_" + artifact.name, None)
+                        raise_task_error_if(
+                            unpack is None, task,
+                            "Artifact unpack method not found: unpack_{}", artifact.name)
+                        unpack(artifact, t)
+
+                    self.commit(artifact, uploadable=False, temporary=False)
+
                 except NotImplementedError:
-                    self.commit(artifact)
-                except Exception as e:
+                    self.commit(artifact, temporary=False)
+
+                except (Exception, KeyboardInterrupt) as e:
                     # Restore the temporary copy
-                    fs.rmtree(artifact.path, ignore_errors=True)
-                    fs.rename(artifact.temporary_path, artifact.path)
+                    fs.rmtree(artifact.final_path, ignore_errors=True)
+                    fs.rename(artifact.temporary_path, artifact.final_path)
+                    artifact._error("Unpack failed")
                     raise e
         return True
 
-    def commit(self, artifact, uploadable=True):
+    @utils.delay_interrupt
+    def commit(self, artifact, uploadable=True, temporary=True):
         """
         Commits a task artifact to the cache.
 
@@ -1503,14 +1949,16 @@ class ArtifactCache(StorageProvider):
         take place if the resulting cache size exceeds the configured
         limit.
         """
-        if not artifact.get_task().is_cacheable():
+        if not artifact.is_cacheable():
             return
+
         with self._cache_lock(), self._db() as db:
-            self._fs_commit_artifact(artifact, uploadable)
+            self._fs_commit_artifact(artifact, uploadable, temporary)
             with utils.ignore_exception():  # Possibly already exists in DB, e.g. unpacked
-                self._db_insert_artifact(db, artifact.get_task().identity, artifact.get_task().canonical_name, artifact.get_size())
-            self._db_update_artifact_size(db, artifact.get_task().identity, artifact.get_size())
-            self._db_insert_reference(db, artifact.get_task().identity)
+                self._db_insert_artifact(db, artifact.identity, artifact.task.canonical_name, artifact.get_size())
+            self._db_update_artifact_size(db, artifact.identity, artifact.get_size())
+            self._db_insert_reference(db, artifact.identity)
+            artifact.reload()
 
             evict_size = self._db_select_sum_artifact_size(db) - self._max_size
             if evict_size < 0:
@@ -1522,21 +1970,19 @@ class ArtifactCache(StorageProvider):
                 if self._discard(db, [candidate], True):
                     evict_size -= candidate[2]
 
-    def discard(self, node, if_expired=False, onerror=None):
+    @utils.delay_interrupt
+    def discard(self, artifact, if_expired=False, onerror=None):
         with self._cache_lock(), self._db() as db:
             self._db_invalidate_locks(db)
             self._db_invalidate_references(db)
             self._fs_invalidate_pids(db)
-            discarded = self._discard(
+            return self._discard(
                 db,
-                self._db_select_artifact_not_in_use(db, node.identity),
+                self._db_select_artifact_not_in_use(db, artifact.identity),
                 if_expired,
                 onerror=onerror)
-            if discarded and hasattr(node, "_ArtifactCache__available"):
-                del node.__available
-            return discarded
 
-    def _discard_wait(self, node):
+    def _discard_wait(self, artifact):
         """
         Discards an artifact without expiration consideration.
 
@@ -1550,13 +1996,14 @@ class ArtifactCache(StorageProvider):
             self._db_invalidate_locks(db)
             self._db_invalidate_references(db)
             self._fs_invalidate_pids(db)
-            artifacts = self._db_select_artifact(db, node.identity)
-            self._db_delete_artifact(db, node.identity, and_refs=False)
-            refpids = self._db_select_artifact_reference_pids(db, node.identity)
-            lockpids = self._db_select_artifact_lock_pids(db, node.identity)
-
-        if len(refpids) > 1:
-            node.info("Artifact is temporarily in use, forced discard on hold")
+            artifacts = self._db_select_artifact(db, artifact.identity)
+            self._db_delete_artifact(db, artifact.identity, and_refs=False)
+            refpids = self._db_select_artifact_reference_pids(db, artifact.identity)
+            refpids = list(filter(lambda pid: pid != self._pid, refpids))
+            lockpids = self._db_select_artifact_lock_pids(db, artifact.identity)
+
+        if len(refpids) > 0:
+            artifact._info("Artifact is temporarily in use, forced discard on hold")
             for pid in refpids:
                 # Loop waiting for other processes to surrender the artifact
                 while True:
@@ -1572,13 +2019,12 @@ class ArtifactCache(StorageProvider):
                             break
                     except RuntimeError:
                         with self._cache_lock(), self._db() as db:
-                            lockpids = self._db_select_artifact_lock_pids(db, node.identity)
+                            lockpids = self._db_select_artifact_lock_pids(db, artifact.identity)
 
         with self._cache_lock(), self._db() as db:
             assert self._discard(db, artifacts, False), "Failed to discard artifact"
-            if hasattr(node, "_ArtifactCache__available"):
-                del node.__available
-        return self._fs_get_artifact(node)
+            artifact.reset()
+        return artifact
 
     def discard_all(self, if_expired=False, onerror=None):
         with self._cache_lock(), self._db() as db:
@@ -1594,11 +2040,30 @@ class ArtifactCache(StorageProvider):
     def get_context(self, node):
         return Context(self, node)
 
-    def get_artifact(self, node, tools=None):
-        return self._fs_get_artifact(node, tools)
+    def get_artifact(self, node, name, tools=None, session=False):
+        artifact = self._fs_get_artifact(node, name=name, tools=tools, session=session)
+        if not artifact.is_temporary():
+            with self._cache_lock(), self._db() as db:
+                if not self._db_select_artifact(db, artifact.identity) and not self._db_select_reference(db, artifact.identity):
+                    log.verbose("Artifact not present in db, discarding archive ({} )", artifact.task.short_qualified_name, artifact.identity)
+                    fs.rmtree(artifact.final_path, ignore_errors=True)
+                    artifact.reload()
+        return artifact
+
+    @contextlib.contextmanager
+    def lock_artifact(self, artifact: Artifact, discard: bool = False, why: str = "publish"):
+        """
+        Locks the task artifact, both with process thread locks and interprocess file locks.
+        """
+        try:
+            self._artifact_thread_lock.acquire(artifact.identity)
+            with self._lock_artifact_interprocess(artifact, discard=discard, why=why) as artifact:
+                yield artifact
+        finally:
+            self._artifact_thread_lock.release(artifact.identity)
 
     @contextlib.contextmanager
-    def get_locked_artifact(self, node, discard=False):
+    def _lock_artifact_interprocess(self, artifact: Artifact, discard: bool = False, why: str = "publish"):
         """
         Locks the task artifact.
 
@@ -1608,35 +2073,42 @@ class ArtifactCache(StorageProvider):
         """
         with self._cache_lock():
             with self._db() as db:
-                self._db_insert_lock(db, node.identity)
-                self._db_insert_reference(db, node.identity)
-            lock_path = self._fs_get_artifact_lockpath(node.identity)
+                self._db_insert_lock(db, artifact.identity)
+                self._db_insert_reference(db, artifact.identity)
+            lock_path = self._fs_get_artifact_lockpath(artifact.identity)
             lock = fasteners.InterProcessLock(lock_path)
             is_locked = lock.acquire(blocking=False)
         if not is_locked:
-            node.info("Artifact is temporarily locked by another process")
+            artifact._info("Artifact is temporarily locked by another process")
             lock.acquire()
 
+        artifact._debug("Artifact locked for {}", why)
+
         try:
-            artifact = self.get_artifact(node)
             if discard:
-                artifact = self._discard_wait(node)
+                artifact = self._discard_wait(artifact)
+            else:
+                artifact.reload()
+
             if artifact.is_temporary():
                 fs.rmtree(artifact.temporary_path, ignore_errors=True)
                 fs.makedirs(artifact.temporary_path)
 
-            with contextlib.ExitStack() as stack:
-                stack.enter_context(artifact)
-                yield artifact
+            yield artifact
         finally:
+            artifact._debug("Artifact unlocked for {}", why)
             fs.rmtree(artifact.temporary_path, ignore_errors=True)
             with self._cache_lock():
                 with self._db() as db:
-                    self._db_delete_lock(db, node.identity)
+                    self._db_delete_lock(db, artifact.identity)
                 lock.release()
                 with self._db() as db:
-                    if self._db_select_lock_count(db, node.identity) == 0:
+                    if self._db_select_lock_count(db, artifact.identity) == 0:
                         fs.unlink(lock_path, ignore_errors=True)
 
-    def get_path(self, node):
-        return self._fs_get_artifact_path(node.identity, node.canonical_name)
+    def precheck(self, artifacts, remote=True):
+        """ Precheck artifacts for availability and cache status. """
+        if not self.has_availability():
+            return
+        present, missing = self.availability(artifacts, remote=remote)
+        log.verbose("Cache: {}/{} artifacts present", len(present), len(artifacts))