jolt 0.9.172__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/cache.py

@@ -23,7 +23,7 @@ from jolt.error import raise_task_error, raise_task_error_if
 from jolt.expires import ArtifactEvictionStrategyRegister
 
 
-DEFAULT_ARCHIVE_TYPE = ".tar.gz"
+DEFAULT_ARCHIVE_TYPE = ".tar.zst"
 
 
 def locked(func):
@@ -33,194 +33,122 @@ def locked(func):
     return _f
 
 
-class StorageProvider(object):
-    def download(self, artifact, force=False):
-        return False
-
-    def download_enabled(self):
-        return True
-
-    def upload(self, artifact, force=False):
-        return False
-
-    def upload_enabled(self):
-        return True
-
-    def location(self, artifact):
-        return ''  # URL
-
-    def availability(self, artifacts):
-        # Ensure artifacts is a list
-        artifacts = utils.as_list(artifacts)
-
-        present = set()
-        missing = set()
-
-        for artifact in artifacts:
-            if self.location(artifact):
-                present.add(artifact)
-            else:
-                missing.add(artifact)
-
-        return list(present), list(missing)
-
-
-class StorageProviderFactory(StorageProvider):
-    def create(self):
-        pass
-
-
-def RegisterStorage(cls):
-    ArtifactCache.storage_provider_factories.append(cls)
-
-
-class ArtifactAttributeSet(object):
-    def __init__(self):
-        super(ArtifactAttributeSet, self).__setattr__("_attributes", {})
-
-    def _get_attributes(self):
-        return self._attributes
-
-    def __getattr__(self, name):
-        attributes = self._get_attributes()
-        if name not in attributes:
-            attributes[name] = self.create(name)
-        return attributes[name]
-
-    def __setattr__(self, name, value):
-        attributes = self._get_attributes()
-        if name not in attributes:
-            attributes[name] = self.create(name)
-        attributes[name].set_value(value)
-        return attributes[name]
-
-    def __dict__(self):
-        return {key: str(value) for key, value in self.items()}
-
-    def items(self):
-        return self._get_attributes().items()
-
-    def apply(self, task, artifact):
-        for _, value in self.items():
-            value.apply(task, artifact)
-
-    def apply_deps(self, task, deps):
-        pass
-
-    def unapply(self, task, artifact):
-        for _, value in self.items():
-            value.unapply(task, artifact)
-
-    def unapply_deps(self, task, deps):
-        pass
-
-    def visit(self, task, artifact, visitor):
-        for _, value in self.items():
-            value.visit(task, artifact, visitor)
-
-
 class ArtifactAttributeSetRegistry(object):
-    providers = []
+    """
+    Registry for providers of artifact attribute sets.
+    """
+
+    providers = []  # List of objects that implement ArtifactAttributeSetProvider
 
     @staticmethod
     def create_all(artifact):
+        """ Create all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().create(artifact)
 
     @staticmethod
     def parse_all(artifact, content):
+        """ Parse all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().parse(artifact, content)
 
     @staticmethod
     def format_all(artifact, content):
+        """ Format all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().format(artifact, content)
 
     @staticmethod
     def apply_all(task, artifact):
+        """ Apply all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().apply(task, artifact)
 
-    @staticmethod
-    def apply_all_deps(task, deps):
-        for provider in ArtifactAttributeSetRegistry.providers:
-            provider().apply_deps(task, deps)
-
     @staticmethod
     def unapply_all(task, artifact):
+        """ Unapply all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().unapply(task, artifact)
 
-    @staticmethod
-    def unapply_all_deps(task, deps):
-        for provider in ArtifactAttributeSetRegistry.providers:
-            provider().unapply_deps(task, deps)
-
     @staticmethod
     def visit_all(task, artifact, visitor):
+        """ Visit all artifact attribute sets. """
         for provider in ArtifactAttributeSetRegistry.providers:
             provider().visit(task, artifact, visitor)
 
 
-def visit_artifact(task, artifact, visitor):
-    ArtifactAttributeSetRegistry.visit_all(task, artifact, visitor)
-
-
-class ArtifactAttributeSetProvider(object):
-    @staticmethod
-    def Register(cls):
-        ArtifactAttributeSetRegistry.providers.append(cls)
-
-    def create(self, artifact):
-        raise NotImplementedError()
-
-    def parse(self, artifact, content):
-        raise NotImplementedError()
-
-    def format(self, artifact, content):
-        raise NotImplementedError()
-
-    def apply(self, task, artifact):
-        pass
-
-    def apply_deps(self, task, deps):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
-
-    def unapply_deps(self, task, deps):
-        pass
+class ArtifactAttribute(object):
+    """
+    An artifact attribute.
 
-    def visit(self, task, artifact, visitor):
-        pass
+    An artifact attribute is a key-value pair that can be set and retrieved
+    from an artifact attribute set. Attributes are used to store metadata and other
+    information that is associated with an artifact. They communicate information
+    between tasks and store information that is used by tasks when they consume an artifact.
 
+    Artifact attributes can also perform actions when the artifact is consumed.
 
-class ArtifactAttribute(object):
+    """
     def __init__(self, name):
         self._name = name
 
     def get_name(self):
+        """ Get the name of the attribute. """
         return self._name
 
     def set_value(self, value, expand=True):
+        """
+        Set the value of the attribute.
+
+        Must be implemented by subclasses.
+
+        Args:
+            value: The value to set.
+            expand: If True, the value is macro expanded using the tools.expand() method.
+        """
         raise NotImplementedError()
 
     def get_value(self):
+        """
+        Get the value of the attribute.
+
+        Must be implemented by subclasses.
+        """
         raise NotImplementedError()
 
     def apply(self, task, artifact):
+        """
+        Perform an action when the artifact is being used.
+
+        Args:
+            task (Task): The task that is using the artifact.
+            artifact (Artifact): The artifact that is being used.
+
+        """
         pass
 
     def unapply(self, task, artifact):
+        """
+        Undo an action when the artifact is no longer being used.
+
+        Args:
+            task (Task): The task that is no longer using the artifact.
+            artifact (Artifact): The artifact that is no longer being used.
+        """
         pass
 
-    def __str__(self):
+    def __str__(self) -> str:
+        """
+        Get a string representation of the attribute.
+
+        Must be implemented by subclasses.
+        """
         raise NotImplementedError()
 
 
 class ArtifactStringAttribute(ArtifactAttribute):
+    """ An artifact attribute that stores a string value. """
+
     def __init__(self, artifact, name):
         self._artifact = artifact
         self._name = name
@@ -235,22 +163,27 @@ class ArtifactStringAttribute(ArtifactAttribute):
     def get_value(self):
         return self._value
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
-
-    def __str__(self):
+    def __str__(self) -> str:
         return str(self._value)
 
 
 class ArtifactListAttribute(ArtifactAttribute):
+    """ An artifact attribute that stores a list of values. """
+
     def __init__(self, artifact, name):
         self._artifact = artifact
         self._name = name
         self._value = []
 
+    def __getitem__(self, key):
+        return self._value[key]
+
+    def __getslice__(self, i, j):
+        return self._value[i:j]
+
+    def __len__(self):
+        return len(self._value)
+
     def get_name(self):
         return self._name
 
@@ -281,23 +214,16 @@ class ArtifactListAttribute(ArtifactAttribute):
     def count(self):
         return len(self.items())
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
+    def __str__(self) -> str:
+        return fs.pathsep.join(str(v) for v in self._value)
 
 
 class ArtifactFileAttribute(object):
+    """ An attribute that stores a list of source and destination path tuples for files collected into the artifact. """
+
     def __init__(self):
         self._files = []
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
-
     def append(self, src, dst):
         self._files.append((fs.as_posix(src), fs.as_posix(dst)))
 
@@ -308,8 +234,145 @@ class ArtifactFileAttribute(object):
         return self._files
 
 
+class ArtifactAttributeSet(object):
+    """
+    A set of artifact attributes.
+
+    An attribute set is a collection of attributes. Each attribute is
+    accessed using the attribute name as an attribute of the set. For
+    example, to access an attribute named 'version' in an attribute set
+    named 'strings', you would write:
+
+    .. code-block:: python
+
+        artifact.strings.version = "1.0"
+
+    """
+
+    def __init__(self):
+        super(ArtifactAttributeSet, self).__setattr__("_attributes", {})
+
+    def _get_attributes(self):
+        return self._attributes
+
+    def __getattr__(self, name) -> ArtifactAttribute:
+        """
+        Get or create an attribute by name.
+
+        Args:
+            name (str): The name of the attribute.
+
+        Returns:
+            An attribute object.
+        """
+        attributes = self._get_attributes()
+        if name not in attributes:
+            attributes[name] = self.create(name)
+        return attributes[name]
+
+    def __setattr__(self, name, value):
+        """
+        Set an attribute by name.
+
+        Args:
+            name (str): The name of the attribute.
+            value: The value to set.
+        """
+        attributes = self._get_attributes()
+        if name not in attributes:
+            attributes[name] = self.create(name)
+        attributes[name].set_value(value)
+        return attributes[name]
+
+    def __dict__(self):
+        """ Get a dictionary representation of the attribute set. """
+        return {key: str(value) for key, value in self.items()}
+
+    def __iter__(self):
+        """ Iterate over the attribute set. """
+        return iter(self.items())
+
+    def get(self, name, default=None):
+        """ Get an attribute by name.
+
+        Args:
+            name (str): The name of the attribute.
+
+        Returns:
+            The attribute object, or None if it does not exist.
+        """
+        attributes = self._get_attributes()
+        return attributes.get(name, default)
+
+    def items(self):
+        """ Get a list of tuples containing the attribute name and value. """
+        return self._get_attributes().items()
+
+    def apply(self, task, artifact):
+        """ Perform attribute actions when the artifact is being used. """
+        for _, value in self.items():
+            value.apply(task, artifact)
+
+    def unapply(self, task, artifact):
+        """ Undo attribute actions when the artifact is no longer being used. """
+        for _, value in self.items():
+            value.unapply(task, artifact)
+
+    def visit(self, task, artifact, visitor):
+        """ Visit all attributes in the set. """
+        for _, value in self.items():
+            value.visit(task, artifact, visitor)
+
+
+class ArtifactAttributeSetProvider(object):
+    """ Base class for artifact attribute set providers.
+
+    An artifact attribute set provider is a factory for creating and managing
+    attribute sets in an artifact.
+    """
+
+    @staticmethod
+    def Register(cls):
+        """ Decorator for registering a provider class. """
+        ArtifactAttributeSetRegistry.providers.append(cls)
+
+    def create(self, artifact):
+        """ Create an attribute set for an artifact. """
+        raise NotImplementedError()
+
+    def parse(self, artifact, content):
+        """
+        Parse an attribute set from a dictionary.
+
+        The dictionary is loaded from a JSON file embedded in the artifact.
+        """
+        raise NotImplementedError()
+
+    def format(self, artifact, content):
+        """
+        Format an attribute set to a dictionary.
+
+        The dictionary is saved to a JSON file embedded in the artifact.
+        """
+        raise NotImplementedError()
+
+    def apply(self, task, artifact):
+        """ Perform actions when the artifact is being used. """
+        pass
+
+    def unapply(self, task, artifact):
+        """ Undo actions when the artifact is no longer being used. """
+        pass
+
+    def visit(self, task, artifact, visitor):
+        """ Visit all attributes in the set. """
+        pass
+
+
 @ArtifactAttributeSetProvider.Register
 class ArtifactFileAttributeProvider(ArtifactAttributeSetProvider):
+    """ Provider for the artifact 'files' attribute set. """
+
     def create(self, artifact):
         setattr(artifact, "files", ArtifactFileAttribute())
 
@@ -323,22 +386,19 @@ class ArtifactFileAttributeProvider(ArtifactAttributeSetProvider):
     def format(self, artifact, content):
         content["files"] = [{"src": src, "dst": dst} for src, dst in artifact.files.items()]
 
-    def apply(self, task, artifact):
-        pass
-
-    def unapply(self, task, artifact):
-        pass
 
-    def visit(self, task, artifact, visitor):
-        pass
+def visit_artifact(task, artifact, visitor):
+    ArtifactAttributeSetRegistry.visit_all(task, artifact, visitor)
 
 
 def json_serializer(obj):
+    """ JSON serializer for datetime objects. """
     if isinstance(obj, datetime):
         return dict(type="datetime", value=obj.strftime("%Y-%m-%d %H:%M:%S.%f"))
 
 
 def json_deserializer(dct):
+    """ JSON deserializer for datetime objects. """
     if dct.get("type") == "datetime":
         return datetime.strptime(dct["value"], "%Y-%m-%d %H:%M:%S.%f")
     return dct
@@ -480,6 +540,8 @@ class Artifact(object):
             self._identity = name + "@" + self._identity
         self._main = name == "main"
         self._name = name or "main"
+        self._full_name = f"{self._name}@{node.short_qualified_name}" if node else self._name
+        self._log_name = f"{self._full_name} {node.identity[:8]}" if node else self._full_name
         self._node = node
         self._session = session
         self._task = node.task if node else None
@@ -491,6 +553,18 @@ class Artifact(object):
         ArtifactAttributeSetRegistry.create_all(self)
         self.reload()
 
+    def _info(self, fmt, *args, **kwargs):
+        log.info(fmt + f" ({self._log_name})", *args, **kwargs)
+
+    def _debug(self, fmt, *args, **kwargs):
+        log.debug(fmt + f" ({self._log_name})", *args, **kwargs)
+
+    def _warning(self, fmt, *args, **kwargs):
+        log.warning(fmt + f" ({self._log_name})", *args, **kwargs)
+
+    def _error(self, fmt, *args, **kwargs):
+        log.error(fmt + f" ({self._log_name})", *args, **kwargs)
+
     def __enter__(self):
         return self
 
@@ -500,7 +574,7 @@ class Artifact(object):
     def __getattr__(self, name):
         raise_task_error(self._node, "Attempt to access invalid artifact attribute '{0}'", name)
 
-    def _write_manifest(self):
+    def _write_manifest(self, temporary=False):
         content = {}
         content["size"] = self._get_size()
         content["unpacked"] = self._unpacked
@@ -525,13 +599,19 @@ class Artifact(object):
 
         ArtifactAttributeSetRegistry.format_all(self, content)
 
-        manifest = fs.path.join(self.path, ".manifest.json")
+        if temporary:
+            manifest = fs.path.join(self.temporary_path, ".manifest.json")
+        else:
+            manifest = fs.path.join(self.final_path, ".manifest.json")
         with open(manifest, "wb") as f:
             f.write(json.dumps(content, indent=2, default=json_serializer).encode())
 
-    def _read_manifest(self):
+    def _read_manifest(self, temporary=False):
         try:
-            manifest_path = fs.path.join(self.path, ".manifest.json")
+            if temporary:
+                manifest_path = fs.path.join(self.temporary_path, ".manifest.json")
+            else:
+                manifest_path = fs.path.join(self.final_path, ".manifest.json")
             with open(manifest_path) as manifest_file:
                 content = json.load(manifest_file, object_hook=json_deserializer)
             self._valid = True
@@ -611,6 +691,17 @@ class Artifact(object):
         self._read_manifest()
         self._temporary = not self._valid
 
+    def reset(self):
+        self._unpacked = False
+        self._uploadable = True
+        self._created = datetime.now()
+        self._modified = datetime.now()
+        self._expires = self._task.expires if not self._session else expires.Immediately()
+        self._size = 0
+        self._influence = None
+        self._valid = False
+        self._temporary = True
+
     @property
     def name(self):
         """ str: The name of the artifact. Default: 'main'. """
@@ -670,7 +761,7 @@ class Artifact(object):
         raise_task_error_if(
             not self.is_temporary(),
             self._node,
-            "Can't collect files into an already published task artifact")
+            "Can't collect files into an already published task artifact ({})", self._log_name)
 
         files = self.tools.expand_path(files)
         files = self.tools.glob(files)
@@ -741,7 +832,7 @@ class Artifact(object):
         raise_task_error_if(
             self.is_temporary(),
             self._node,
-            "Can't copy files from an unpublished task artifact")
+            "Can't copy files from an unpublished task artifact ({})", self._log_name)
 
         files = fs.path.join(self._path, files)
         files = self.tools.expand_path(files)
@@ -807,13 +898,15 @@ class Artifact(object):
     def get_node(self):
         return self._node
 
-    def is_temporary(self):
+    def is_temporary(self) -> bool:
         return self._temporary
 
-    def is_unpackable(self):
+    def is_unpackable(self) -> bool:
         if not self._node:
             return True
-        return self._node.is_unpackable()
+        if self.name == "main":
+            return self._task.unpack.__func__ is not tasks.Task.unpack
+        return getattr(self._task, "unpack_" + self.name, tasks.Task.unpack) is not tasks.Task.unpack
 
     def is_unpacked(self):
         return self._unpacked
@@ -824,6 +917,8 @@ class Artifact(object):
     def is_cacheable(self):
         if not self._node:
             return True
+        if self.is_session():
+            return True
         return self.task.is_cacheable()
 
     @property
@@ -838,6 +933,38 @@ class Artifact(object):
         return self._node.task
 
 
+class ArtifactToolsProxy(object):
+    """
+    An artifact proxy that uses a specific tools object.
+
+    Used when artifacts are consumed by tasks. The proxy allows the
+    task to access the artifact's methods and attributes using the
+    task's own tools object. This is useful when the consumer task
+    wishes to copy files, read files, etc, using the current working
+    directory and environment of the task.
+    """
+
+    def __init__(self, artifact, tools):
+        self._artifact = artifact
+        self._tools = tools
+
+    def __getattr__(self, name):
+        if name == "tools":
+            return self._tools
+        if name == "_artifact":
+            return self._artifact
+        attr = getattr(self._artifact.__class__, name, None)
+        if attr is not None and callable(attr):
+            return attr.__get__(self, ArtifactToolsProxy)
+        return getattr(self._artifact, name)
+
+    def __setattr__(self, name, value):
+        if name == "_artifact" or name == "_tools":
+            super(ArtifactToolsProxy, self).__setattr__(name, value)
+        else:
+            setattr(self._artifact, name, value)
+
+
 class Context(object):
     """
     Execution context and dependency wrapper.
@@ -863,18 +990,14 @@ class Context(object):
             for dep in reversed(self._node.children):
                 for artifact in dep.artifacts:
                     # Create clone with tools from this task
-                    artifact = self._cache.get_artifact(
-                        dep,
-                        name=artifact.name,
-                        session=artifact.is_session(),
-                        tools=self._node.tools,
-                    )
+                    artifact = ArtifactToolsProxy(artifact, self._node.tools)
 
                     # Don't include session artifacts that don't exist,
-                    # i.e. were no build has taken place due to presence
+                    # i.e. where no build has taken place due to presence
                     # of the persistent artifacts.
-                    if artifact.is_session() and not self._cache.is_available_locally(artifact):
-                        continue
+                    if not dep.is_resource():
+                        if artifact.is_session() and not self._cache.is_available_locally(artifact):
+                            continue
 
                     self._cache.unpack(artifact)
 
@@ -886,10 +1009,8 @@ class Context(object):
                     self._artifacts_index[artifact.name + "@" + dep.short_qualified_name] = artifact
                     artifact.apply()
                     ArtifactAttributeSetRegistry.apply_all(self._node.task, artifact)
-            ArtifactAttributeSetRegistry.apply_all_deps(self._node.task, self)
-        except Exception as e:
+        except (Exception, KeyboardInterrupt) as e:
             # Rollback all attributes/resources except the last failing one
-            ArtifactAttributeSetRegistry.unapply_all_deps(self._node.task, self)
             for name, artifact in reversed(list(self._artifacts.items())[:-1]):
                 with utils.ignore_exception():
                     ArtifactAttributeSetRegistry.unapply_all(self._node.task, artifact)
@@ -898,7 +1019,6 @@ class Context(object):
         return self
 
     def __exit__(self, type, value, tb):
-        ArtifactAttributeSetRegistry.unapply_all_deps(self._node.task, self)
         for name, artifact in reversed(self._artifacts.items()):
             ArtifactAttributeSetRegistry.unapply_all(self._node.task, artifact)
             artifact.unapply()
@@ -925,7 +1045,7 @@ class Context(object):
         key = self._node.tools.expand(key)
 
         alias, artifact, task, params = utils.parse_aliased_task_name(key)
-        raise_task_error_if(alias, self._node, "Cannot define alias when indexing dependencies")
+        raise_task_error_if(alias, self._node, "Cannot define alias when indexing dependencies: {}", alias)
         task_name = utils.format_task_name(task, params)
         task_artifact_name = utils.format_task_name(task, params, artifact)
 
@@ -960,11 +1080,148 @@ class Context(object):
 
 
 class PidProvider(object):
-    def __init__(self):
-        self._uuid = uuid.uuid4()
-
     def __call__(self):
-        return str(self._uuid)
+        pid = str(uuid.uuid4())
+        log.debug("New cache lock file: {0}", pid)
+        return pid
+
+
+class StorageProvider(object):
+    """
+    Base class for remote artifact storage providers.
+
+    A storage provider is responsible for uploading and downloading
+    artifacts to and from a remote storage location. The storage
+    location can be a file system path, a cloud storage service, or
+    any other type of storage.
+
+    """
+
+    def download(self, artifact: Artifact, force: bool = False) -> bool:
+        """
+        Download an artifact from the storage location.
+
+        The should be downloaded to the path returned by the artifact's
+        :func:`~jolt.Artifact.get_archive_path` method. The downloaded artifact
+        must be in the format specified by DEFAULT_ARCHIVE_TYPE.
+
+        The download should be retried if it fails due to network issues.
+        The method may raise an exception on errors.
+
+        Args:
+            artifact (Artifact): The artifact to download.
+            force (bool, optional): If True, the download should be forced,
+                even if the artifact is already present locally, or if the
+                download is disabled. The default is False.
+
+        Returns:
+            bool: True if the download was successful, False otherwise.
+
+        """
+        return False
+
+    def download_enabled(self) -> bool:
+        """ Return True if downloading is enabled. Default is True. """
+        return True
+
+    def upload(self, artifact: Artifact, force: bool = False) -> bool:
+        """
+        Upload an artifact to the storage location.
+
+        The artifact to be uploaded is located at the path returned by
+        the artifact's :func:`~jolt.Artifact.get_archive_path` method. The
+        uploaded artifact is in the format specified by DEFAULT_ARCHIVE_TYPE.
+        The provider may choose to upload the artifact using a different
+        format, but it must be able to download the artifact in the
+        DEFAULT_ARCHIVE_TYPE format.
+
+        The upload should be retried if it fails due to network issues.
+        The method may raise an exception on errors.
+
+        Args:
+            artifact (Artifact): The artifact to upload.
+            force (bool, optional): If True, the upload should be forced,
+                even if the artifact is already present remotely, or if the
+                upload is disabled. The default is False.
+
+        Returns:
+            bool: True if the upload was successful, False otherwise.
+
+        """
+        return False
+
+    def upload_enabled(self) -> bool:
+        """ Return True if uploading is enabled. Default is True. """
+        return True
+
+    def location(self, artifact) -> str:
+        """
+        Return the URL of the artifact in the storage location.
+
+        This method is sometimes used to identify if an artifact is
+        present in the storage location. The URL should point to the
+        artifact if present, or an empty string if the artifact is
+        absent.
+
+        Args:
+            artifact (Artifact): The artifact to locate.
+        """
+        return ''  # URL
+
+    def availability(self, artifacts: list) -> tuple:
+        """
+        Check the availability of a list of artifacts.
+
+        This method is used to determine which artifacts are present in the
+        storage location. The method should return a tuple of two lists:
+        the first list contains the artifacts that are present, and the
+        second list contains the artifacts that are missing.
+
+        The default implementation of this method calls the :func:`~jolt.StorageProvider.location`
+        method for each artifact in the list. Subclasses may override this
+        method to provide a more efficient implementation.
+
+        Args:
+            artifacts (list): A list of artifacts to check.
+
+        Returns:
+            tuple: A tuple of two lists: the first list contains the artifacts
+                that are present, and the second list contains the artifacts
+                that are missing.
+
+        """
+        # Ensure artifacts is a list
+        artifacts = utils.as_list(artifacts)
+
+        present = set()
+        missing = set()
+
+        for artifact in artifacts:
+            if self.location(artifact):
+                present.add(artifact)
+            else:
+                missing.add(artifact)
+
+        return list(present), list(missing)
+
+
+class StorageProviderFactory(StorageProvider):
+    """ A factory for store providers. """
+
+    def create(self) -> StorageProvider:
+        """
+        Create a new storage provider.
+
+        This method should return a new instance of a storage provider,
+        which must be a subclass of :class:`~jolt.StorageProvider`.
+
+        """
+        pass
+
+
+def RegisterStorage(cls):
+    """ Decorator used to register a storage provider factory. """
+    ArtifactCache.storage_provider_factories.append(cls)
 
 
 @utils.Singleton
@@ -1032,13 +1289,12 @@ class ArtifactCache(StorageProvider):
 
         # If no storage providers supports the availability method,
         # we will not only use the local presence cache.
-        self._local_presence_cache = set()
         self._remote_presence_cache = set()
         self._presence_cache_only = self.has_availability()
 
         # Read configuration
         self._max_size = config.getsize(
-            "jolt", "cachesize", os.environ.get("JOLT_CACHESIZE", 1 * 1024 ** 3))
+            "jolt", "cachesize", os.environ.get("JOLT_CACHE_SIZE", 1 * 1024 ** 3))
 
         # Create cache directory
         self._fs_create_cachedir()
@@ -1047,10 +1303,12 @@ class ArtifactCache(StorageProvider):
         self._cache_locked = False
         self._lock_file = fasteners.InterProcessLock(self._fs_get_lock_file())
         self._thread_lock = RLock()
+        self._artifact_thread_lock = utils.IdLock()
 
         # Create process lock file
         with self._cache_lock():
-            self._pid = pidprovider() if pidprovider else PidProvider()()
+            self._pid_provider = pidprovider or PidProvider()
+            self._pid = self._pid_provider()
             self._pid_file = fasteners.InterProcessLock(self._fs_get_pid_file(self._pid))
             self._pid_file.acquire()
 
@@ -1082,6 +1340,7 @@ class ArtifactCache(StorageProvider):
         db = sqlite3.connect(self._db_path, detect_types=sqlite3.PARSE_DECLTYPES)
         try:
             db.execute("PRAGMA journal_mode=OFF")
+            # db.set_trace_callback(log.warning)
             yield db
         finally:
             db.close()
@@ -1279,14 +1538,16 @@ class ArtifactCache(StorageProvider):
     def _fs_get_artifact(self, node, name, tools=None, session=False):
         return Artifact(self, node, name=name, tools=tools, session=session)
 
-    def _fs_commit_artifact(self, artifact, uploadable):
+    def _fs_commit_artifact(self, artifact: Artifact, uploadable: bool, temporary: bool):
         artifact._set_uploadable(uploadable)
         if not artifact.is_unpackable():
             artifact._set_unpacked()
-        artifact._write_manifest()
-        if artifact.is_temporary():
+        if temporary:
+            artifact._write_manifest(temporary=True)
             fs.rmtree(artifact.final_path, ignore_errors=True)
-            fs.rename(artifact.path, artifact.final_path)
+            fs.rename(artifact.temporary_path, artifact.final_path)
+        else:
+            artifact._write_manifest(temporary=False)
 
     @contextlib.contextmanager
     def _fs_compress_artifact(self, artifact):
@@ -1295,14 +1556,14 @@ class ArtifactCache(StorageProvider):
 
         raise_task_error_if(
             artifact.is_temporary(), task,
-            "Can't compress an unpublished task artifact")
+            "Can't compress an unpublished task artifact ({})", artifact._log_name)
 
         try:
             artifact.tools.archive(artifact.path, archive)
         except KeyboardInterrupt as e:
             raise e
         except Exception:
-            raise_task_error(task, "Failed to compress task artifact")
+            raise_task_error(task, "Failed to compress task artifact ({})", artifact._log_name)
         try:
             yield
         finally:
@@ -1313,29 +1574,57 @@ class ArtifactCache(StorageProvider):
         archive = artifact.get_archive_path()
         try:
             task.tools.extract(archive, artifact.temporary_path, ignore_owner=True)
+            artifact._read_manifest(temporary=True)
         except KeyboardInterrupt as e:
+            fs.rmtree(artifact.temporary_path, ignore_errors=True)
             raise e
         except Exception:
-            raise_task_error(task, "Failed to extract task artifact archive")
+            fs.rmtree(artifact.temporary_path, ignore_errors=True)
+            raise_task_error(task, "Failed to extract task artifact archive ({})", artifact._log_name)
         finally:
             fs.unlink(archive, ignore_errors=True)
-        artifact._read_manifest()
 
     def _fs_delete_artifact(self, identity, task_name, onerror=None):
         fs.rmtree(self._fs_get_artifact_path(identity, task_name), ignore_errors=True, onerror=onerror)
         fs.rmtree(self._fs_get_artifact_tmppath(identity, task_name), ignore_errors=True, onerror=onerror)
+        fs.rmtree(self._fs_get_artifact_path_legacy(identity, task_name), ignore_errors=True, onerror=onerror)
+        fs.rmtree(self._fs_get_artifact_tmppath_legacy(identity, task_name), ignore_errors=True, onerror=onerror)
         fs.unlink(fs.path.join(self.root, task_name), ignore_errors=True)
 
+    def _fs_identity(self, identity):
+        parts = identity.split("@", 1)
+        if len(parts) <= 1:
+            parts = ["main"] + parts
+        return parts[1] + "-" + utils.canonical(parts[0])
+
+    def _fs_identity_legacy(self, identity):
+        parts = identity.split("@", 1)
+        if len(parts) <= 1:
+            parts = ["main"] + parts
+        return parts[0] + "@" + utils.canonical(parts[1])
+
     def _fs_get_artifact_archivepath(self, identity, task_name):
-        return fs.get_archive(fs.path.join(self.root, task_name, identity))
+        identity = self._fs_identity(identity)
+        return fs.path.join(self.root, task_name, identity) + DEFAULT_ARCHIVE_TYPE
 
     def _fs_get_artifact_lockpath(self, identity):
+        identity = self._fs_identity(identity)
         return fs.path.join(self.root, "locks", identity + ".lock")
 
     def _fs_get_artifact_tmppath(self, identity, task_name):
+        identity = self._fs_identity(identity)
         return fs.path.join(self.root, task_name, "." + identity)
 
     def _fs_get_artifact_path(self, identity, task_name):
+        identity = self._fs_identity(identity)
+        return fs.path.join(self.root, task_name, identity)
+
+    def _fs_get_artifact_tmppath_legacy(self, identity, task_name):
+        identity = self._fs_identity_legacy(identity)
+        return fs.path.join(self.root, task_name, "." + identity)
+
+    def _fs_get_artifact_path_legacy(self, identity, task_name):
+        identity = self._fs_identity_legacy(identity)
         return fs.path.join(self.root, task_name, identity)
 
     def _fs_get_artifact_manifest_path(self, identity, task_name):
@@ -1409,16 +1698,36 @@ class ArtifactCache(StorageProvider):
         evicted = 0
         for identity, task_name, _, used in artifacts:
             if not if_expired or self._fs_is_artifact_expired(identity, task_name, used):
-                self._db_delete_artifact(db, identity)
-                self._fs_delete_artifact(identity, task_name, onerror=onerror)
-                evicted += 1
-                log.debug("Evicted {}: {}", identity, task_name)
+                with utils.delayed_interrupt():
+                    self._db_delete_artifact(db, identity)
+                    self._fs_delete_artifact(identity, task_name, onerror=onerror)
+                    evicted += 1
+                    log.debug("Evicted {}: {}", identity, task_name)
         return evicted == len(artifacts)
 
     ############################################################################
     # Public API
     ############################################################################
 
+    def release(self):
+        """
+        Release references to artifacts held by the current process.
+
+        Effectively, a new pid lock file is created and the old one is deleted. This
+        allows other processes to detect termination of the current process and
+        garbage collect any references owned by the process.
+        """
+        with self._cache_lock(), self._db() as db:
+            self._db_invalidate_locks(db, try_all=True)
+            self._db_invalidate_references(db, try_all=True)
+            self._fs_invalidate_pids(db, try_all=True)
+            self._pid_file.release()
+
+            self._pid = self._pid_provider()
+            self._pid_file = fasteners.InterProcessLock(self._fs_get_pid_file(self._pid))
+            self._pid_file.acquire()
+
+    @utils.delay_interrupt
     def is_available_locally(self, artifact):
         """
         Check presence of task artifact in cache.
@@ -1430,20 +1739,13 @@ class ArtifactCache(StorageProvider):
         if not artifact.is_cacheable():
             return False
 
-        # Cache availability in node
-        try:
-            assert artifact.identity in self._local_presence_cache
-        except AssertionError:
-            pass
-
         with self._cache_lock(), self._db() as db:
             if self._db_select_artifact(db, artifact.identity) or self._db_select_reference(db, artifact.identity):
                 artifact.reload()
                 if artifact.is_temporary():
-                    self._db_delete_artifact(db, artifact.identity)
+                    self._db_delete_artifact(db, artifact.identity, and_refs=False)
                     return False
                 self._db_insert_reference(db, artifact.identity)
-                self._local_presence_cache.add(artifact.identity)
                 return True
         return False
 
@@ -1471,7 +1773,7 @@ class ArtifactCache(StorageProvider):
         # Returns true if all storage providers implement the availability method
         return all([provider.availability.__func__ != StorageProvider.availability for provider in self._storage_providers])
 
-    def availability(self, artifacts):
+    def availability(self, artifacts, remote=True):
         """ Check presence of task artifacts in any cache, local or remote """
         present = set()
         missing = set()
@@ -1486,6 +1788,9 @@ class ArtifactCache(StorageProvider):
             else:
                 missing.add(artifact)
 
+        if not remote:
+            return list(present), list(missing)
+
         # Check presence of all artifacts in the remote caches
         missing_remotely = artifacts
 
@@ -1507,31 +1812,35 @@ class ArtifactCache(StorageProvider):
         return self._options.download and \
             any([provider.download_enabled() for provider in self._storage_providers])
 
+    def download_session_enabled(self):
+        return self._options.download_session and \
+            any([provider.download_enabled() for provider in self._storage_providers])
+
     def upload_enabled(self):
         return self._options.upload and \
             any([provider.upload_enabled() for provider in self._storage_providers])
 
-    def download_all(self, node, force=False):
-        pass
-
     def download(self, artifact, force=False):
         """
         Downloads an artifact from a remote cache to the local cache.
 
         The artifact is interprocess locked during the operation.
         """
-        if not force and not self.download_enabled():
-            return False
+        if not force:
+            if not artifact.is_session() and not self.download_enabled():
+                return False
+            if artifact.is_session() and not self.download_session_enabled():
+                return False
         if not artifact.is_cacheable():
             return False
-        with self.lock_artifact(artifact) as artifact:
+        with self.lock_artifact(artifact, why="download") as artifact:
             if self.is_available_locally(artifact):
-                artifact.task.info("Download skipped, already in local cache ({name})")
+                artifact._info("Download skipped, already in local cache")
                 return True
             for provider in self._storage_providers:
                 if provider.download(artifact, force):
                     self._fs_decompress_artifact(artifact)
-                    self.commit(artifact)
+                    self.commit(artifact, temporary=True)
                     return True
         return len(self._storage_providers) == 0
 
@@ -1547,11 +1856,11 @@ class ArtifactCache(StorageProvider):
             return True
         raise_task_error_if(
             not self.is_available_locally(artifact), artifact.task,
-            "Can't upload task artifact, no artifact present in the local cache")
-        with self.lock_artifact(artifact) if locked else artifact as artifact:
+            "Can't upload task artifact, no artifact present in the local cache ({})", artifact._log_name)
+        with self.lock_artifact(artifact, why="upload") if locked else artifact as artifact:
             raise_task_error_if(
                 not artifact.is_uploadable(), artifact.task,
-                "Artifact was modified locally by another process and can no longer be uploaded, try again")
+                "Artifact was modified locally by another process and can no longer be uploaded, try again ({})", artifact._log_name)
             if self._storage_providers:
                 with self._fs_compress_artifact(artifact):
                     return all([provider.upload(artifact, force) for provider in self._storage_providers])
@@ -1578,18 +1887,24 @@ class ArtifactCache(StorageProvider):
         """
         if not artifact.is_unpackable():
             return True
-        with self._thread_lock, self.lock_artifact(artifact) as artifact:
-            if not self.is_available_locally(artifact):
-                raise_task_error(
-                    artifact.task,
-                    "Locked artifact is missing in cache (forcibly removed?)")
+        with self._thread_lock, self.lock_artifact(artifact, why="unpack") as artifact:
+            raise_task_error_if(
+                not self.is_available_locally(artifact),
+                artifact.task,
+                "Locked artifact is missing in cache (forcibly removed?) ({})", artifact._log_name)
+
+            raise_task_error_if(
+                artifact.is_temporary(),
+                artifact.task,
+                "Can't unpack an unpublished task artifact ({})", artifact._log_name)
+
             if artifact.is_unpacked():
                 return True
 
             # Keep a temporary copy of the artifact if the task
             # unpack() method fails. The copy is removed in
             # get_locked_artifact() if left unused.
-            fs.copy(artifact.path, artifact.temporary_path, symlinks=True)
+            fs.copy(artifact.final_path, artifact.temporary_path, symlinks=True)
 
             task = artifact.task
             with tools.Tools(task) as t:
@@ -1597,23 +1912,32 @@ class ArtifactCache(StorageProvider):
                     # Note: unpack() will run on the original
                     # artifact, not in the temporary copy.
                     if task.unpack.__func__ is not tasks.Task.unpack:
-                        task.info("Unpack started {}", artifact.get_node().log_name)
+                        artifact._info("Unpack started")
                     artifact._set_unpacked()
-                    task.unpack(artifact, t)
-                    self.commit(artifact, uploadable=False)
+                    if artifact.name == "main":
+                        task.unpack(artifact, t)
+                    else:
+                        unpack = getattr(task, "unpack_" + artifact.name, None)
+                        raise_task_error_if(
+                            unpack is None, task,
+                            "Artifact unpack method not found: unpack_{}", artifact.name)
+                        unpack(artifact, t)
+
+                    self.commit(artifact, uploadable=False, temporary=False)
 
                 except NotImplementedError:
-                    self.commit(artifact)
+                    self.commit(artifact, temporary=False)
 
-                except Exception as e:
+                except (Exception, KeyboardInterrupt) as e:
                     # Restore the temporary copy
-                    fs.rmtree(artifact.path, ignore_errors=True)
-                    fs.rename(artifact.temporary_path, artifact.path)
-                    artifact.task.error("Unpack failed {}", artifact.get_node().log_name)
+                    fs.rmtree(artifact.final_path, ignore_errors=True)
+                    fs.rename(artifact.temporary_path, artifact.final_path)
+                    artifact._error("Unpack failed")
                     raise e
         return True
 
-    def commit(self, artifact, uploadable=True):
+    @utils.delay_interrupt
+    def commit(self, artifact, uploadable=True, temporary=True):
         """
         Commits a task artifact to the cache.
 
@@ -1629,7 +1953,7 @@ class ArtifactCache(StorageProvider):
             return
 
         with self._cache_lock(), self._db() as db:
-            self._fs_commit_artifact(artifact, uploadable)
+            self._fs_commit_artifact(artifact, uploadable, temporary)
             with utils.ignore_exception():  # Possibly already exists in DB, e.g. unpacked
                 self._db_insert_artifact(db, artifact.identity, artifact.task.canonical_name, artifact.get_size())
             self._db_update_artifact_size(db, artifact.identity, artifact.get_size())
@@ -1646,19 +1970,17 @@ class ArtifactCache(StorageProvider):
                 if self._discard(db, [candidate], True):
                     evict_size -= candidate[2]
 
+    @utils.delay_interrupt
     def discard(self, artifact, if_expired=False, onerror=None):
         with self._cache_lock(), self._db() as db:
             self._db_invalidate_locks(db)
             self._db_invalidate_references(db)
             self._fs_invalidate_pids(db)
-            discarded = self._discard(
+            return self._discard(
                 db,
                 self._db_select_artifact_not_in_use(db, artifact.identity),
                 if_expired,
                 onerror=onerror)
-            if discarded:
-                self._local_presence_cache.discard(artifact.identity)
-            return discarded
 
     def _discard_wait(self, artifact):
         """
@@ -1677,10 +1999,11 @@ class ArtifactCache(StorageProvider):
             artifacts = self._db_select_artifact(db, artifact.identity)
             self._db_delete_artifact(db, artifact.identity, and_refs=False)
             refpids = self._db_select_artifact_reference_pids(db, artifact.identity)
+            refpids = list(filter(lambda pid: pid != self._pid, refpids))
             lockpids = self._db_select_artifact_lock_pids(db, artifact.identity)
 
-        if len(refpids) > 1:
-            artifact.task.info("Artifact is temporarily in use, forced discard on hold ({name})")
+        if len(refpids) > 0:
+            artifact._info("Artifact is temporarily in use, forced discard on hold")
             for pid in refpids:
                 # Loop waiting for other processes to surrender the artifact
                 while True:
@@ -1700,8 +2023,7 @@ class ArtifactCache(StorageProvider):
 
         with self._cache_lock(), self._db() as db:
             assert self._discard(db, artifacts, False), "Failed to discard artifact"
-            self._local_presence_cache.discard(artifact.identity)
-            artifact.reload()
+            artifact.reset()
         return artifact
 
     def discard_all(self, if_expired=False, onerror=None):
@@ -1719,10 +2041,29 @@ class ArtifactCache(StorageProvider):
         return Context(self, node)
 
     def get_artifact(self, node, name, tools=None, session=False):
-        return self._fs_get_artifact(node, name=name, tools=tools, session=session)
+        artifact = self._fs_get_artifact(node, name=name, tools=tools, session=session)
+        if not artifact.is_temporary():
+            with self._cache_lock(), self._db() as db:
+                if not self._db_select_artifact(db, artifact.identity) and not self._db_select_reference(db, artifact.identity):
+                    log.verbose("Artifact not present in db, discarding archive ({} )", artifact.task.short_qualified_name, artifact.identity)
+                    fs.rmtree(artifact.final_path, ignore_errors=True)
+                    artifact.reload()
+        return artifact
 
     @contextlib.contextmanager
-    def lock_artifact(self, artifact, discard=False):
+    def lock_artifact(self, artifact: Artifact, discard: bool = False, why: str = "publish"):
+        """
+        Locks the task artifact, both with process thread locks and interprocess file locks.
+        """
+        try:
+            self._artifact_thread_lock.acquire(artifact.identity)
+            with self._lock_artifact_interprocess(artifact, discard=discard, why=why) as artifact:
+                yield artifact
+        finally:
+            self._artifact_thread_lock.release(artifact.identity)
+
+    @contextlib.contextmanager
+    def _lock_artifact_interprocess(self, artifact: Artifact, discard: bool = False, why: str = "publish"):
         """
         Locks the task artifact.
 
@@ -1738,21 +2079,24 @@ class ArtifactCache(StorageProvider):
             lock = fasteners.InterProcessLock(lock_path)
             is_locked = lock.acquire(blocking=False)
         if not is_locked:
-            artifact.task.info("Artifact is temporarily locked by another process ({name})")
+            artifact._info("Artifact is temporarily locked by another process")
             lock.acquire()
 
+        artifact._debug("Artifact locked for {}", why)
+
         try:
-            artifact.reload()
             if discard:
                 artifact = self._discard_wait(artifact)
+            else:
+                artifact.reload()
+
             if artifact.is_temporary():
                 fs.rmtree(artifact.temporary_path, ignore_errors=True)
                 fs.makedirs(artifact.temporary_path)
 
-            with contextlib.ExitStack() as stack:
-                stack.enter_context(artifact)
-                yield artifact
+            yield artifact
         finally:
+            artifact._debug("Artifact unlocked for {}", why)
             fs.rmtree(artifact.temporary_path, ignore_errors=True)
             with self._cache_lock():
                 with self._db() as db:
@@ -1762,9 +2106,9 @@ class ArtifactCache(StorageProvider):
                     if self._db_select_lock_count(db, artifact.identity) == 0:
                         fs.unlink(lock_path, ignore_errors=True)
 
-    def precheck(self, artifacts):
+    def precheck(self, artifacts, remote=True):
         """ Precheck artifacts for availability and cache status. """
         if not self.has_availability():
             return
-        present, missing = self.availability(artifacts)
+        present, missing = self.availability(artifacts, remote=remote)
         log.verbose("Cache: {}/{} artifacts present", len(present), len(artifacts))