jolt 0.9.123__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/tasks.py

@@ -7,10 +7,13 @@ import fnmatch
 import functools
 import hashlib
 import platform
+from threading import RLock
 import subprocess
 from os import environ
+from os import sys as os_sys
 import sys
 import unittest as ut
+from urllib.parse import urlparse
 import uuid
 import re
 import traceback
@@ -21,10 +24,11 @@ from jolt import utils
 from jolt.cache import ArtifactAttributeSetProvider
 from jolt.error import raise_error_if, raise_task_error, raise_task_error_if
 from jolt.error import raise_unreported_task_error_if
-from jolt.error import JoltError, JoltCommandError
+from jolt.error import JoltError, JoltCommandError, LoggedJoltError
 from jolt.expires import Immediately
 from jolt.influence import FileInfluence, TaintInfluenceProvider
 from jolt.influence import TaskClassSourceInfluence
+from jolt.influence import CallbackInfluence
 from jolt.influence import attribute as attribute_influence
 from jolt.influence import environ as environ_influence
 from jolt.influence import source as source_influence
@@ -133,7 +137,7 @@ class Parameter(object):
         self.name = name
 
     def __init__(self, default=None, values=None, required=True,
-                 const=False, influence=True, help=None):
+                 const=False, influence=True, help=None, valuesfn=None):
         """
         Creates a new parameter.
 
@@ -141,6 +145,10 @@ class Parameter(object):
             default (str, optional): An optional default value.
             values (list, optional): A list of accepted values. An
                 assertion is raised if an unlisted value is assigned to the parameter.
+            valuesfn (func, optional); A function that validates the assigned value.
+                If both values and valuesfn are specified, the values list is validated
+                first. The function is passed the assigned value and should return.
+                boolean value. If the function returns False, a ParameterValueError is raised.
             required (boolean, optional): If required, the parameter must be assigned
                 a value before the task can be executed. The default is ``True``.
             const (boolean, optional): If const is True, the parameter is immutable
@@ -162,6 +170,7 @@ class Parameter(object):
         self._default = default
         self._value = default
         self._accepted_values = values
+        self._accepted_values_fn = valuesfn
         self._required = required
         self._const = const
         self._influence = influence
@@ -188,7 +197,7 @@ class Parameter(object):
         def highlight(value):
             return colors.bright(value) if self._is_default(value) else colors.dim(value)
 
-        return "[{}]".format(", ".join([highlight(value) for value in accepted])) if accepted else ""
+        return "[{}]".format(", ".join([highlight(str(value)) for value in accepted])) if accepted else ""
 
     def __str__(self):
         """ Returns the parameter value as a string """
@@ -197,6 +206,8 @@ class Parameter(object):
     def _validate(self, value, what=None):
         if self._accepted_values is not None and value not in self._accepted_values:
             raise ParameterValueError(self, value, what=what)
+        if self._accepted_values_fn is not None and not self._accepted_values_fn(value):
+            raise ParameterValueError(self, value, what=what)
 
     def get_default(self):
         """ Get the default value of the parameter.
@@ -350,7 +361,7 @@ class BooleanParameter(Parameter):
 
         """
         default = str(default).lower() if default is not None else None
-        super(BooleanParameter, self).__init__(
+        super().__init__(
             default,
             values=["false", "true", "0", "1", "no", "yes"],
             required=required,
@@ -372,7 +383,7 @@ class BooleanParameter(Parameter):
             ParameterValueError: If the parameter is assigned an illegal value.
         """
         value = str(value).lower()
-        super(BooleanParameter, self).set_value(value)
+        super().set_value(value)
 
     @property
     def is_true(self):
@@ -434,7 +445,7 @@ class IntParameter(Parameter):
     """
 
     def __init__(self, default=None, min=None, max=None, values=None, required=True, const=False,
-                 influence=True, help=None):
+                 influence=True, help=None, valuesfn=None):
         """
         Creates a new parameter.
 
@@ -482,7 +493,8 @@ class IntParameter(Parameter):
             required=required,
             const=const,
             influence=influence,
-            help=help)
+            help=help,
+            valuesfn=valuesfn)
 
     def _validate(self, value, what=None):
         if self._min is not None and value < self._min:
@@ -686,6 +698,10 @@ class ListParameter(Parameter):
             for item in value:
                 if item not in self._accepted_values:
                     raise ParameterValueError(self, item, what=what)
+        if self._accepted_values_fn is not None:
+            for item in value:
+                if not self._accepted_values_fn(item):
+                    raise ParameterValueError(self, item, what=what)
 
     def get_value(self):
         return "+".join(self._value)
@@ -767,6 +783,7 @@ class TaskRegistry(object):
         self.env = env
         self.tasks = {}
         self.instances = {}
+        self._workspace_resources = []
 
     @staticmethod
     def get(*args, **kwargs):
@@ -775,6 +792,21 @@ class TaskRegistry(object):
         return TaskRegistry._instance
 
     def add_task_class(self, cls):
+        """
+        Add a task class to the registry.
+
+        The class is decorated to require workspace resources.
+        """
+
+        registry = self
+
+        def _workspace_resources(self):
+            return registry._workspace_resources
+
+        if not issubclass(cls, WorkspaceResource):
+            cls = attributes.requires("_workspace_resources")(cls)
+            cls._workspace_resources = property(_workspace_resources)
+
         self.tasks[cls.name] = cls
 
     def add_task(self, task, extra_params):
@@ -783,13 +815,20 @@ class TaskRegistry(object):
         full_name = utils.format_task_name(name, params)
         self.instances[full_name] = task
 
+    def require_workspace_resource(self, taskname):
+        name, _ = utils.parse_task_name(taskname)
+        task = self.get_task_class(name)
+        raise_task_error_if(task is None, name, "Resource not found")
+        raise_task_error_if(not issubclass(task, WorkspaceResource), name, "Not a workspace resource")
+        self._workspace_resources.append(taskname)
+
     def get_task_class(self, name):
         return self.tasks.get(name)
 
     def get_task_classes(self):
         return list(self.tasks.values())
 
-    def get_task(self, name, extra_params=None, manifest=None):
+    def get_task(self, name, extra_params=None, manifest=None, buildenv=None):
         name, params = utils.parse_task_name(name)
         params.update(extra_params or {})
         full_name = utils.format_task_name(name, params)
@@ -800,14 +839,19 @@ class TaskRegistry(object):
 
         cls = self.tasks.get(name)
         if cls:
-            task = cls(parameters=params, manifest=manifest)
+            task = cls(parameters=params, manifest=manifest, buildenv=buildenv)
             task = self.instances.get(task.qualified_name, task)
-            self.instances[task.qualified_name] = task
-            self.instances[full_name] = task
+            if not isinstance(task, Resource) or isinstance(task, WorkspaceResource):
+                self.instances[task.qualified_name] = task
+                self.instances[full_name] = task
             return task
 
         raise_task_error_if(not task, full_name, "No such task")
 
+    def has_task(self, name):
+        name, params = utils.parse_task_name(name)
+        return self.tasks.get(name) is not None
+
     def set_default_parameters(self, task):
         name, params = utils.parse_task_name(task)
 
@@ -880,18 +924,195 @@ class TaskGenerator(object):
 
 class attributes:
     @staticmethod
-    def requires(attrib):
+    def arch(cls):
+        """ Return the architecture name (x86_64, arm64, etc.). """
+
+        cls._arch = Export(lambda t: platform.machine().lower().replace("amd64", "x86_64"))
+        cls.arch = property(lambda t: t._arch.value)
+        return cls
+
+    @staticmethod
+    def artifact(name, session=False):
+        """Decorator adding an additional artifact to a task.
+
+        Jolt calls the new publish method `publish_<name>` with the
+        new artifact as argument. Non-alphanumeric characters in the
+        name are replaced with underscores (_). The new artifact is
+        consumable from another task by using the string format
+        `<artifact-name>@<task-name>` to index dependencies.  The
+        standard artifact is named `main`. See the example below.
+
+        If `session` is `True`, the new artifact will be a session
+        artifact that is only valid during a single Jolt invokation.
+        Session artifacts are published even if the task fails and may
+        be used to save logs and data for post-mortem analysis.
+
+        Args:
+          name (str): Name of artifact. Used as reference from
+              consuming tasks.
+          session (boolean, False): Session artifact.
+
+        Example:
+
+          .. literalinclude:: ../examples/artifacts/build.jolt
+            :language: python
+            :caption: examples/artifacts/build.jolt
+
         """
-        Decorates a task with an alternative ``requires`` attribute.
+        name = utils.canonical(name)
 
-        The new attribute will be concatenated with the regular
-        ``requires`` attribute.
+        def decorate(cls):
+            _old_artifacts = cls._artifacts
+
+            @functools.wraps(cls._artifacts)
+            def _artifacts(self, cache, node):
+                artifacts = _old_artifacts(self, cache, node)
+                artifacts += [cache.get_artifact(node, name, session=session or isinstance(cls, Resource))]
+                return artifacts
+
+            cls._artifacts = _artifacts
+            return cls
+
+        return decorate
+
+    @staticmethod
+    def artifact_upload(uri, name="main", condition=None):
+        """
+        Decorator to add uploading of an artifact to a server.
+
+        Upon successful completion of the task, the resulting
+        artifact uploaded to the specified URI. The URI should
+        be in the format `protocol://user:password@host/path`.
+
+        The following protocols are supported:
+
+          - `http://`
+          - `https://`
+          - `file://`
+
+        Local path are also supported in which case the artifact
+        is copied to the specified location.
+
+        If the path ends with a slash, the artifact is treated as
+        a directory and copied into the root of the directory.
+
+        If the path ends with a supported archive extension, the
+        artifact is archived and optionally compressed before being
+        copied. See :func:`jolt.Tools.archive` for supported archive
+        and compression formats.
+
+        Usernames and passwords are optional. If omitted, the
+        artifact is uploaded anonymously. Environment variables
+        can be used to store sensitive information, such as
+        passwords. Specify the environment variable name in the
+        URI as `protocol://{environ[USER]}:{environ[PASS]}@host/path`.
+
+        The upload can be conditioned on the return value of a
+        function. The function is passed the task instance as
+        an argument and should return a boolean value. If the
+        function returns ``False``, the upload is skipped. The value
+        of the condition influences the hash of the task.
 
         Args:
-            attrib (str): Name of alternative attribute.
-                Keywords are expanded.
+            condition (str): Condition function to evaluate before
+                uploading the artifact. The function is passed the
+                task instance as an argument and should return a
+                boolean value. By default, the artifact is always
+                uploaded.
+            name (str): Name of the artifact to upload.
+            uri (str): Destination URI for the artifact.
+
+        Example:
+
+            .. literalinclude:: ../examples/artifacts/upload.jolt
+                :language: python
+                :caption: examples/artifacts/upload.jolt
         """
-        return utils.concat_attributes("requires", attrib)
+
+        def decorate(cls):
+            if name == "main":
+                old_publish = cls.publish
+            else:
+                old_publish = getattr(cls, "publish_" + name, None)
+
+            raise_error_if(old_publish is None, f"Cannot upload artifact '{name}' from task '{cls.name or cls.__name__.lower()}', it does not exist")
+
+            def upload_file(self, tools, cwd, src, dst):
+                with tools.cwd(cwd):
+                    tools.upload(src, dst + src)
+
+            def copy_file(self, tools, cwd, src, dst):
+                with tools.cwd(cwd):
+                    tools.copy(src, dst + src)
+
+            def list_files(self, artifact, tools):
+                with tools.cwd(artifact.path):
+                    for path in tools.glob("**"):
+                        # Skip directories
+                        if tools.isdir(path):
+                            continue
+                        yield artifact.path, path
+
+            def archive_files(self, artifact, tools, filename):
+                out = tools.builddir(artifact.identity)
+                with tools.cwd(out):
+                    tools.archive(artifact.path, filename)
+                    yield out, filename
+
+            @functools.wraps(cls.publish)
+            def publish(self, artifact, tools):
+                old_publish(self, artifact, tools)
+
+                if condition and not bool(condition(self)):
+                    return
+
+                # Expand keywords
+                uri_exp = tools.expand(uri)
+
+                # Parse URI to determine protocol
+                uri_parsed = urlparse(uri_exp)
+                if uri_parsed.scheme in ["http", "https"]:
+                    action = upload_file
+                elif uri_parsed.scheme in ["", "file"]:
+                    uri_exp = uri_parsed.path
+                    action = copy_file
+                else:
+                    raise_task_error(self, f"Unsupported protocol '{uri_parsed.scheme}' in URI '{uri_exp}'")
+
+                if uri_exp.endswith("/"):
+                    generator = list_files
+                else:
+                    generator = functools.partial(archive_files, filename=fs.path.basename(uri_parsed.path))
+                    uri_exp = fs.path.dirname(uri_exp) + "/"
+
+                if action is copy_file:
+                    uri_exp = tools.expand_path(uri_exp) + "/"
+
+                for cwd, file in generator(self, artifact, tools):
+                    action(self, tools, cwd, file, uri_exp)
+
+            if name == "main":
+                setattr(cls, "publish", publish)
+            else:
+                setattr(cls, "publish_" + name, publish)
+
+            if condition:
+                old_init = cls.__init__
+
+                @functools.wraps(cls.__init__)
+                def new_init(self, *args, **kwargs):
+                    old_init(self, *args, **kwargs)
+
+                    def make_bool(fn, *args, **kwargs):
+                        return bool(fn(*args, **kwargs))
+
+                    self.influence.append(CallbackInfluence(f"Upload {name}", make_bool, condition, self))
+
+                cls.__init__ = new_init
+
+            return cls
+
+        return decorate
 
     @staticmethod
     def attribute(alias, target, influence=True, default=False):
@@ -995,6 +1216,79 @@ class attributes:
             return utils.concat_attributes("_publish_files", attrib)(cls)
         return decorate
 
+    @staticmethod
+    def common_metadata(aclocal=True, cmake=True, cxxinfo=True, path=True, pkgconfig=True):
+        """
+        Decorator adding common metadata to published artifacts.
+
+        The decorator adds common environment variables and C/C++ build information
+        to the published artifact:
+
+            - Adds `bin/` to `PATH` environment variable if it exists.
+            - Adds `lib/` and `lib64/` to C++ library paths if they exist.
+            - Adds `include/` to C++ include paths if it exists.
+            - Adds `lib/pkgconfig/`, `lib64/pkgconfig/` and `share/pkgconfig/` to
+              `PKG_CONFIG_PATH` environment variable if `.pc` files are found.
+              The `prefix` variable in `.pc` files is relocated to allow
+              installation in arbitrary locations.
+            - Adds `lib/cmake/`, `lib64/cmake/` and `share/cmake/` to
+              `CMAKE_PREFIX_PATH` environment variable if they exist.
+            - Adds `share/aclocal/` to `ACLOCAL_PATH` environment variable if it exists.
+
+        """
+        def decorate(cls):
+            _old_publish = cls.publish
+
+            @functools.wraps(cls.publish)
+            def publish(self, artifact, tools):
+                _old_publish(self, artifact, tools)
+
+                with tools.cwd(artifact.path):
+                    if path and tools.exists("bin"):
+                        artifact.environ.PATH.append("bin")
+
+                    pcdirs = set()
+
+                    for pcpath in tools.glob("lib/pkgconfig/*.pc") + tools.glob("lib64/pkgconfig/*.pc") + tools.glob("share/pkgconfig/*.pc"):
+                        pcdirs.add(fs.path.dirname(pcpath))
+                        tools.replace_in_file(pcpath, artifact.strings.install_prefix, "${{pcfiledir}}/../..")
+                    for pcpath in tools.glob("lib/*/pkgconfig/*.pc") + tools.glob("lib64/*/pkgconfig/*.pc"):
+                        pcdirs.add(fs.path.dirname(pcpath))
+                        tools.replace_in_file(pcpath, artifact.strings.install_prefix, "${{pcfiledir}}/../../..")
+
+                    for pcdir in pcdirs:
+                        artifact.environ.PKG_CONFIG_PATH.append(pcdir)
+
+                    if cmake and tools.exists("lib/cmake"):
+                        artifact.environ.CMAKE_PREFIX_PATH.append(".")
+                    if cmake and tools.exists("lib64/cmake"):
+                        artifact.environ.CMAKE_PREFIX_PATH.append(".")
+                    if cmake and tools.exists("share/cmake"):
+                        artifact.environ.CMAKE_PREFIX_PATH.append(".")
+
+                    if aclocal and tools.exists("share/aclocal"):
+                        artifact.environ.ACLOCAL_PATH.append("share/aclocal")
+
+            cls.publish = publish
+
+            return cls
+
+        return decorate
+
+    @staticmethod
+    def requires(attrib):
+        """
+        Decorates a task with an alternative ``requires`` attribute.
+
+        The new attribute will be concatenated with the regular
+        ``requires`` attribute.
+
+        Args:
+            attrib (str): Name of alternative attribute.
+                Keywords are expanded.
+        """
+        return utils.concat_attributes("requires", attrib)
+
     @staticmethod
     def load(filepath):
         """
@@ -1066,6 +1360,20 @@ class attributes:
             return cls
         return _decorate
 
+    @staticmethod
+    def platform(attrib):
+        """
+        Decorates a task with an alternative ``platform`` attribute.
+
+        The new attribute will be concatenated with the regular
+        ``platform`` attribute.
+
+        Args:
+            attrib (str): Name of alternative attribute.
+                Keywords are expanded.
+        """
+        return utils.concat_attributes("platform", attrib)
+
     @staticmethod
     def system(cls):
         """
@@ -1077,6 +1385,40 @@ class attributes:
         cls.system = property(lambda t: t._system.value)
         return cls
 
+    @staticmethod
+    def timeout(seconds):
+        """
+        Decorator setting a timeout for a task.
+
+        The timeout applies to the task's run method. A JoltTimeoutError
+        is raised if the task does not complete within the specified time.
+
+        Args:
+            seconds (int): Timeout in seconds.
+
+        Example:
+
+        .. code-block:: python
+
+            @attributes.timeout(5)
+            class Example(Task):
+                def run(self, deps, tools):
+                    time.sleep(10)
+
+        """
+        def decorate(cls):
+            _old_run = cls.run
+
+            @functools.wraps(cls.run)
+            def run(self, deps, tools):
+                with tools.timeout(seconds):
+                    _old_run(self, deps, tools)
+
+            cls.run = run
+            return cls
+
+        return decorate
+
 
 class TaskBase(object):
     """ Task base class. """
@@ -1160,9 +1502,37 @@ class TaskBase(object):
     joltproject = None
     """ Name of project this task belongs to. """
 
+    local = False
+    """ A local task is only executed on the local node where the build is initiated. """
+
     name = None
     """ Name of the task. Derived from class name if not set. """
 
+    platform = {}
+    """
+    Dictionary of task platform requirements.
+
+    Platform requirements control where tasks are allowed to execute.
+    Multiple requirement key/values may be specified in which case all
+    must be fulfilled in order for the task to be schedulable on a node.
+
+    The following builtin requirement labels exist:
+
+      - node.arch ["arm", "amd64"]
+      - node.os ["linux", "windows"]
+
+    Example:
+
+      .. code-block:: python
+
+        class Hello(Task):
+            # This task must run on Linux.
+            platform = {
+                "node.os": "linux",
+            }
+
+    """
+
     requires = []
     """ List of dependencies to other tasks. """
 
@@ -1209,7 +1579,15 @@ class TaskBase(object):
     execution requests, the instance ID won't be.
     """
 
-    def __init__(self, parameters=None, manifest=None, **kwargs):
+    @property
+    def instance(self):
+        return str(self._instance.value)
+
+    @instance.setter
+    def instance(self, value):
+        self._instance.assign(value)
+
+    def __init__(self, parameters=None, manifest=None, buildenv=None, **kwargs):
         self._identity = None
         self._report = _JoltTask()
         self.name = self.__class__.name
@@ -1230,6 +1608,7 @@ class TaskBase(object):
         self.selfsustained = utils.call_or_return(self, self.__class__._selfsustained)
         self.tools = Tools(self, self.joltdir)
         self._apply_manifest(manifest)
+        self._apply_protobuf(buildenv)
         self.requires = self.expand(self.requires)
 
     def _apply_manifest(self, manifest):
@@ -1246,6 +1625,26 @@ class TaskBase(object):
                     .format(attrib.name, self.qualified_name)
                 export.assign(attrib.value)
 
+    def _apply_protobuf(self, buildenv):
+        if buildenv is None:
+            return
+        task = buildenv.tasks.get(self.exported_name)
+        if not task:
+            return
+        if task.identity:
+            self.identity = task.identity
+        if task.taint:
+            self.taint = task.taint
+        for prop in task.properties:
+            export = utils.getattr_safe(self, prop.key)
+            assert isinstance(export, Export), \
+                "'{0}' is not an exportable attribute of task '{1}'"\
+                .format(prop.key, self.qualified_name)
+            export.assign(prop.value)
+
+    def _artifacts(self, cache, node):
+        return [cache.get_artifact(node, "main")]
+
     def _influence(self):
         return utils.as_list(self.__class__.influence)
 
@@ -1305,6 +1704,9 @@ class TaskBase(object):
                 "Required parameter '{0}' has not been set", key)
 
     def _verify_influence(self, deps, artifact, tools, sources=None):
+        if "configurationrepository" in self.name:
+            breakpoint()
+
         # Verify that any transformed sources are influencing
         sources = set(map(tools.expand_path, sources or []))
 
@@ -1321,29 +1723,29 @@ class TaskBase(object):
                 return not fs.is_relative_to(fname, rootpath)
             return _filter
 
+        # Ignore any files in build directories
+        sources = filter(_subpath_filter(tools.expand_path(tools.buildroot)), sources)
+
         for _, dep in deps.items():
-            deptask = dep.get_task()
+            deptask = dep.task
             if isinstance(deptask, FileInfluence):
                 # Resource dependencies may cover the influence implicitly
                 deppath = self.tools.expand_path(str(deptask.path))
-                sources = set(filter(lambda d: not deptask.is_influenced_by(self, d), sources))
+                sources = filter(lambda d, dt=deptask: not dt.is_influenced_by(self, d), sources)
             else:
                 # Ignore any files in artifacts
                 deppath = self.tools.expand_path(dep.path)
-                sources = set(filter(_subpath_filter(deppath), sources))
-
-        # Ignore any files in build directories
-        sources = filter(_subpath_filter(tools.expand_path(tools.buildroot)), sources)
-        sources = set(sources)
+                sources = filter(_subpath_filter(deppath), sources)
 
         for ip in self.influence:
             if not isinstance(ip, FileInfluence):
                 continue
-            ok = [source for source in sources if ip.is_influenced_by(self, source)]
-            sources.difference_update(ok)
+            sources = {source for source in sources if not ip.is_influenced_by(self, source)}
+
         for source in sources:
             log.warning("Missing influence: {} ({})", source, self.name)
-        raise_task_error_if(sources, self, "Task is missing source influence")
+
+        raise_task_error_if(set(sources), self, "Task is missing source influence")
 
     def _get_export_objects(self):
         return self._exports
@@ -1384,6 +1786,16 @@ class TaskBase(object):
             self.name,
             self._get_explicitly_set_parameters())
 
+    @property
+    def exported_name(self):
+        if hasattr(self, "_exported_name"):
+            return self._exported_name
+        return self.short_qualified_name
+
+    @exported_name.setter
+    def exported_name(self, name):
+        self._exported_name = name
+
     def expand(self, string_or_list, *args, **kwargs):
         """ Expands keyword arguments/macros in a format string.
 
@@ -1422,8 +1834,15 @@ class Task(TaskBase):
     Abstract tasks can't be executed and won't be listed.
     """
 
+    unstable = False
+    """
+    An unstable task is allowed to fail without stopping or failing the entire build.
+
+    The unstable task is still reported as a failure at the end of the build.
+    """
+
     def __init__(self, parameters=None, **kwargs):
-        super(Task, self).__init__(parameters, **kwargs)
+        super().__init__(parameters, **kwargs)
 
     def info(self, fmt, *args, **kwargs):
         """
@@ -1478,6 +1897,9 @@ class Task(TaskBase):
         as ``target`` above,  are automatically expanded to their values.
         """
 
+    def nopublish(self, artifact, tools):
+        raise NotImplementedError()
+
     def publish(self, artifact, tools):
         """
         Publishes files produced by :func:`~run`.
@@ -1512,7 +1934,7 @@ class Task(TaskBase):
         """
         raise NotImplementedError()
 
-    def shell(self, deps, tools):
+    def debugshell(self, deps, tools):
         """
         Invoked to start a debug shell.
 
@@ -1543,7 +1965,7 @@ class SubTask(object):
         self._message = None
         self._outputs = []
         self._task = task
-        self._tools = Tools(task, task.joltdir)
+        self._tools = copy.copy(task.tools)
 
     def __str__(self):
         if self.message:
@@ -1629,15 +2051,21 @@ class SubTask(object):
         self._influence.append(infl)
 
     def add_influence_file(self, path):
+        path = self._tools.expand_path(path)
         self.add_influence(utils.filesha1(path))
 
     def add_influence_depfile(self, path):
         def depfile():
             result = ""
-            deps = self._tools.read_depfile(fs.path.join(self._task.joltdir, path))
-            for output in self.outputs:
-                for input in deps.get(output, []):
-                    result += utils.filesha1(input)
+            try:
+                deps = self._tools.read_depfile(fs.path.join(self._task.joltdir, path))
+            except OSError:
+                return "N/A"
+            with self._tools.cwd(self._task.joltdir):
+                for output in self.outputs:
+                    for input in deps.get(output, []):
+                        input = self._tools.expand_path(input)
+                        result += utils.filesha1(input)
             return result
         self.add_influence(depfile)
 
@@ -1683,7 +2111,7 @@ class CommandSubtask(SubTask):
 
     def __str__(self):
         s = super().__str__()
-        return s if s is not None and not log.is_verbose() else self._command
+        return s if s is not None and not log.is_verbose() else self._tools.expand(self._command)
 
     def run(self):
         self._tools.run(self._command)
@@ -1770,11 +2198,11 @@ class MultiTask(Task):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._subtasks = []
+        self._subtasks = set()
         self._subtasks_by_output = {}
 
     def _add_subtask(self, subtask):
-        self._subtasks.append(subtask)
+        self._subtasks.add(subtask)
 
         for output in subtask.outputs:
             if output in self._subtasks_by_output:
@@ -1799,19 +2227,22 @@ class MultiTask(Task):
             inputsubtask = Input(self, input)
         else:
             inputsubtask = input
-        self._subtasks.append(inputsubtask)
+        self._subtasks.add(inputsubtask)
         self._subtasks_by_output[input] = inputsubtask
         return inputsubtask
 
     def _to_subtask_list(self, inputs):
         inputs = utils.as_list(inputs)
-        return [self._add_input(input) for input in inputs]
+        return utils.unique_list([self._add_input(input) for input in inputs])
 
     def _to_output_files(self, subtasks):
         subtasks = utils.as_list(subtasks)
         outputs = []
         for subtask in subtasks:
-            outputs.extend(subtask.outputs)
+            if isinstance(subtask, SubTask):
+                outputs.extend(subtask.outputs)
+            else:
+                outputs.append(subtask)
         return outputs
 
     def _to_input_subtasks(self, inputs, **kwargs):
@@ -1851,12 +2282,11 @@ class MultiTask(Task):
                   outputs=["executable"])
 
         """
-        inputs = self._to_input_subtasks(inputs, **kwargs)
+        input_jobs = self._to_input_subtasks(inputs, **kwargs)
         inputfiles = self._to_output_files(inputs)
 
         outputs = utils.as_list(outputs)
-        outputs = [self.expand(output, **kwargs) for output in outputs]
-        outputs = [fs.path.relpath(output) for output in outputs]
+        outputs = [self.tools.expand_relpath(output, self.joltdir, **kwargs) for output in outputs]
 
         dirs = set()
         if mkdir:
@@ -1868,7 +2298,7 @@ class MultiTask(Task):
 
         for dir in dirs:
             subtask.add_dependency(dir)
-        for input in inputs:
+        for input in input_jobs:
             subtask.add_dependency(input)
         for output in outputs:
             output = self.expand(output, inputs=inputfiles, outputs=outputs, **kwargs)
@@ -2051,7 +2481,6 @@ class MultiTask(Task):
 
         This method should typically not be overridden in subclasses.
         """
-
         self.generate(deps, tools)
 
         log.debug("About to start executing these subtasks:")
@@ -2070,6 +2499,7 @@ class MultiTask(Task):
         for subtask in self._subtasks:
             if subtask not in subtasks:
                 subtasks[subtask] = []
+            if subtask not in deps:
                 deps[subtask] = []
             for dep in subtask.dependencies:
                 if dep not in deps:
@@ -2079,12 +2509,18 @@ class MultiTask(Task):
 
         # Prune up-to-date subtasks
         for subtask in list(filter(lambda subtask: not subtask.is_outdated, subtasks.keys())):
+            log.debug("Pruning {}", subtask)
             del subtasks[subtask]
             for dep in deps[subtask]:
-                subtasks[dep].remove(subtask)
+                try:
+                    subtasks[dep].remove(subtask)
+                except KeyError:
+                    pass
+
+        self.subtaskindex = 0
+        self.subtaskcount = len(subtasks)
 
-        subtaskindex = 0
-        subtaskcount = len(subtasks)
+        lock = RLock()
 
         with ThreadPoolExecutor(max_workers=tools.cpu_count()) as pool:
             futures = {}
@@ -2097,11 +2533,14 @@ class MultiTask(Task):
                     break
 
                 for subtask in candidates:
-                    subtaskindex += 1
                     del subtasks[subtask]
                     if subtask.is_outdated:
-                        log.info("[{}/{}] {}", subtaskindex, subtaskcount, str(subtask))
-                        futures[pool.submit(subtask.run)] = subtask
+                        def runner(subtask):
+                            with lock:
+                                self.subtaskindex += 1
+                                log.info("[{}/{}] {}", self.subtaskindex, self.subtaskcount, str(subtask))
+                            subtask.run()
+                        futures[pool.submit(functools.partial(runner, subtask))] = subtask
                     else:
                         completed.append(subtask)
 
@@ -2132,6 +2571,13 @@ class MultiTask(Task):
 
             raise_task_error_if(subtasks, self, "Subtasks with unresolved dependencies could not be executed")
 
+    def inputs(self, jobs):
+        return self._to_subtask_list(jobs)
+
+    def outputs(self, jobs):
+        jobs = utils.as_list(jobs)
+        return [output for job in jobs for output in job.outputs]
+
 
 class Runner(Task):
     """
@@ -2212,9 +2658,9 @@ class Runner(Task):
         found = False
 
         for task, artifact in deps.items():
-            if not artifact.get_task().is_cacheable():
+            if not artifact.task.is_cacheable():
                 continue
-            if artifact.strings.executable.get_value() is None:
+            if artifact.strings.executable is None:
                 self.verbose("No executable found in task artifact for '{}'", task)
                 continue
             with tools.cwd(artifact.path):
@@ -2233,6 +2679,22 @@ class ErrorProxy(object):
     def __init__(self, error):
         self._error = error
 
+    @property
+    def type(self):
+        return self._error.type
+
+    @type.setter
+    def type(self, value):
+        self._error.type = value
+
+    @property
+    def details(self):
+        return self._error.details
+
+    @property
+    def location(self):
+        return self._error.location
+
     @property
     def message(self):
         return self._error.message
@@ -2240,11 +2702,21 @@ class ErrorProxy(object):
 
 class ReportProxy(object):
     def __init__(self, task, report):
+        from jolt import config
         self._task = task
         self._report = report
+        self._max_errors = config.getint("jolt", "task_max_errors", 100)
 
     def add_error(self, type, location, message, details=""):
         """ Add an error to the build report. """
+        if len(self.errors) >= self._max_errors:
+            if not hasattr(self._report, "truncated"):
+                error = self._report.create_error()
+                error.type = "Error"
+                error.message = "Too many errors, list truncated"
+                self._report.truncated = True
+            return None
+
         error = self._report.create_error()
         error.type = type
         error.location = location
@@ -2265,11 +2737,8 @@ class ReportProxy(object):
         """
         for match in re.finditer(regex, logbuf, re.MULTILINE):
             error = match.groupdict()
-            self.add_error(
-                type,
-                error.get("location", ""),
-                error.get("message", ""),
-                error.get("details", ""))
+            if not self.add_error(type, error.get("location", ""), error.get("message", ""), error.get("details", "")):
+                break
 
     def add_regex_errors_with_file(self, type, regex, logbuf, reldir, filterfn=lambda n: True):
         """
@@ -2290,20 +2759,29 @@ class ReportProxy(object):
             if not filterfn(error):
                 continue
             if error["location"] not in errors_by_location:
-                errors_by_location[error["location"]] = (error, [error["message"]])
+                errors_by_location[error["location"]] = (error, [error["message"]], error.get("details", ""))
             else:
                 errors_by_location[error["location"]][1].append(error["message"])
 
-        for error, msgs in errors_by_location.values():
+        for error, msgs, details in errors_by_location.values():
             message = "\n".join(utils.unique_list(msgs))
-            with self._task.tools.cwd(reldir):
-                try:
-                    details = self._task.tools.read_file(error["file"])
-                    details = details.splitlines()
-                    details = str(error["line"]) + ": " + details[int(error["line"]) - 1]
-                except Exception:
-                    details = ""
-            self.add_error(type, error.get("location", ""), message, details)
+            if not details:
+                with self._task.tools.cwd(self._task.tools.wsroot):
+                    try:
+                        details = self._task.tools.read_file(error["file"])
+                        details = details.splitlines()
+                        details = str(error["line"]) + ": " + details[int(error["line"]) - 1]
+                    except Exception:
+                        details = ""
+
+            location = error.get("location", "")
+            if location:
+                with self._task.tools.cwd(self._task.tools.wsroot):
+                    location = self._task.tools.expand_path(location)
+                location = self._task.tools.expand_relpath(location, self._task.tools.wsroot)
+
+            if not self.add_error(type, location, message, details):
+                break
 
     def add_exception(self, exc, errtype=None, location=None):
         """
@@ -2317,13 +2795,14 @@ class ReportProxy(object):
 
         """
         tb = traceback.format_exception(type(exc), value=exc, tb=exc.__traceback__)
+
         installdir = fs.path.dirname(__file__)
         if any(map(lambda frame: installdir not in frame, tb[1:-1])):
             while len(tb) > 2 and installdir in tb[1]:
                 del tb[1]
-        loc = re.findall("\"(.*?\", line [0-9]+, in .*?)\n", tb[1])
+        loc = re.findall("(\".*?\", line [0-9]+, in .*?)\n", tb[1])
         location = location or (loc[0] if loc and len(loc) > 0 else "")
-        message = str(exc)
+        message = log.format_exception_msg(exc)
         if isinstance(exc, JoltCommandError):
             details = "\n".join(exc.stderr)
         elif isinstance(exc, JoltError):
@@ -2341,10 +2820,26 @@ class ReportProxy(object):
     def errors(self):
         return [ErrorProxy(error) for error in self._report.errors]
 
+    @errors.setter
+    def errors(self, errlist):
+        assert all(isinstance(err, ErrorProxy) for err in errlist), "Invalid error list"
+        self._report.clear_errors()
+        for err in errlist:
+            self.add_error(err.type, err.location, err.message, err.details)
+
     @property
     def manifest(self):
         return self._report
 
+    def raise_for_status(self, log_details=False, log_error=False):
+        for error in self.errors:
+            if log_error:
+                log.error("{}: {}", error.type, error.message, context=self._task.identity[:7])
+            if log_details:
+                for line in error.details.splitlines():
+                    log.transfer(line, context=self._task.identity[:7])
+            raise LoggedJoltError(JoltError(f"{error.type}: {error.message}"))
+
 
 class Resource(Task):
     """
@@ -2358,13 +2853,17 @@ class Resource(Task):
 
     """
 
-    cacheable = False
-
     abstract = True
     """ An abstract resource class indended to be subclassed. """
 
+    release_on_error = False
+    """ Call release if an exception occurs during acquire. """
+
     def __init__(self, *args, **kwargs):
-        super(Resource, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
+
+    def _artifacts(self, cache, node):
+        return [cache.get_artifact(node, "main", session=True)]
 
     def is_runnable(self):
         return False
@@ -2425,17 +2924,25 @@ class WorkspaceResource(Resource):
     """
 
     def __init__(self, *args, **kwargs):
-        super(WorkspaceResource, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         raise_task_error_if(len(self.requires) > 0, self,
                             "Workspace resource is not allowed to have requirements")
 
-    def acquire(self, **kwargs):
+    def acquire(self, *args, **kwargs):
         return self.acquire_ws()
 
-    def release(self, **kwargs):
+    def release(self, *args, **kwargs):
         return self.release_ws()
 
-    def acquire_ws(self):
+    def prepare_ws_for(self, task):
+        """ Called to prepare the workspace for a task.
+
+        An implementor overrides this method in a subclass. The method
+        is called before the task influence is calculated and the workspace
+        resource is acquired.
+        """
+
+    def acquire_ws(self, force=False):
         """ Called to acquire the resource.
 
         An implementor overrides this method in a subclass. The acquired
@@ -2539,6 +3046,7 @@ class Download(Task):
     Once downloaded, archives are extracted and all of their files are published.
     If the file is not an archive it is published as is. Recognized archive extensions are:
 
+        - .7z
         - .tar
         - .tar.bz2
         - .tar.gz
@@ -2601,7 +3109,7 @@ class Download(Task):
         return fs.posixpath.basename(url.path) or "file"
 
     def run(self, deps, tools):
-        supported_formats = [".tar", ".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".zip"]
+        supported_formats = [".7z", ".tar", ".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".zip"]
 
         raise_task_error_if(not self.url, self, "No URL(s) specified")
 
@@ -2698,7 +3206,8 @@ class Script(Task):
         doc = self.__doc__.split("---", 1)
         script = doc[1] if len(doc) > 1 else doc[0]
         script = script.splitlines()
-        script = [line[4:] for line in script]
+        if os_sys.version_info < (3, 13):
+            script = [line[4:] for line in script]
         script = "\n".join(script)
         script = script.lstrip()
         if not script.startswith("#!"):
@@ -2825,7 +3334,7 @@ class Test(Task):
     Abstract test tasks can't be executed and won't be listed.
     """
 
-    pattern = Parameter(required=False, help="Test-case filter wildcard.")
+    filter = Parameter(required=False, help="Test-case filter wildcard.")
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -2854,24 +3363,21 @@ class Test(Task):
         """
         raise_error_if(type(args) is not list, "Test.parameterized() expects a list as argument")
 
-        class partialmethod(functools.partialmethod):
-            def __init__(self, index, func, *args):
-                super().__init__(func, *args)
-                self.__index = index
-
-            def __get__(self, obj, cls=None):
-                retval = super().__get__(obj, cls)
-                retval.__name__ = f"{self.func.__name__}[{self.__index}]"
-                retval.__doc__ = self.func.__doc__
-                return retval
+        def make_method(index, func, *args):
+            def testmethod(self):
+                return func(self, *args)
+            testmethod.__name__ = f"{func.__name__}[{index}]"
+            testmethod.__doc__ = func.__doc__
+            return testmethod
 
         def decorate(method):
             frame = sys._getframe().f_back.f_locals
             for index, arg in enumerate(args):
-                testmethod = partialmethod(index, method, *utils.as_list(arg))
+                testmethod = make_method(index, method, *utils.as_list(arg))
                 name = f"{method.__name__}[{index}]"
                 frame[name] = testmethod
             return None
+
         return decorate
 
     def setup(self, deps, tools):
@@ -2980,7 +3486,7 @@ class Test(Task):
     def run(self, deps, tools):
         testsuite = ut.TestSuite()
         for test in self._get_test_names():
-            if self.pattern.is_unset() or fnmatch.fnmatch(test, str(self.pattern)):
+            if self.filter.is_unset() or fnmatch.fnmatch(test, str(self.filter)):
                 testfunc = getattr(self, test)
                 if not testfunc:
                     continue
@@ -3001,7 +3507,7 @@ class Test(Task):
 
 
 @ArtifactAttributeSetProvider.Register
-class ResourceAttributeSetProvider(ArtifactAttributeSetProvider):
+class WorkspaceResourceAttributeSetProvider(ArtifactAttributeSetProvider):
     def create(self, artifact):
         pass
 
@@ -3012,33 +3518,31 @@ class ResourceAttributeSetProvider(ArtifactAttributeSetProvider):
         pass
 
     def apply(self, task, artifact):
-        resource = artifact.get_task()
-        if isinstance(resource, Resource):
-            from inspect import signature
+        resource = artifact.task
+        node = artifact.get_node()
+        if not node.is_workspace_resource():
+            return
 
-            deps = resource._run_env
-            deps.__enter__()
-            sig = signature(resource.acquire)
-            try:
-                ba = sig.bind_partial(artifact=artifact, deps=deps, tools=resource.tools, owner=task)
-                acquire = resource.acquire
-            except Exception:
-                ba = sig.bind_partial(artifact, deps, resource.tools)
-                acquire = utils.deprecated(resource.acquire)
-            acquire(*ba.args, **ba.kwargs)
+        resource.deps = node.cache.get_context(node)
+        resource.deps.__enter__()
+
+        try:
+            resource.acquire(artifact=artifact, deps=resource.deps, tools=resource.tools, owner=task)
+        except (KeyboardInterrupt, Exception) as e:
+            if resource.release_on_error:
+                with utils.ignore_exception():
+                    self.unapply(task, artifact)
+            raise e
 
     def unapply(self, task, artifact):
-        resource = artifact.get_task()
-        if isinstance(resource, Resource):
-            from inspect import signature
+        resource = artifact.task
+        node = artifact.get_node()
+        if not node.is_workspace_resource():
+            return
 
-            deps = resource._run_env
-            sig = signature(resource.release)
-            try:
-                ba = sig.bind_partial(artifact=artifact, deps=deps, tools=resource.tools, owner=task)
-                release = resource.release
-            except Exception:
-                ba = sig.bind_partial(artifact, deps, resource.tools)
-                release = utils.deprecated(resource.release)
-            release(*ba.args, **ba.kwargs)
-            deps.__exit__(None, None, None)
+        try:
+            resource.release(artifact=artifact, deps=resource.deps, tools=resource.tools, owner=task)
+        except Exception as e:
+            raise e
+
+        resource.deps.__exit__(None, None, None)