jolt 0.9.76__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/tools.py

@@ -1,27 +1,36 @@
+import py7zr
 import bz2
 import copy
 import getpass
 import gzip
+import json
 import lzma
 import subprocess
 import os
 import platform
 import sys
 import threading
+import time
 if os.name != "nt":
     import termios
 import glob
 import multiprocessing
+import re
 import shutil
 import tarfile
 import zipfile
 import bz2file
 import hashlib
+import zstandard
 from contextlib import contextmanager
 from psutil import NoSuchProcess, Process
-
-from jinja2 import Environment, FileSystemLoader, select_autoescape
+from jinja2 import Environment, FileSystemLoader
+from jinja2.exceptions import TemplateError
 from jinja2.runtime import Context
+from jinja2.utils import missing
+from requests import Session
+from requests.auth import HTTPBasicAuth
+from urllib.parse import urlparse, urlunparse
 
 
 from jolt import cache
@@ -29,11 +38,17 @@ from jolt import filesystem as fs
 from jolt import log
 from jolt import utils
 from jolt import config
-from jolt.error import JoltCommandError
+from jolt.error import JoltCommandError, JoltTimeoutError
 from jolt.error import raise_error_if
 from jolt.error import raise_task_error, raise_task_error_if
 
 
+SUPPORTED_ARCHIVE_TYPES = [".tar", ".tar.bz2", ".tar.gz", ".tgz", ".tar.xz", ".tar.zst", ".zip"]
+
+
+http_session = Session()
+
+
 def stdout_write(line):
     sys.stdout.write(line + "\n")
     sys.stdout.flush()
@@ -44,65 +59,54 @@ def stderr_write(line):
     sys.stderr.flush()
 
 
-def _run(cmd, cwd, env, preexec_fn, *args, **kwargs):
+class Reader(threading.Thread):
+    def __init__(self, parent, stream, output=None, logbuf=None, output_rstrip=True):
+        super(Reader, self).__init__()
+        self.output = output
+        self.output_rstrip = output_rstrip
+        self.parent = parent
+        self.stream = stream
+        self.logbuf = logbuf if logbuf is not None else []
+        self.start()
+
+    def run(self):
+        line = ""
+        try:
+            with log.map_thread(self, self.parent):
+                for line in iter(self.stream.readline, b''):
+                    if self.output_rstrip:
+                        line = line.rstrip()
+                    line = line.decode(errors='ignore')
+                    if self.output:
+                        self.output(line)
+                    self.logbuf.append((self, line))
+        except Exception as e:
+            if self.output:
+                self.output("{0}", str(e))
+                self.output(line)
+            self.logbuf.append((self, line))
+
+
+def _run(cmd, cwd, env, *args, **kwargs):
     output = kwargs.get("output")
     output_on_error = kwargs.get("output_on_error")
     output_rstrip = kwargs.get("output_rstrip", True)
     output_stdio = kwargs.get("output_stdio", False)
+    output_stderr = kwargs.get("output_stderr", True)
+    output_stdout = kwargs.get("output_stdout", True)
+    return_stderr = kwargs.get("return_stderr", False)
     output = output if output is not None else True
     output = False if output_on_error else output
     shell = kwargs.get("shell", True)
-    timeout = kwargs.get("timeout", None)
+    timeout = kwargs.get("timeout", config.getint("jolt", "command_timeout", 0))
+    timeout = timeout if type(timeout) is int and timeout > 0 else None
 
     log.debug("Running: '{0}' (CWD: {1})", cmd, cwd)
 
-    p = subprocess.Popen(
-        cmd,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        shell=shell,
-        cwd=cwd,
-        env=env,
-        preexec_fn=preexec_fn,
-    )
-
-    class Reader(threading.Thread):
-        def __init__(self, parent, stream, output=None, logbuf=None):
-            super(Reader, self).__init__()
-            self.output = output
-            self.parent = parent
-            self.stream = stream
-            self.logbuf = logbuf if logbuf is not None else []
-            self.start()
-
-        def run(self):
-            line = ""
-            try:
-                with log.map_thread(self, self.parent):
-                    for line in iter(self.stream.readline, b''):
-                        if output_rstrip:
-                            line = line.rstrip()
-                        line = line.decode(errors='ignore')
-                        if self.output:
-                            self.output(line)
-                        self.logbuf.append((self, line))
-            except Exception as e:
-                if self.output:
-                    self.output("{0}", str(e))
-                    self.output(line)
-                self.logbuf.append((self, line))
-
-    stdout_func = log.stdout if not output_stdio else stdout_write
-    stderr_func = log.stderr if not output_stdio else stderr_write
-
-    logbuf = []
-    stdout = Reader(
-        threading.current_thread(), p.stdout,
-        output=stdout_func if output else None, logbuf=logbuf)
-    stderr = Reader(
-        threading.current_thread(), p.stderr,
-        output=stderr_func if output else None, logbuf=logbuf)
+    p = None
+    stdout = None
+    stderr = None
+    timedout = False
 
     def terminate(pid):
         try:
@@ -122,23 +126,73 @@ def _run(cmd, cwd, env, preexec_fn, *args, **kwargs):
         except NoSuchProcess:
             pass
 
-    timedout = False
     try:
+        with utils.delayed_interrupt():
+            p = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                shell=shell,
+                cwd=cwd,
+                env=env,
+            )
+
+            if output_stdout:
+                stdout_func = log.stdout if not output_stdio else stdout_write
+            else:
+                stdout_func = None
+
+            if output_stderr:
+                stderr_func = log.stderr if not output_stdio else stderr_write
+            else:
+                stderr_func = None
+
+            logbuf = []
+            stdout = Reader(
+                threading.current_thread(),
+                p.stdout,
+                output=stdout_func if output else None,
+                logbuf=logbuf,
+                output_rstrip=output_rstrip)
+            stderr = Reader(
+                threading.current_thread(),
+                p.stderr,
+                output=stderr_func if output else None,
+                logbuf=logbuf,
+                output_rstrip=output_rstrip)
+
         p.wait(timeout=timeout)
-    except subprocess.TimeoutExpired:
+
+    except KeyboardInterrupt:
+        if not p:
+            raise
+        try:
+            terminate(p.pid)
+            p.wait(10)
+        except subprocess.TimeoutExpired:
+            kill(p.pid)
+            utils.call_and_catch(p.wait, 10)
+        raise
+
+    except (subprocess.TimeoutExpired, JoltTimeoutError):
         timedout = True
         try:
             terminate(p.pid)
             p.wait(10)
         except subprocess.TimeoutExpired:
             kill(p.pid)
-            p.wait()
+            utils.call_and_catch(p.wait, 10)
+
     finally:
-        stdout.join()
-        stderr.join()
-        p.stdin.close()
-        p.stdout.close()
-        p.stderr.close()
+        if stdout:
+            stdout.join()
+        if stderr:
+            stderr.join()
+        if p:
+            p.stdin.close()
+            p.stdout.close()
+            p.stderr.close()
 
     if p.returncode != 0 and output_on_error:
         for reader, line in logbuf:
@@ -157,11 +211,16 @@ def _run(cmd, cwd, env, preexec_fn, *args, **kwargs):
 
     if p.returncode != 0:
         stderrbuf = [line for reader, line in logbuf if reader is stderr]
-        raise JoltCommandError(
-            "Command {0}: {1}".format(
-                "timeout" if timedout else "failed",
-                " ".join(cmd) if type(cmd) == list else cmd.format(*args, **kwargs)),
-            stdoutbuf, stderrbuf, p.returncode)
+        if timedout:
+            raise JoltTimeoutError(
+                "Command timeout: " + (" ".join(cmd) if type(cmd) is list else cmd))
+        else:
+            raise JoltCommandError(
+                "Command failed: " + (" ".join(cmd) if type(cmd) is list else cmd),
+                stdoutbuf, stderrbuf, p.returncode)
+    if return_stderr:
+        return "\n".join(stdoutbuf) if output_rstrip else "".join(stdoutbuf), \
+            "\n".join(stderrbuf) if output_rstrip else "".join(stderrbuf)
     return "\n".join(stdoutbuf) if output_rstrip else "".join(stdoutbuf)
 
 
@@ -195,36 +254,6 @@ class _String(object):
         return self._str.startswith(substr)
 
 
-class _tmpdir(object):
-    def __init__(self, name, cwd=None):
-        self._name = name
-        self._path = None
-        self._cwd = cwd or os.getcwd()
-
-    def __enter__(self):
-        try:
-            dirname = self._cwd
-            fs.makedirs(fs.path.join(dirname, fs.path.dirname(self._name)))
-            self._path = fs.mkdtemp(prefix=self._name + "-", dir=dirname)
-        except KeyboardInterrupt as e:
-            raise e
-        except Exception as e:
-            raise e
-        raise_error_if(not self._path, "failed to create temporary directory")
-        return self
-
-    def __exit__(self, type, value, tb):
-        if self._path:
-            fs.rmtree(self._path, ignore_errors=True)
-
-    @property
-    def path(self):
-        return self.get_path()
-
-    def get_path(self):
-        return self._path
-
-
 class _CMake(object):
     def __init__(self, deps, tools, incremental=False):
         self.deps = deps
@@ -232,6 +261,10 @@ class _CMake(object):
         self.builddir = self.tools.builddir(incremental=incremental)
         self.installdir = self.tools.builddir("install", incremental=False)
 
+    def clean(self):
+        self.tools.rmtree(self.builddir, ignore_errors=True)
+        self.tools.rmtree(self.installdir, ignore_errors=True)
+
     def configure(self, sourcedir, *args, generator=None, **kwargs):
         sourcedir = self.tools.expand_path(sourcedir)
 
@@ -242,81 +275,85 @@ class _CMake(object):
 
         with self.tools.cwd(self.builddir):
             self.tools.run(
-                "cmake {0} -B {1} -DCMAKE_INSTALL_PREFIX={2} {3} {4}",
+                "cmake {0} {1} -DCMAKE_INSTALL_PREFIX=/jolt-prefix {1} {2} {3}",
                 sourcedir,
-                self.builddir,
-                self.installdir,
+                utils.option("-B", self.builddir),
                 utils.option("-G", generator),
                 extra_args,
                 output=True)
 
-    def build(self, release=True, *args, **kwargs):
-        threading_args = ''
-        try:
-            threading_args = ' -j {}'.format(kwargs.get("threads", self.tools.thread_count())) \
-                if "--parallel" in self.tools.run("cmake --help-manual cmake 2>&1", output=False) \
-                   else ''
-        except Exception:
-            pass
-
+    def build(self, *args, config="Release", **kwargs):
+        threading_args = ' -j {}'.format(kwargs.get("threads", self.tools.thread_count()))
         with self.tools.cwd(self.builddir):
-            release = "--config Release" if release else ""
-            self.tools.run("cmake --build . {0}{1}", release, threading_args, output=True)
+            self.tools.run("cmake --build . --config {0} {1}", config, threading_args, output=True)
 
-    def install(self, release=True, *args, **kwargs):
-        with self.tools.cwd(self.builddir):
-            release = "--config Release" if release else ""
-            self.tools.run("cmake --build . --target install {0}", release, output=True)
+    def install(self, target="install", config="Release", **kwargs):
+        with self.tools.cwd(self.builddir), self.tools.environ(DESTDIR=self.installdir):
+            self.tools.run("cmake --build . --config {0} --target {1}", config, target, output=True)
 
-    def publish(self, artifact, files='*', *args, **kwargs):
-        with self.tools.cwd(self.installdir):
-            artifact.collect(files, *args, **kwargs)
+    def publish(self, artifact, files='*', symlinks=True, *args, **kwargs):
+        with self.tools.cwd(self.installdir, "jolt-prefix"):
+            artifact.collect(files, *args, symlinks=symlinks, **kwargs)
+        artifact.strings.install_prefix = "/jolt-prefix"
 
 
 class _Meson(object):
-    def __init__(self, deps, tools):
+    def __init__(self, deps, tools, incremental=False):
         self.deps = deps
         self.tools = tools
-        self.builddir = self.tools.builddir()
-        self.installdir = self.tools.builddir("install")
+        self.builddir = self.tools.builddir(incremental=incremental)
+        self.installdir = self.tools.builddir("install", incremental=False)
+        self.prefix = "/jolt-prefix" if os.name != "nt" else "C:\\jolt-prefix"
+
+    def clean(self):
+        self.tools.rmtree(self.builddir, ignore_errors=True)
+        self.tools.rmtree(self.installdir, ignore_errors=True)
 
     def configure(self, sourcedir, *args, **kwargs):
         sourcedir = self.tools.expand_path(sourcedir)
-        self.tools.run("meson --prefix=/ {0} {1}", sourcedir, self.builddir,
+        options = " ".join([f"-D{arg}" for arg in args]) + " "
+        options += " ".join(["-D{0}={1}".format(key, self.tools.expand(val)) for key, val in kwargs.items()])
+        self.tools.run("meson setup --prefix={0} {1} {2} {3}", self.prefix, sourcedir, self.builddir, options,
                        output=True)
 
     def build(self, *args, **kwargs):
         self.tools.run("ninja -C {0} ", self.builddir, output=True)
 
     def install(self, *args, **kwargs):
-        self.tools.run("DESTDIR={0} ninja -C {1} install",
-                       self.installdir, self.builddir,
-                       output=True)
+        with self.tools.environ(DESTDIR=self.installdir):
+            self.tools.run("ninja -C {0} install", self.builddir, output=True)
 
-    def publish(self, artifact, files='*', *args, **kwargs):
-        with self.tools.cwd(self.installdir):
-            artifact.collect(files, *args, **kwargs)
+    def publish(self, artifact, files='*', symlinks=True, *args, **kwargs):
+        with self.tools.cwd(self.installdir, "jolt-prefix"):
+            artifact.collect(files, *args, symlinks=symlinks, **kwargs)
+        artifact.strings.install_prefix = self.prefix
 
 
 class _AutoTools(object):
-    def __init__(self, deps, tools):
+    def __init__(self, deps, tools, incremental=False):
         self.deps = deps
         self.tools = tools
-        self.builddir = self.tools.builddir()
-        self.installdir = self.tools.builddir("install")
+        self.builddir = self.tools.builddir(incremental=incremental)
+        self.installdir = self.tools.builddir("install", incremental=False)
+        self.prefix = "jolt-prefix"
 
-    def configure(self, sourcedir, *args, **kwargs):
+    def clean(self):
+        self.tools.rmtree(self.builddir, ignore_errors=True)
+        self.tools.rmtree(self.installdir, ignore_errors=True)
+
+    def configure(self, sourcedir, *args):
         sourcedir = self.tools.expand_path(sourcedir)
-        prefix = kwargs.get("prefix", "/")
 
         if not fs.path.exists(fs.path.join(sourcedir, "configure")):
             with self.tools.cwd(sourcedir):
                 self.tools.run("autoreconf -visf", output=True)
 
         with self.tools.cwd(self.builddir), self.tools.environ(DESTDIR=self.installdir):
-            self.tools.run("{0}/configure --prefix={1} {2}",
-                           sourcedir, prefix,
-                           self.tools.getenv("CONFIGURE_FLAGS"),
+            self.tools.run("{0}/configure --prefix=/{1} {2} {3}",
+                           sourcedir,
+                           self.prefix,
+                           self.tools.getenv("CONFIGURE_FLAGS", ""),
+                           " ".join(args),
                            output=True)
 
     def build(self, *args, **kwargs):
@@ -324,13 +361,14 @@ class _AutoTools(object):
             self.tools.run("make VERBOSE=yes Q= V=1 -j{0}",
                            self.tools.cpu_count(), output=True)
 
-    def install(self, target="install", **kwargs):
-        with self.tools.cwd(self.builddir), self.tools.environ(DESTDIR=self.installdir):
-            self.tools.run("make {}", target, output=True)
+    def install(self, target="install"):
+        with self.tools.cwd(self.builddir):
+            self.tools.run("make DESTDIR={} {}", self.installdir, target, output=True)
 
-    def publish(self, artifact, files='*', *args, **kwargs):
-        with self.tools.cwd(self.installdir):
-            artifact.collect(files, *args, **kwargs)
+    def publish(self, artifact, files='*', symlinks=True, *args, **kwargs):
+        with self.tools.cwd(self.installdir, self.prefix):
+            artifact.collect(files, *args, symlinks=symlinks, **kwargs)
+        artifact.strings.install_prefix = "/" + self.prefix
 
 
 class ZipFile(zipfile.ZipFile):
@@ -375,11 +413,18 @@ class JinjaTaskContext(Context):
     Attempts to resolves any missing keywords by looking up task class attributes.
     """
     def resolve_or_missing(self, key):
+        if key in self.vars:
+            return self.vars[key]
+
+        if key in self.parent:
+            return self.parent[key]
+
         if key != "task":
             task = self.get("task")
             if task and hasattr(task, key):
                 return getattr(task, key)
-        return super(JinjaTaskContext, self).resolve_or_missing(key)
+
+        return missing
 
 
 class Namespace(object):
@@ -450,8 +495,10 @@ class Tools(object):
 
     def __init__(self, task=None, cwd=None, env=None):
         self._chroot = None
+        self._chroot_prefix = []
+        self._chroot_path = []
+        self._deadline = None
         self._run_prefix = []
-        self._preexec_fn = None
         self._cwd = fs.path.normpath(fs.path.join(config.get_workdir(), cwd or config.get_workdir()))
         self._env = copy.deepcopy(env or os.environ)
         self._task = task
@@ -500,24 +547,50 @@ class Tools(object):
                         zf.write(path, zippath)
         return filename
 
+    def _make_7zfile(self, filename, fmt, rootdir):
+        self.mkdirname(filename)
+        with py7zr.SevenZipFile(filename, 'w') as archive:
+            archive.writeall(rootdir, ".")
+        return filename
+
     def _make_tarfile(self, filename, fmt, rootdir):
-        dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            fs.makedirs(dirname)
+        self.mkdirname(filename)
         with tarfile.open(filename, 'w|%s' % fmt) as tar:
             tar.add(rootdir, ".")
         return filename
 
+    def _make_tarzstd(self, filename, rootdir):
+        self.mkdirname(filename)
+        with open(filename, 'wb') as zstd_file:
+            compressor = zstandard.ZstdCompressor(threads=self.thread_count())
+            with compressor.stream_writer(zstd_file) as stream:
+                with tarfile.open(mode="w|", fileobj=stream) as tar:
+                    tar.add(rootdir, ".")
+        return filename
+
+    def _extract_tarzstd(self, filename, pathname, files=None):
+        with open(filename, 'rb') as zstd_file:
+            decompressor = zstandard.ZstdDecompressor()
+            with decompressor.stream_reader(zstd_file) as stream:
+                with tarfile.open(mode="r|", fileobj=stream) as tar:
+                    if files:
+                        for file in files:
+                            tar.extract(file, pathname)
+                    else:
+                        tar.extractall(pathname)
+
     def archive(self, pathname, filename):
         """ Creates a (compressed) archive.
 
         The type of archive to create is determined by the filename extension.
         Supported formats are:
 
+        - 7z
         - tar
         - tar.bz2
         - tar.gz
         - tar.xz
+        - tar.zst
         - zip
 
         Args:
@@ -533,12 +606,14 @@ class Tools(object):
         elif filename.endswith(".tar"):
             fmt = "tar"
         elif filename.endswith(".tar.gz"):
-            if shutil.which("tar") and shutil.which("pigz"):
+            if self.which("tar") and self.which("pigz"):
                 self.run("tar -I pigz -cf {} -C {} .", filename, pathname)
                 return filename
             fmt = "targz"
+        elif filename.endswith(".tar.zst"):
+            return self._make_tarzstd(filename, rootdir=pathname)
         elif filename.endswith(".tgz"):
-            if shutil.which("tar") and shutil.which("pigz"):
+            if self.which("tar") and self.which("pigz"):
                 self.run("tar -I pigz -cf {} -C {} .", filename, pathname)
                 return filename
             fmt = "targz"
@@ -546,12 +621,16 @@ class Tools(object):
             fmt = "tarbz2"
         elif filename.endswith(".tar.xz"):
             fmt = "tarxz"
+        elif filename.endswith(".7z"):
+            fmt = "7z"
         raise_task_error_if(
             not fmt, self._task,
             "unknown archive type '{0}'", fs.path.basename(filename))
         try:
             if fmt == "zip":
                 outfile = self._make_zipfile(filename, fmt, rootdir=pathname)
+            elif fmt == "7z":
+                outfile = self._make_7zfile(filename, fmt, rootdir=pathname)
             else:
                 outfile = self._make_tarfile(filename, fmt[3:], rootdir=pathname)
             if outfile != filename:
@@ -560,9 +639,9 @@ class Tools(object):
         except Exception:
             raise_task_error(self._task, "failed to create archive from directory '{0}'", pathname)
 
-    def autotools(self, deps=None):
+    def autotools(self, deps=None, incremental=False):
         """ Creates an AutoTools invokation helper """
-        return _AutoTools(deps, self)
+        return _AutoTools(deps, self, incremental=incremental)
 
     @utils.locked(lock='_builddir_lock')
     def builddir(self, name=None, incremental=False, unique=True):
@@ -589,8 +668,16 @@ class Tools(object):
             name += "-" + utils.canonical(self._task.short_qualified_name)
 
         dirname = fs.path.join(self.getcwd(), name)
+        if incremental:
+            dirname += "-inc"
+
+        # Check if incremental build directories are disabled in the configuration
+        if incremental not in ["always"] and not config.is_incremental_build():
+            incremental = False
 
         if incremental:
+            # Create a unique build directory for each task
+            # and store the task name in a hidden file.
             if self._task is not None and unique:
                 meta_task = fs.path.join(dirname, ".task")
                 if not fs.path.exists(meta_task) \
@@ -598,6 +685,7 @@ class Tools(object):
                     fs.rmtree(dirname, ignore_errors=True)
                     fs.makedirs(dirname)
 
+                # Remove the build directory if the task taint has changed (--force or --salt)
                 if self._task.taint is not None:
                     meta = fs.path.join(dirname, ".taint")
                     if not fs.path.exists(meta) or self.read_file(meta) != str(self._task.taint):
@@ -611,18 +699,16 @@ class Tools(object):
             return dirname
 
         if name not in self._builddir:
-            fs.makedirs(fs.path.dirname(dirname))
-            self._builddir[name] = fs.mkdtemp(
-                prefix=fs.path.basename(dirname) + "-",
-                dir=fs.path.dirname(dirname))
+            fs.makedirs(dirname)
+            self._builddir[name] = dirname
 
         return self._builddir[name]
 
     @property
     def buildroot(self):
         """ Return the root path of all build directories """
-        from jolt.loader import get_workspacedir
-        return fs.path.normpath(fs.path.join(get_workspacedir(), "build"))
+        from jolt.loader import JoltLoader
+        return fs.path.normpath(JoltLoader.get().build_path)
 
     def checksum_file(self, filelist, concat=False, hashfn=hashlib.sha1, filterfn=None):
         """ Calculate a checksum of one or multiple files.
@@ -654,7 +740,7 @@ class Tools(object):
             if not concat:
                 checksum = hashfn()
 
-        return result[-1] if concat or type(filelist) == str else result
+        return result[-1] if concat or type(filelist) is str else result
 
     def chmod(self, pathname, mode):
         """ Changes permissions of files and directories.
@@ -679,6 +765,7 @@ class Tools(object):
         - .bz2
         - .gz
         - .xz
+        - .zst
 
         Args:
             src (str): Source file to be compressed.
@@ -695,7 +782,7 @@ class Tools(object):
                     for block in iter(lambda: infp.read(0x10000), b''):
                         outfp.write(block)
         elif ext == "gz":
-            if shutil.which("pigz"):
+            if self.which("pigz"):
                 return self.run("pigz -p {} {}", self.thread_count(), src)
             with open(src, 'rb') as infp:
                 with gzip.open(dst, 'wb') as outfp:
@@ -706,6 +793,13 @@ class Tools(object):
                 with lzma.open(dst, 'wb') as outfp:
                     for block in iter(lambda: infp.read(0x10000), b''):
                         outfp.write(block)
+        elif ext == "zst":
+            with open(src, 'rb') as infp:
+                with open(dst, 'wb') as outfp:
+                    compressor = zstandard.ZstdCompressor(threads=self.thread_count())
+                    with compressor.stream_writer(outfp) as stream:
+                        for block in iter(lambda: infp.read(0x10000), b''):
+                            stream.write(block)
 
     def copy(self, src, dst, symlinks=False):
         """ Copies file and directories (recursively).
@@ -774,10 +868,19 @@ class Tools(object):
         finally:
             self._cwd = prev
 
-    def download(self, url, pathname, exceptions=True, **kwargs):
+    def download(self, url, pathname, exceptions=True, auth=None, **kwargs):
         """
         Downloads a file using HTTP.
 
+        Automatically expands any {keyword} arguments in the URL and pathname.
+
+        Basic authentication is supported by including the credentials in the URL.
+        Environment variables can be used to hide sensitive information. Specify
+        the environment variable name in the URI as e.g.
+        ``http://{environ[USER]}:{environ[PASS]}@host``.
+        Alternatively, the auth parameter can be used to provide an authentication
+        object that is passed to the requests.get() function.
+
         Throws a JoltError exception on failure.
 
         Args:
@@ -790,17 +893,32 @@ class Tools(object):
 
         url = self.expand(url)
         pathname = self.expand_path(pathname)
-        try:
-            from requests.api import get
 
-            response = get(url, stream=True, **kwargs)
+        url_parsed = urlparse(url)
+        raise_task_error_if(
+            not url_parsed.scheme or not url_parsed.netloc,
+            self._task,
+            "Invalid URL: '{}'", url)
+
+        if auth is None and url_parsed.username and url_parsed.password:
+            auth = HTTPBasicAuth(url_parsed.username, url_parsed.password)
+
+        # Redact password from URL if present
+        if url_parsed.password:
+            url_parsed = url_parsed._replace(netloc=url_parsed.netloc.replace(url_parsed.password, "****"))
+
+        url_cleaned = urlunparse(url_parsed)
+
+        try:
+            response = http_session.get(url, stream=True, auth=auth, **kwargs)
             raise_error_if(
                 exceptions and response.status_code not in [200],
-                f"Download from '{url}' failed with status '{response.status_code}'")
+                f"Download from '{url_cleaned}' failed with status '{response.status_code}'")
 
             name = fs.path.basename(pathname)
             size = int(response.headers.get('content-length', 0))
-            with log.progress("Downloading {0}".format(name), size, "B") as pbar:
+            with log.progress("Downloading {0}".format(utils.shorten(name)), size, "B") as pbar:
+                log.verbose("{} -> {}", url_cleaned, pathname)
                 with open(pathname, 'wb') as out_file:
                     chunk_size = 4096
                     for data in response.iter_content(chunk_size=chunk_size):
@@ -808,7 +926,7 @@ class Tools(object):
                         pbar.update(len(data))
                 actual_size = self.file_size(pathname)
                 raise_error_if(
-                    size != 0 and size != actual_size,
+                    size != 0 and size > actual_size,
                     f"Downloaded file was truncated to {actual_size}/{size} bytes: {name}")
 
             return response.status_code == 200
@@ -837,16 +955,29 @@ class Tools(object):
                 with tools.environ(CC="clang"):
                     tools.run("make all")
         """
-        for key, value in kwargs.items():
-            kwargs[key] = self.expand(value)
-
         restore = {key: value for key, value in self._env.items()}
-        self._env.update(kwargs)
-        yield self._env
+
         for key, value in kwargs.items():
-            if key not in restore:
-                del self._env[key]
-        self._env.update(restore)
+            if value is not None:
+                self._env[key] = self.expand(value)
+            else:
+                self._env.pop(key, None)
+
+        try:
+            yield self._env
+        finally:
+            self._env = restore
+
+    def exists(self, pathname):
+        """ Checks if a file or directory exists.
+
+        Args:
+            pathname (str): Path to file or directory.
+
+        Returns:
+            bool: True if the file or directory exists, False otherwise.
+        """
+        return fs.path.exists(self.expand_path(pathname))
 
     def expand(self, string, *args, **kwargs):
         """ Expands keyword arguments/macros in a format string.
@@ -907,6 +1038,9 @@ class Tools(object):
             str: Expanded string.
         """
 
+        if type(pathname) is list:
+            return [self.expand_path(path) for path in pathname]
+
         path = fs.path.join(self.getcwd(), self.expand(pathname, *args, **kwargs))
         # Ensure to retain any trailing path separator which is used as
         # indicator of directory paths
@@ -936,8 +1070,10 @@ class Tools(object):
             str: Expanded string.
         """
 
+        if not relpath:
+            relpath = self._task.joltdir if self._task else self.getcwd()
         pathname = self.expand(pathname, *args, **kwargs)
-        relpath = self.expand(relpath or self._task.joltdir, *args, **kwargs)
+        relpath = self.expand(relpath, *args, **kwargs)
         pathname = fs.path.join(self.getcwd(), pathname)
         # Ensure to retain any trailing path separator which is used as
         # indicator of directory paths
@@ -949,10 +1085,12 @@ class Tools(object):
 
         Supported formats are:
 
+        - 7z
         - tar
         - tar.bz2
         - tar.gz
         - tar.xz
+        - tar.zst
         - zip
 
         Args:
@@ -979,7 +1117,7 @@ class Tools(object):
                     else:
                         tar.extractall(filepath)
             elif filename.endswith(".tar.gz") or filename.endswith(".tgz"):
-                if shutil.which("tar") and shutil.which("pigz"):
+                if self.which("tar") and self.which("pigz"):
                     self.run("tar -I pigz {} -xf {} -C {} {}",
                              ignore_owner_tar, filename, filepath,
                              " ".join(files) if files else "")
@@ -1006,6 +1144,18 @@ class Tools(object):
                             tar.extract(file, filepath)
                     else:
                         tar.extractall(filepath)
+            elif filename.endswith(".tar.zst"):
+                try:
+                    self._extract_tarzstd(filename, filepath, files)
+                except tarfile.StreamError as e:
+                    raise_task_error(self._task, "failed to extract archive '{0}': {1}", filename, str(e))
+            elif filename.endswith(".7z"):
+                with py7zr.SevenZipFile(filename, 'r') as archive:
+                    if files:
+                        for file in files:
+                            archive.extract(file, filepath)
+                    else:
+                        archive.extractall(filepath)
             else:
                 raise_task_error(self._task, "unknown archive type '{0}'", fs.path.basename(filename))
         except Exception:
@@ -1035,7 +1185,7 @@ class Tools(object):
         """ Returns the current working directory. """
         return fs.path.normpath(self._cwd)
 
-    def getenv(self, key, default=""):
+    def getenv(self, key, default=None):
         """ Returns the value of an environment variable.
 
         Only child processes spawned by the same tools object can see
@@ -1072,7 +1222,19 @@ class Tools(object):
             files = [self.expand_path(file) for file in files]
         elif not fs.path.isabs(pathname):
             files = [self.expand_relpath(file, self.getcwd()) for file in files]
-        return files
+        return list(sorted(files))
+
+    def isdir(self, pathname):
+        """ Determines if a path is a directory.
+
+        Args:
+            pathname (str): Path to a file or directory.
+
+        Returns:
+            boolean: True if the path is a directory, False otherwise.
+        """
+        pathname = self.expand_path(pathname)
+        return fs.path.isdir(pathname)
 
     def mkdir(self, pathname, recursively=True):
         """ Create directory. """
@@ -1083,6 +1245,33 @@ class Tools(object):
         else:
             fs.mkdir(pathname)
 
+    def mkdirname(self, pathname, recursively=True):
+        """ Create parent directory. """
+
+        pathname = self.expand_path(pathname)
+        pathname = fs.path.dirname(pathname)
+        if pathname:
+            self.mkdir(pathname, recursively)
+
+    def move(self, src, dst):
+        """
+        Move/rename file.
+
+        Args:
+            src (str): Path to a file or directory to be moved.
+            dest (str): Destination path. If the destination is
+                an existing directory, then src is moved inside
+                that directory. If the destination already exists
+                but is not a directory, it may be overwritten.
+                If the destination is not on the same filesystem, the
+                source file or directory is copied to the destination
+                and then removed.
+        """
+
+        src = self.expand_path(src)
+        dst = self.expand_path(dst)
+        return shutil.move(src, dst)
+
     def map_consecutive(self, callable, iterable):
         """ Same as ``map()``. """
         return utils.map_consecutive(callable, iterable)
@@ -1118,9 +1307,80 @@ class Tools(object):
         """
         return utils.map_concurrent(callable, iterable, max_workers)
 
-    def meson(self, deps=None):
+    def meson(self, deps=None, incremental=False):
         """ Creates a Meson invokation helper """
-        return _Meson(deps, self)
+        return _Meson(deps, self, incremental=incremental)
+
+    @contextmanager
+    def nixpkgs(self, nixfile=None, packages=None, pure=False, path=None, options=None):
+        """
+        Creates a Nix environment with the specified packages.
+
+        Args:
+            nixfile (str): Path to a Nix expression file.
+            packages (list): List of Nix packages to include in environment.
+            pure (boolean): Create a pure environment.
+            path (list): List of Nix expression paths.
+            options (dict): Nix configuration options.
+
+        Example:
+
+            .. code-block:: python
+
+                def run(self, deps, tools):
+                    with tools.nixpkgs(packages=["gcc13"]):
+                        tools.run("gcc --version")
+
+        """
+
+        # Check if Nix is available
+        raise_task_error_if(
+            not self.which("nix-shell"),
+            self._task,
+            "Nix not available on this system")
+
+        nixfile = self.expand_path(nixfile) if nixfile else ""
+        pathflags = " ".join([f"-I {path}" for path in path or []])
+        options = " ".join([f"--option {k} {v}" for k, v in (options or {}).items()])
+        pureflag = "--pure" if pure else ""
+        packages = "-p " + " ".join(packages) if packages else ""
+
+        # Expand all placeholders
+        options = self.expand(options)
+        packages = self.expand(packages)
+        pathflags = self.expand(pathflags)
+
+        # Use cached-nix-shell is available
+        nixshell = "cached-nix-shell" if self.which("cached-nix-shell") else "nix-shell"
+
+        # Run nix-shell to stage packages and environment
+        env = self.run(
+            "{} {} {} {} --run 'env -0' {}",
+            nixshell,
+            pathflags,
+            pureflag,
+            packages,
+            nixfile,
+            output_on_error=True)
+        env = env.strip().strip("\x00")
+        env = dict(line.split("=", 1) for line in env.split('\x00'))
+
+        # Add host path first to environment PATH
+        host_path = env.get("HOST_PATH", None)
+        if host_path:
+            env["PATH"] = host_path + os.pathsep + env["PATH"]
+
+        # Enter the environment
+        old_env = self._env
+        try:
+            if pure:
+                self._env = env
+            else:
+                self._env = copy.deepcopy(env)
+                self._env.update(env)
+            yield
+        finally:
+            self._env = old_env
 
     def render(self, template, **kwargs):
         """ Render a Jinja template string.
@@ -1134,14 +1394,20 @@ class Tools(object):
             str: Renderered template data.
 
         """
-        env = Environment(
-            loader=FileSystemLoader(self.getcwd()),
-            autoescape=select_autoescape(),
-            trim_blocks=True,
-            lstrip_blocks=True)
-        env.context_class = JinjaTaskContext
-        tmpl = env.from_string(template)
-        return tmpl.render(task=self._task, tools=self, **kwargs)
+        try:
+            env = Environment(
+                loader=FileSystemLoader(self.getcwd()),
+                autoescape=False,
+                trim_blocks=True,
+                lstrip_blocks=True)
+            env.context_class = JinjaTaskContext
+            env.filters["prefix"] = utils.prefix
+            env.filters["suffix"] = utils.suffix
+            tmpl = env.from_string(template)
+            return tmpl.render(task=self._task, tools=self, **kwargs)
+        except TemplateError as e:
+            log.debug("Template error: {}", template)
+            raise_task_error(self._task, "Template error: {}", e)
 
     def render_file(self, template, **kwargs):
         """ Render a Jinja template file.
@@ -1155,22 +1421,29 @@ class Tools(object):
             str: Renderered template data.
 
         """
-        env = Environment(
-            loader=FileSystemLoader(self.getcwd()),
-            autoescape=select_autoescape(),
-            trim_blocks=True,
-            lstrip_blocks=True)
-        env.context_class = JinjaTaskContext
-        tmpl = env.get_template(self.expand(template))
-        return tmpl.render(task=self._task, tools=self, **kwargs)
-
-    def replace_in_file(self, pathname, search, replace):
+        try:
+            env = Environment(
+                loader=FileSystemLoader(self.getcwd()),
+                autoescape=False,
+                trim_blocks=True,
+                lstrip_blocks=True)
+            env.context_class = JinjaTaskContext
+            tmpl = env.get_template(self.expand(template))
+            return tmpl.render(task=self._task, tools=self, **kwargs)
+        except TemplateError as e:
+            log.debug("Template error: {}", template)
+            raise_task_error(self._task, "Template error: {}", e)
+
+    def replace_in_file(self, pathname, search, replace, regex=False):
         """ Replaces all occurrences of a substring in a file.
 
         Args:
             pathname (str): Name/path of file to modify.
             search (str): Substring to be replaced.
             replace (str): Replacement substring.
+            regex (boolean): Interpret search parameter as
+                a regular expression matching the string to
+                be replaced.
 
         Example:
 
@@ -1187,13 +1460,16 @@ class Tools(object):
         try:
             with open(pathname, "rb") as f:
                 data = f.read()
-            data = data.replace(search.encode(), replace.encode())
+            if regex:
+                data = re.sub(search.encode(), replace.encode(), data)
+            else:
+                data = data.replace(search.encode(), replace.encode())
             with open(pathname, "wb") as f:
                 f.write(data)
         except KeyboardInterrupt as e:
             raise e
         except Exception:
-            raise_task_error(self._task, "failed to replace string in file '{0}'", pathname)
+            raise_task_error(self._task, "Failed to replace string in file '{0}'", pathname)
 
     def rmtree(self, pathname, *args, **kwargs):
         """Removes a directory tree from disk.
@@ -1309,6 +1585,9 @@ class Tools(object):
                 refuses to terminate, it will be killed after an additional
                 10 seconds have passed. Default: None.
 
+        Returns:
+           str: stdout from command unless output=False
+
         Example:
 
             .. code-block:: python
@@ -1320,7 +1599,22 @@ class Tools(object):
                     tools.run("make {target} VERBOSE={verbose} JOBS={0}", tools.cpu_count())
 
         """
+        kwargs.setdefault("shell", True)
+
+        # Append command prefix before expanding string
+        if self._chroot_prefix or self._run_prefix:
+            if type(cmd) is list:
+                cmd = self._chroot_prefix + self._run_prefix + cmd
+            else:
+                cmd = " ".join(self._chroot_prefix + self._run_prefix) + " " + cmd
+
+        if self._deadline is not None:
+            remaining = int(self._deadline - time.time() + 0.5)
+            timeout = kwargs.get("timeout", remaining)
+            kwargs["timeout"] = min(remaining, timeout)
+
         cmd = self.expand(cmd, *args, **kwargs)
+
         stdi, stdo, stde = None, None, None
         try:
             stdi, stdo, stde = None, None, None
@@ -1332,12 +1626,9 @@ class Tools(object):
                 raise e
             except Exception:
                 pass
-            if self._run_prefix:
-                if type(cmd) == list:
-                    cmd = self._run_prefix + cmd
-                else:
-                    cmd = " ".join(self._run_prefix) + " " + cmd
-            return _run(cmd, self._cwd, self._env, self._preexec_fn, *args, **kwargs)
+
+            return _run(cmd, self._cwd, self._env, *args, **kwargs)
+
         finally:
             if stdi:
                 termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, stdi)
@@ -1377,7 +1668,7 @@ class Tools(object):
 
         """
         cmdprefix = self.expand(cmdprefix, *args, **kwargs)
-        if type(cmdprefix) == str:
+        if type(cmdprefix) is str:
             cmdprefix = cmdprefix.split()
 
         old_prefix = copy.copy(self._run_prefix)
@@ -1416,10 +1707,10 @@ class Tools(object):
         """
 
         raise_error_if(
-            type(artifact) is not cache.Artifact,
+            type(artifact) not in [cache.Artifact, cache.ArtifactToolsProxy],
             "non-artifact passed as argument to Tools.sandbox()")
 
-        suffix = utils.canonical(artifact.get_task().short_qualified_name)
+        suffix = utils.canonical(artifact.task.short_qualified_name)
 
         if reflect:
             sandbox_name = "sandboxes-reflected/" + suffix
@@ -1449,8 +1740,16 @@ class Tools(object):
             fs.rmtree(path)
             fs.makedirs(path)
             for relsrcpath, reldstpath in artifact.files.items():
-                srcpath = fs.path.normpath(fs.path.join(artifact.get_task().joltdir, relsrcpath))
+                srcpath = fs.path.normpath(fs.path.join(artifact.task.joltdir, relsrcpath))
+                srcpath = self.expand_path(srcpath)
                 dstpath = fs.path.normpath(fs.path.join(path, reldstpath))
+                dstpath = self.expand_path(dstpath)
+
+                if dstpath != fs.path.realpath(dstpath):
+                    log.debug("Cannot symlink '{} -> {}', parent directory already symlinked",
+                              srcpath, dstpath)
+                    continue
+
                 if fs.path.isdir(dstpath):
                     files = fs.scandir(srcpath)
                     for file in files:
@@ -1460,7 +1759,8 @@ class Tools(object):
                     self.symlink(srcpath, dstpath)
 
                 # Restore missing srcfiles if they resided in a build directory
-                if srcpath.startswith(artifact.get_task().tools.buildroot) and \
+                buildroot_abs = self.expand_path(artifact.tools.buildroot)
+                if srcpath.startswith(buildroot_abs) and \
                    not fs.path.exists(srcpath):
                     fs.copy(fs.path.join(artifact.path, reldstpath), srcpath, symlinks=True)
             self.write_file(meta, artifact.path)
@@ -1507,7 +1807,41 @@ class Tools(object):
             fs.makedirs(dstdir)
         fs.symlink(src, dst)
 
-    def tmpdir(self, name):
+    @contextmanager
+    def timeout(self, seconds):
+        """ Context manager to set a timeout for a block of code.
+
+        A TimeoutError exception is raised if the block of code does not
+        complete within the specified time.
+
+        Args:
+            seconds (int): Timeout in seconds.
+
+        Example:
+
+                .. code-block:: python
+
+                    with tools.timeout(5):
+                        tools.run("sleep 10")
+
+        """
+        if seconds is None:
+            yield
+            return
+
+        with utils.timeout(seconds, JoltTimeoutError):
+            old_deadline = self._deadline
+            try:
+                if old_deadline is None:
+                    self._deadline = time.time() + seconds
+                else:
+                    self._deadline = min(old_deadline, time.time() + seconds)
+                yield
+            finally:
+                self._deadline = old_deadline
+
+    @contextmanager
+    def tmpdir(self, name=None):
         """ Creates a temporary directory.
 
         The directory is only valid within a context and it is removed
@@ -1522,11 +1856,18 @@ class Tools(object):
 
             .. code-block:: python
 
-                with tools.tmpdir("temp") as tmp, tools.cwd(tmp.path):
+                with tools.tmpdir() as tmp, tools.cwd(tmp):
                     tools.write_file("tempfile", "tempdata")
 
         """
-        return _tmpdir(name, cwd=self._cwd)
+        dirname = None
+        try:
+            self.mkdir(self.buildroot)
+            dirname = fs.mkdtemp(prefix=(name or "tmpdir") + "-", dir=self.buildroot)
+            yield fs.path.normpath(dirname)
+        finally:
+            if dirname:
+                self.rmtree(dirname, ignore_errors=True)
 
     def unlink(self, pathname, *args, **kwargs):
         """Removes a file from disk.
@@ -1538,10 +1879,11 @@ class Tools(object):
 
         """
         pathname = self.expand_path(pathname, *args, **kwargs)
-        return fs.unlink(pathname)
+        return fs.unlink(pathname, ignore_errors=kwargs.get("ignore_errors", False))
 
     @contextmanager
-    def chroot(self, chroot, *args, **kwargs):
+    @utils.deprecated
+    def chroot(self, chroot, *args, path=None, **kwargs):
         """
         Experimental: Use chroot as root filesystem when running commands.
 
@@ -1553,6 +1895,9 @@ class Tools(object):
         Args:
             chroot (str, artifact): Path to rootfs directory, or an artifact
                 with a 'rootfs' metadata path (artifact.paths.rootfs).
+            path (list): List of directory paths within the chroot to add to
+                the PATH environment variable, e.g. ["/usr/bin", "/bin"].
+                By default, the current PATH is used also within the chroot.
 
         Example:
 
@@ -1564,9 +1909,17 @@ class Tools(object):
         """
         raise_error_if(platform.system() != "Linux", "Tools.chroot() is only supported on Linux")
 
-        if type(chroot) == cache.Artifact:
+        raise_task_error_if(
+            not self.which("newuidmap") and not self.which("/usr/bin/newuidmap"), self._task,
+            "No usable 'newuidmap' found in PATH")
+
+        raise_task_error_if(
+            not self.which("newgidmap") and not self.which("/usr/bin/newuidmap"), self._task,
+            "No usable 'newgidmap' found in PATH")
+
+        if type(chroot) in [cache.Artifact, cache.ArtifactToolsProxy]:
             raise_task_error_if(
-                not str(chroot.paths.rootfs), self._task,
+                not chroot.paths.rootfs, self._task,
                 "No 'rootfs' path in artifact")
             chroot = chroot.paths.rootfs
 
@@ -1575,6 +1928,8 @@ class Tools(object):
             not fs.path.exists(chroot) or not fs.path.isdir(chroot),
             self._task, "failed to change root to '{0}'", chroot)
 
+        bind = []
+
         mount_dev = kwargs.get("mount_dev", True)
         mount_etc = kwargs.get("mount_etc", True)
         mount_home = kwargs.get("mount_home", False)
@@ -1583,93 +1938,73 @@ class Tools(object):
         mount_cachedir = kwargs.get("mount_cachedir", True)
         mount_builddir = kwargs.get("mount_builddir", True)
         mount = kwargs.get("mount", [])
+        raise_task_error_if(
+            type(mount) is not list,
+            self._task, "Expected a list as mount argument to Tools.chroot()")
+        mount = [self.expand(m) for m in mount]
 
-        overlaydir = self.builddir("overlay")
-        overlayrootdir = fs.path.join(overlaydir, "root")
-        with self.cwd(overlaydir):
-            self.mkdir("root")
-            self.mkdir("work")
-            self.mkdir("uppr")
-        overlayopts = f"upperdir={overlaydir}/uppr,workdir={overlaydir}/work,lowerdir={chroot}"
-
-        def unshare_chroot(overlayrootdir):
-            uid = os.geteuid()
-            gid = os.geteuid()
-            self._unshare([(uid, uid, 1)], [(gid, gid, 1)])
-
-            from ctypes import CDLL, c_char_p
-            libc = CDLL("libc.so.6")
-
-            MS_BIND = 4096
-            MS_REC = 16384
-
-            def mount_overlay():
-                return libc.mount(
-                    c_char_p("overlay".encode("utf-8")),
-                    c_char_p(overlayrootdir.encode("utf-8")),
-                    c_char_p("overlay".encode("utf-8")),
-                    0,
-                    c_char_p(overlayopts.encode("utf-8"))) == 0
-
-            # If the overlay mount fails, just don't use one.
-            if not mount_overlay():
-                overlayrootdir = chroot
-
-            def mount_bind(path):
-                if os.path.isdir(path):
-                    os.makedirs(overlayrootdir + path, exist_ok=True)
-                else:
-                    os.makedirs(os.path.dirname(overlayrootdir + path), exist_ok=True)
-                    if not os.path.exists(overlayrootdir + path):
-                        with open(overlayrootdir + path, "a"):
-                            pass
-                assert libc.mount(
-                    c_char_p(path.encode("utf-8")),
-                    c_char_p((overlayrootdir + path).encode("utf-8")),
-                    None,
-                    MS_BIND | MS_REC,
-                    None) == 0
-
-            if mount_etc:
-                mount_bind("/etc/group")
-                mount_bind("/etc/hostname")
-                mount_bind("/etc/hosts")
-                mount_bind("/etc/passwd")
-                mount_bind("/etc/resolv.conf")
-            if mount_home:
-                mount_bind("/home")
-            if mount_joltdir and self._task:
-                mount_bind(self._task.joltdir)
-            if mount_cachedir:
-                mount_bind(config.get_cachedir())
-            if mount_builddir:
-                mount_bind(self.buildroot)
-            if mount:
-                for m in mount:
-                    mount_bind(m)
-            if mount_dev:
-                mount_bind("/dev")
-            if mount_proc:
-                mount_bind("/proc")
-            os.chroot(overlayrootdir)
-            os.chdir(self.getcwd())
-
-        def unshare_chroot_catch():
-            try:
-                unshare_chroot(overlayrootdir)
-            except Exception as e:
-                log.exception(e)
-                raise e
+        if mount_etc:
+            bind.append("/etc/group")
+            bind.append("/etc/hostname")
+            bind.append("/etc/hosts")
+            bind.append("/etc/passwd")
+            bind.append("/etc/resolv.conf")
+
+        if mount_home:
+            bind.append("/home")
+
+        if mount_joltdir and self._task:
+            from jolt.loader import get_workspacedir
+            bind.append(get_workspacedir())
+
+        if mount_cachedir:
+            bind.append(config.get_cachedir())
+
+        if mount_builddir:
+            bind.append(self.buildroot)
+
+        if mount:
+            for m in mount:
+                bind.append(m)
+
+        if mount_dev:
+            bind.append("/dev")
+
+        if mount_proc:
+            bind.append("/proc")
+
+        unshare = os.path.join(os.path.dirname(__file__), "chroot.py")
 
         old_chroot = self._chroot
-        old_preexec_fn = self._preexec_fn
+        old_chroot_path = self._chroot_path
+        old_chroot_prefix = self._chroot_prefix
         self._chroot = chroot
-        self._preexec_fn = unshare_chroot_catch
+
+        if path:
+            self._chroot_path = path
+        else:
+            self._chroot_path = self._env.get("PATH")
+            self._chroot_path = self._chroot_path.split(fs.pathsep) if self._chroot_path else []
+
         try:
-            yield self._chroot
+            with self.tmpdir("chroot") as bindroot:
+                self._chroot_prefix = [
+                    sys.executable,
+                    unshare,
+                    "-b",
+                ] + bind + [
+                    "-c",
+                    chroot,
+                    "-t",
+                    bindroot,
+                    "--shell={shell}",
+                    "--",
+                ]
+                yield
         finally:
             self._chroot = old_chroot
-            self._preexec_fn = old_preexec_fn
+            self._chroot_path = old_chroot_path
+            self._chroot_prefix = old_chroot_prefix
 
     def _unshare(self, uidmap, gidmap):
         from ctypes import CDLL
@@ -1684,6 +2019,7 @@ class Tools(object):
         raise_task_error_if(
             not newuidmap, self._task,
             "No usable 'newuidmap' found in PATH")
+
         newgidmap = self.which("newgidmap")
         raise_task_error_if(
             not newgidmap, self._task,
@@ -1758,6 +2094,12 @@ class Tools(object):
         raise_task_error_if(
             not uidmap, self._task,
             "Invalid uid map: {}", uidmap)
+        raise_task_error_if(
+            not self.which("newuidmap"), self._task,
+            "No usable 'newuidmap' found in PATH")
+        raise_task_error_if(
+            not self.which("newgidmap"), self._task,
+            "No usable 'newgidmap' found in PATH")
 
         msgq = multiprocessing.JoinableQueue()
         pid = os.fork()
@@ -1789,6 +2131,15 @@ class Tools(object):
         """
         Uploads a file using HTTP (PUT).
 
+        Automatically expands any {keyword} arguments in the URL and pathname.
+
+        Basic authentication is supported by including the credentials in the URL.
+        Environment variables can be used to hide sensitive information. Specify
+        the environment variable name in the URI as e.g.
+        ``http://{environ[USER]}:{environ[PASS]}@host``.
+        Alternatively, the auth parameter can be used to provide an authentication
+        object that is passed to the requests.get() function.
+
         Throws a JoltError exception on failure.
 
         Args:
@@ -1801,18 +2152,38 @@ class Tools(object):
 
         """
         pathname = self.expand_path(pathname)
+        url = self.expand(url)
         name = fs.path.basename(pathname)
         size = self.file_size(pathname)
-        with log.progress("Uploading " + name, size, "B") as pbar, open(pathname, 'rb') as fileobj:
+
+        url_parsed = urlparse(url)
+        raise_task_error_if(
+            not url_parsed.scheme or not url_parsed.netloc,
+            self._task,
+            "Invalid URL: '{}'", url)
+
+        if auth is None and url_parsed.username and url_parsed.password:
+            auth = HTTPBasicAuth(url_parsed.username, url_parsed.password)
+
+        # Redact password from URL if present
+        if url_parsed.password:
+            url_parsed = url_parsed._replace(netloc=url_parsed.netloc.replace(url_parsed.password, "****"))
+
+        url_cleaned = urlunparse(url_parsed)
+
+        with log.progress("Uploading " + utils.shorten(name), size, "B") as pbar, \
+             open(pathname, 'rb') as fileobj:
+            log.verbose("{} -> {}", pathname, url_cleaned)
+
             def read():
                 data = fileobj.read(4096)
                 pbar.update(len(data))
                 return data
-            from requests.api import put
-            response = put(url, data=iter(read, b''), auth=auth, **kwargs)
+
+            response = http_session.put(url, data=iter(read, b''), auth=auth, **kwargs)
             raise_error_if(
                 exceptions and response.status_code not in [201, 204],
-                f"Upload to '{url}' failed with status '{response.status_code}'")
+                f"Upload to '{url_cleaned}' failed with status '{response.status_code}'")
         return response.status_code in [201, 204]
 
     def read_file(self, pathname, binary=False):
@@ -1821,6 +2192,60 @@ class Tools(object):
         with open(pathname, "rb" if binary else "r") as f:
             return f.read()
 
+    def read_depfile(self, pathname):
+        """
+        Reads a Make dependency file.
+
+        Returns:
+            dict: Dictionary of files and their dependencies.
+        """
+        pathname = self.expand_path(pathname)
+        with open(pathname) as f:
+            data = f.read()
+
+        data = data.strip()
+        data = data.replace("\\\n", "")
+        data = data.splitlines()
+
+        deps = {}
+
+        for line in data:
+            # Skip empty lines and comments
+            if not line or line[0] == "#":
+                continue
+
+            parts = line.split(":", 1)
+            raise_error_if(len(parts) != 2, "Depfile parse error: '{}'", line)
+            outputs, inputs = parts[0], parts[1]
+            outputs, inputs = outputs.strip(), inputs.strip()
+            # Temporarily replace escaped spaces in names so that
+            # the list of dependencies can be split.
+            outputs, inputs = outputs.replace("\\ ", "\x00"), inputs.replace("\\ ", "\x00")
+
+            for output in outputs.split():
+                output = output.replace("\x00", " ")
+                for input in inputs.split():
+                    input = input.replace("\x00", " ")
+                    if output not in deps:
+                        deps[output] = []
+                    deps[output].append(input)
+
+        return deps
+
+    def read_json(self, pathname):
+        """
+        Reads a JSON file.
+
+        Args:
+            pathname (str): Name/path of file to be read.
+
+        Returns:
+            dict: Dictionary of JSON data.
+        """
+        pathname = self.expand_path(pathname)
+        with open(pathname) as f:
+            return json.load(f)
+
     def which(self, executable):
         """ Find executable in PATH.
 
@@ -1832,9 +2257,12 @@ class Tools(object):
         """
         executable = self.expand(executable)
         path = self._env.get("PATH")
+
         if self._chroot:
-            path = fs.pathsep.join(
-                [self._chroot + p for p in path.split(fs.pathsep)]) + fs.pathsep + path
+            path = path.split(fs.pathsep) if path else []
+            path = [os.path.join(self._chroot, p.lstrip(fs.sep)) for p in self._chroot_path] + path
+            path = fs.pathsep.join(path)
+
         result = shutil.which(executable, path=path)
         if result and self._chroot and result.startswith(self._chroot):
             result = result[len(self._chroot):]
@@ -1861,3 +2289,22 @@ class Tools(object):
             content = self.expand(content, **kwargs)
         with open(pathname, "wb") as f:
             f.write(content.encode())
+
+    def write_json(self, pathname, data, indent=4, sort_keys=True):
+        """
+        Writes a JSON file.
+
+        Args:
+            pathname (str): Name/path of file to be written.
+            data (dict): Dictionary of JSON data.
+            indent (int): Indentation level for JSON output.
+        """
+        pathname = self.expand_path(pathname)
+        with open(pathname, "w") as f:
+            json.dump(data, f, indent=indent, sort_keys=sort_keys)
+
+    @property
+    def wsroot(self):
+        """ Return the root path of all build directories """
+        from jolt.loader import get_workspacedir
+        return fs.path.normpath(get_workspacedir())