jolt 0.9.172__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/tools.py

@@ -1,13 +1,16 @@
+import py7zr
 import bz2
 import copy
 import getpass
 import gzip
+import json
 import lzma
 import subprocess
 import os
 import platform
 import sys
 import threading
+import time
 if os.name != "nt":
     import termios
 import glob
@@ -18,6 +21,7 @@ import tarfile
 import zipfile
 import bz2file
 import hashlib
+import zstandard
 from contextlib import contextmanager
 from psutil import NoSuchProcess, Process
 from jinja2 import Environment, FileSystemLoader
@@ -25,6 +29,8 @@ from jinja2.exceptions import TemplateError
 from jinja2.runtime import Context
 from jinja2.utils import missing
 from requests import Session
+from requests.auth import HTTPBasicAuth
+from urllib.parse import urlparse, urlunparse
 
 
 from jolt import cache
@@ -32,11 +38,14 @@ from jolt import filesystem as fs
 from jolt import log
 from jolt import utils
 from jolt import config
-from jolt.error import JoltCommandError
+from jolt.error import JoltCommandError, JoltTimeoutError
 from jolt.error import raise_error_if
 from jolt.error import raise_task_error, raise_task_error_if
 
 
+SUPPORTED_ARCHIVE_TYPES = [".tar", ".tar.bz2", ".tar.gz", ".tgz", ".tar.xz", ".tar.zst", ".zip"]
+
+
 http_session = Session()
 
 
@@ -50,66 +59,54 @@ def stderr_write(line):
     sys.stderr.flush()
 
 
-def _run(cmd, cwd, env, preexec_fn, *args, **kwargs):
+class Reader(threading.Thread):
+    def __init__(self, parent, stream, output=None, logbuf=None, output_rstrip=True):
+        super(Reader, self).__init__()
+        self.output = output
+        self.output_rstrip = output_rstrip
+        self.parent = parent
+        self.stream = stream
+        self.logbuf = logbuf if logbuf is not None else []
+        self.start()
+
+    def run(self):
+        line = ""
+        try:
+            with log.map_thread(self, self.parent):
+                for line in iter(self.stream.readline, b''):
+                    if self.output_rstrip:
+                        line = line.rstrip()
+                    line = line.decode(errors='ignore')
+                    if self.output:
+                        self.output(line)
+                    self.logbuf.append((self, line))
+        except Exception as e:
+            if self.output:
+                self.output("{0}", str(e))
+                self.output(line)
+            self.logbuf.append((self, line))
+
+
+def _run(cmd, cwd, env, *args, **kwargs):
     output = kwargs.get("output")
     output_on_error = kwargs.get("output_on_error")
     output_rstrip = kwargs.get("output_rstrip", True)
     output_stdio = kwargs.get("output_stdio", False)
+    output_stderr = kwargs.get("output_stderr", True)
+    output_stdout = kwargs.get("output_stdout", True)
     return_stderr = kwargs.get("return_stderr", False)
     output = output if output is not None else True
     output = False if output_on_error else output
     shell = kwargs.get("shell", True)
-    timeout = kwargs.get("timeout", None)
+    timeout = kwargs.get("timeout", config.getint("jolt", "command_timeout", 0))
+    timeout = timeout if type(timeout) is int and timeout > 0 else None
 
     log.debug("Running: '{0}' (CWD: {1})", cmd, cwd)
 
-    p = subprocess.Popen(
-        cmd,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        shell=shell,
-        cwd=cwd,
-        env=env,
-        preexec_fn=preexec_fn,
-    )
-
-    class Reader(threading.Thread):
-        def __init__(self, parent, stream, output=None, logbuf=None):
-            super(Reader, self).__init__()
-            self.output = output
-            self.parent = parent
-            self.stream = stream
-            self.logbuf = logbuf if logbuf is not None else []
-            self.start()
-
-        def run(self):
-            line = ""
-            try:
-                with log.map_thread(self, self.parent):
-                    for line in iter(self.stream.readline, b''):
-                        if output_rstrip:
-                            line = line.rstrip()
-                        line = line.decode(errors='ignore')
-                        if self.output:
-                            self.output(line)
-                        self.logbuf.append((self, line))
-            except Exception as e:
-                if self.output:
-                    self.output("{0}", str(e))
-                    self.output(line)
-                self.logbuf.append((self, line))
-
-    stdout_func = log.stdout if not output_stdio else stdout_write
-    stderr_func = log.stderr if not output_stdio else stderr_write
-
-    logbuf = []
-    stdout = Reader(
-        threading.current_thread(), p.stdout,
-        output=stdout_func if output else None, logbuf=logbuf)
-    stderr = Reader(
-        threading.current_thread(), p.stderr,
-        output=stderr_func if output else None, logbuf=logbuf)
+    p = None
+    stdout = None
+    stderr = None
+    timedout = False
 
     def terminate(pid):
         try:
@@ -129,23 +126,73 @@ def _run(cmd, cwd, env, preexec_fn, *args, **kwargs):
         except NoSuchProcess:
             pass
 
-    timedout = False
     try:
+        with utils.delayed_interrupt():
+            p = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                shell=shell,
+                cwd=cwd,
+                env=env,
+            )
+
+            if output_stdout:
+                stdout_func = log.stdout if not output_stdio else stdout_write
+            else:
+                stdout_func = None
+
+            if output_stderr:
+                stderr_func = log.stderr if not output_stdio else stderr_write
+            else:
+                stderr_func = None
+
+            logbuf = []
+            stdout = Reader(
+                threading.current_thread(),
+                p.stdout,
+                output=stdout_func if output else None,
+                logbuf=logbuf,
+                output_rstrip=output_rstrip)
+            stderr = Reader(
+                threading.current_thread(),
+                p.stderr,
+                output=stderr_func if output else None,
+                logbuf=logbuf,
+                output_rstrip=output_rstrip)
+
         p.wait(timeout=timeout)
-    except subprocess.TimeoutExpired:
+
+    except KeyboardInterrupt:
+        if not p:
+            raise
+        try:
+            terminate(p.pid)
+            p.wait(10)
+        except subprocess.TimeoutExpired:
+            kill(p.pid)
+            utils.call_and_catch(p.wait, 10)
+        raise
+
+    except (subprocess.TimeoutExpired, JoltTimeoutError):
         timedout = True
         try:
             terminate(p.pid)
             p.wait(10)
         except subprocess.TimeoutExpired:
             kill(p.pid)
-            p.wait()
+            utils.call_and_catch(p.wait, 10)
+
     finally:
-        stdout.join()
-        stderr.join()
-        p.stdin.close()
-        p.stdout.close()
-        p.stderr.close()
+        if stdout:
+            stdout.join()
+        if stderr:
+            stderr.join()
+        if p:
+            p.stdin.close()
+            p.stdout.close()
+            p.stderr.close()
 
     if p.returncode != 0 and output_on_error:
         for reader, line in logbuf:
@@ -164,11 +211,13 @@ def _run(cmd, cwd, env, preexec_fn, *args, **kwargs):
 
     if p.returncode != 0:
         stderrbuf = [line for reader, line in logbuf if reader is stderr]
-        raise JoltCommandError(
-            "Command {0}: {1}".format(
-                "timeout" if timedout else "failed",
-                " ".join(cmd) if type(cmd) is list else cmd.format(*args, **kwargs)),
-            stdoutbuf, stderrbuf, p.returncode)
+        if timedout:
+            raise JoltTimeoutError(
+                "Command timeout: " + (" ".join(cmd) if type(cmd) is list else cmd))
+        else:
+            raise JoltCommandError(
+                "Command failed: " + (" ".join(cmd) if type(cmd) is list else cmd),
+                stdoutbuf, stderrbuf, p.returncode)
     if return_stderr:
         return "\n".join(stdoutbuf) if output_rstrip else "".join(stdoutbuf), \
             "\n".join(stderrbuf) if output_rstrip else "".join(stderrbuf)
@@ -205,36 +254,6 @@ class _String(object):
         return self._str.startswith(substr)
 
 
-class _tmpdir(object):
-    def __init__(self, name, cwd=None):
-        self._name = name
-        self._path = None
-        self._cwd = cwd or os.getcwd()
-
-    def __enter__(self):
-        try:
-            dirname = self._cwd
-            fs.makedirs(fs.path.join(dirname, fs.path.dirname(self._name)))
-            self._path = fs.mkdtemp(prefix=self._name + "-", dir=dirname)
-        except KeyboardInterrupt as e:
-            raise e
-        except Exception as e:
-            raise e
-        raise_error_if(not self._path, "failed to create temporary directory")
-        return self
-
-    def __exit__(self, type, value, tb):
-        if self._path:
-            fs.rmtree(self._path, ignore_errors=True)
-
-    @property
-    def path(self):
-        return self.get_path()
-
-    def get_path(self):
-        return self._path
-
-
 class _CMake(object):
     def __init__(self, deps, tools, incremental=False):
         self.deps = deps
@@ -242,6 +261,10 @@ class _CMake(object):
         self.builddir = self.tools.builddir(incremental=incremental)
         self.installdir = self.tools.builddir("install", incremental=False)
 
+    def clean(self):
+        self.tools.rmtree(self.builddir, ignore_errors=True)
+        self.tools.rmtree(self.installdir, ignore_errors=True)
+
     def configure(self, sourcedir, *args, generator=None, **kwargs):
         sourcedir = self.tools.expand_path(sourcedir)
 
@@ -252,74 +275,85 @@ class _CMake(object):
 
         with self.tools.cwd(self.builddir):
             self.tools.run(
-                "cmake {0} -B{1} -DCMAKE_INSTALL_PREFIX={2} {3} {4}",
+                "cmake {0} {1} -DCMAKE_INSTALL_PREFIX=/jolt-prefix {1} {2} {3}",
                 sourcedir,
-                self.builddir,
-                self.installdir,
+                utils.option("-B", self.builddir),
                 utils.option("-G", generator),
                 extra_args,
                 output=True)
 
-    def build(self, release=True, *args, **kwargs):
+    def build(self, *args, config="Release", **kwargs):
         threading_args = ' -j {}'.format(kwargs.get("threads", self.tools.thread_count()))
         with self.tools.cwd(self.builddir):
-            release = "--config Release" if release else ""
-            self.tools.run("cmake --build . {0}{1}", release, threading_args, output=True)
+            self.tools.run("cmake --build . --config {0} {1}", config, threading_args, output=True)
 
-    def install(self, release=True, *args, **kwargs):
-        with self.tools.cwd(self.builddir):
-            release = "--config Release" if release else ""
-            self.tools.run("cmake --build . --target install {0}", release, output=True)
+    def install(self, target="install", config="Release", **kwargs):
+        with self.tools.cwd(self.builddir), self.tools.environ(DESTDIR=self.installdir):
+            self.tools.run("cmake --build . --config {0} --target {1}", config, target, output=True)
 
-    def publish(self, artifact, files='*', *args, **kwargs):
-        with self.tools.cwd(self.installdir):
-            artifact.collect(files, *args, **kwargs)
+    def publish(self, artifact, files='*', symlinks=True, *args, **kwargs):
+        with self.tools.cwd(self.installdir, "jolt-prefix"):
+            artifact.collect(files, *args, symlinks=symlinks, **kwargs)
+        artifact.strings.install_prefix = "/jolt-prefix"
 
 
 class _Meson(object):
-    def __init__(self, deps, tools):
+    def __init__(self, deps, tools, incremental=False):
         self.deps = deps
         self.tools = tools
-        self.builddir = self.tools.builddir()
-        self.installdir = self.tools.builddir("install")
+        self.builddir = self.tools.builddir(incremental=incremental)
+        self.installdir = self.tools.builddir("install", incremental=False)
+        self.prefix = "/jolt-prefix" if os.name != "nt" else "C:\\jolt-prefix"
+
+    def clean(self):
+        self.tools.rmtree(self.builddir, ignore_errors=True)
+        self.tools.rmtree(self.installdir, ignore_errors=True)
 
     def configure(self, sourcedir, *args, **kwargs):
         sourcedir = self.tools.expand_path(sourcedir)
-        self.tools.run("meson --prefix=/ {0} {1}", sourcedir, self.builddir,
+        options = " ".join([f"-D{arg}" for arg in args]) + " "
+        options += " ".join(["-D{0}={1}".format(key, self.tools.expand(val)) for key, val in kwargs.items()])
+        self.tools.run("meson setup --prefix={0} {1} {2} {3}", self.prefix, sourcedir, self.builddir, options,
                        output=True)
 
     def build(self, *args, **kwargs):
         self.tools.run("ninja -C {0} ", self.builddir, output=True)
 
     def install(self, *args, **kwargs):
-        self.tools.run("DESTDIR={0} ninja -C {1} install",
-                       self.installdir, self.builddir,
-                       output=True)
+        with self.tools.environ(DESTDIR=self.installdir):
+            self.tools.run("ninja -C {0} install", self.builddir, output=True)
 
-    def publish(self, artifact, files='*', *args, **kwargs):
-        with self.tools.cwd(self.installdir):
-            artifact.collect(files, *args, **kwargs)
+    def publish(self, artifact, files='*', symlinks=True, *args, **kwargs):
+        with self.tools.cwd(self.installdir, "jolt-prefix"):
+            artifact.collect(files, *args, symlinks=symlinks, **kwargs)
+        artifact.strings.install_prefix = self.prefix
 
 
 class _AutoTools(object):
-    def __init__(self, deps, tools):
+    def __init__(self, deps, tools, incremental=False):
         self.deps = deps
         self.tools = tools
-        self.builddir = self.tools.builddir()
-        self.installdir = self.tools.builddir("install")
+        self.builddir = self.tools.builddir(incremental=incremental)
+        self.installdir = self.tools.builddir("install", incremental=False)
+        self.prefix = "jolt-prefix"
 
-    def configure(self, sourcedir, *args, **kwargs):
+    def clean(self):
+        self.tools.rmtree(self.builddir, ignore_errors=True)
+        self.tools.rmtree(self.installdir, ignore_errors=True)
+
+    def configure(self, sourcedir, *args):
         sourcedir = self.tools.expand_path(sourcedir)
-        prefix = kwargs.get("prefix", "/")
 
         if not fs.path.exists(fs.path.join(sourcedir, "configure")):
             with self.tools.cwd(sourcedir):
                 self.tools.run("autoreconf -visf", output=True)
 
         with self.tools.cwd(self.builddir), self.tools.environ(DESTDIR=self.installdir):
-            self.tools.run("{0}/configure --prefix={1} {2}",
-                           sourcedir, prefix,
-                           self.tools.getenv("CONFIGURE_FLAGS"),
+            self.tools.run("{0}/configure --prefix=/{1} {2} {3}",
+                           sourcedir,
+                           self.prefix,
+                           self.tools.getenv("CONFIGURE_FLAGS", ""),
+                           " ".join(args),
                            output=True)
 
     def build(self, *args, **kwargs):
@@ -327,13 +361,14 @@ class _AutoTools(object):
             self.tools.run("make VERBOSE=yes Q= V=1 -j{0}",
                            self.tools.cpu_count(), output=True)
 
-    def install(self, target="install", **kwargs):
-        with self.tools.cwd(self.builddir), self.tools.environ(DESTDIR=self.installdir):
-            self.tools.run("make {}", target, output=True)
+    def install(self, target="install"):
+        with self.tools.cwd(self.builddir):
+            self.tools.run("make DESTDIR={} {}", self.installdir, target, output=True)
 
-    def publish(self, artifact, files='*', *args, **kwargs):
-        with self.tools.cwd(self.installdir):
-            artifact.collect(files, *args, **kwargs)
+    def publish(self, artifact, files='*', symlinks=True, *args, **kwargs):
+        with self.tools.cwd(self.installdir, self.prefix):
+            artifact.collect(files, *args, symlinks=symlinks, **kwargs)
+        artifact.strings.install_prefix = "/" + self.prefix
 
 
 class ZipFile(zipfile.ZipFile):
@@ -461,8 +496,9 @@ class Tools(object):
     def __init__(self, task=None, cwd=None, env=None):
         self._chroot = None
         self._chroot_prefix = []
+        self._chroot_path = []
+        self._deadline = None
         self._run_prefix = []
-        self._preexec_fn = None
         self._cwd = fs.path.normpath(fs.path.join(config.get_workdir(), cwd or config.get_workdir()))
         self._env = copy.deepcopy(env or os.environ)
         self._task = task
@@ -511,24 +547,50 @@ class Tools(object):
                         zf.write(path, zippath)
         return filename
 
+    def _make_7zfile(self, filename, fmt, rootdir):
+        self.mkdirname(filename)
+        with py7zr.SevenZipFile(filename, 'w') as archive:
+            archive.writeall(rootdir, ".")
+        return filename
+
     def _make_tarfile(self, filename, fmt, rootdir):
-        dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            fs.makedirs(dirname)
+        self.mkdirname(filename)
         with tarfile.open(filename, 'w|%s' % fmt) as tar:
             tar.add(rootdir, ".")
         return filename
 
+    def _make_tarzstd(self, filename, rootdir):
+        self.mkdirname(filename)
+        with open(filename, 'wb') as zstd_file:
+            compressor = zstandard.ZstdCompressor(threads=self.thread_count())
+            with compressor.stream_writer(zstd_file) as stream:
+                with tarfile.open(mode="w|", fileobj=stream) as tar:
+                    tar.add(rootdir, ".")
+        return filename
+
+    def _extract_tarzstd(self, filename, pathname, files=None):
+        with open(filename, 'rb') as zstd_file:
+            decompressor = zstandard.ZstdDecompressor()
+            with decompressor.stream_reader(zstd_file) as stream:
+                with tarfile.open(mode="r|", fileobj=stream) as tar:
+                    if files:
+                        for file in files:
+                            tar.extract(file, pathname)
+                    else:
+                        tar.extractall(pathname)
+
     def archive(self, pathname, filename):
         """ Creates a (compressed) archive.
 
         The type of archive to create is determined by the filename extension.
         Supported formats are:
 
+        - 7z
         - tar
         - tar.bz2
         - tar.gz
         - tar.xz
+        - tar.zst
         - zip
 
         Args:
@@ -548,6 +610,8 @@ class Tools(object):
                 self.run("tar -I pigz -cf {} -C {} .", filename, pathname)
                 return filename
             fmt = "targz"
+        elif filename.endswith(".tar.zst"):
+            return self._make_tarzstd(filename, rootdir=pathname)
         elif filename.endswith(".tgz"):
             if self.which("tar") and self.which("pigz"):
                 self.run("tar -I pigz -cf {} -C {} .", filename, pathname)
@@ -557,12 +621,16 @@ class Tools(object):
             fmt = "tarbz2"
         elif filename.endswith(".tar.xz"):
             fmt = "tarxz"
+        elif filename.endswith(".7z"):
+            fmt = "7z"
         raise_task_error_if(
             not fmt, self._task,
             "unknown archive type '{0}'", fs.path.basename(filename))
         try:
             if fmt == "zip":
                 outfile = self._make_zipfile(filename, fmt, rootdir=pathname)
+            elif fmt == "7z":
+                outfile = self._make_7zfile(filename, fmt, rootdir=pathname)
             else:
                 outfile = self._make_tarfile(filename, fmt[3:], rootdir=pathname)
             if outfile != filename:
@@ -571,9 +639,9 @@ class Tools(object):
         except Exception:
             raise_task_error(self._task, "failed to create archive from directory '{0}'", pathname)
 
-    def autotools(self, deps=None):
+    def autotools(self, deps=None, incremental=False):
         """ Creates an AutoTools invokation helper """
-        return _AutoTools(deps, self)
+        return _AutoTools(deps, self, incremental=incremental)
 
     @utils.locked(lock='_builddir_lock')
     def builddir(self, name=None, incremental=False, unique=True):
@@ -600,8 +668,16 @@ class Tools(object):
             name += "-" + utils.canonical(self._task.short_qualified_name)
 
         dirname = fs.path.join(self.getcwd(), name)
+        if incremental:
+            dirname += "-inc"
+
+        # Check if incremental build directories are disabled in the configuration
+        if incremental not in ["always"] and not config.is_incremental_build():
+            incremental = False
 
         if incremental:
+            # Create a unique build directory for each task
+            # and store the task name in a hidden file.
             if self._task is not None and unique:
                 meta_task = fs.path.join(dirname, ".task")
                 if not fs.path.exists(meta_task) \
@@ -609,6 +685,7 @@ class Tools(object):
                     fs.rmtree(dirname, ignore_errors=True)
                     fs.makedirs(dirname)
 
+                # Remove the build directory if the task taint has changed (--force or --salt)
                 if self._task.taint is not None:
                     meta = fs.path.join(dirname, ".taint")
                     if not fs.path.exists(meta) or self.read_file(meta) != str(self._task.taint):
@@ -622,21 +699,16 @@ class Tools(object):
             return dirname
 
         if name not in self._builddir:
-            if unique:
-                fs.makedirs(fs.path.dirname(dirname))
-                self._builddir[name] = fs.mkdtemp(
-                    prefix=fs.path.basename(dirname) + "-",
-                    dir=fs.path.dirname(dirname))
-            else:
-                fs.makedirs(dirname)
-                self._builddir[name] = dirname
+            fs.makedirs(dirname)
+            self._builddir[name] = dirname
 
         return self._builddir[name]
 
     @property
     def buildroot(self):
         """ Return the root path of all build directories """
-        return fs.path.normpath(fs.path.join(self.wsroot, "build"))
+        from jolt.loader import JoltLoader
+        return fs.path.normpath(JoltLoader.get().build_path)
 
     def checksum_file(self, filelist, concat=False, hashfn=hashlib.sha1, filterfn=None):
         """ Calculate a checksum of one or multiple files.
@@ -693,6 +765,7 @@ class Tools(object):
         - .bz2
         - .gz
         - .xz
+        - .zst
 
         Args:
             src (str): Source file to be compressed.
@@ -720,6 +793,13 @@ class Tools(object):
                 with lzma.open(dst, 'wb') as outfp:
                     for block in iter(lambda: infp.read(0x10000), b''):
                         outfp.write(block)
+        elif ext == "zst":
+            with open(src, 'rb') as infp:
+                with open(dst, 'wb') as outfp:
+                    compressor = zstandard.ZstdCompressor(threads=self.thread_count())
+                    with compressor.stream_writer(outfp) as stream:
+                        for block in iter(lambda: infp.read(0x10000), b''):
+                            stream.write(block)
 
     def copy(self, src, dst, symlinks=False):
         """ Copies file and directories (recursively).
@@ -788,10 +868,19 @@ class Tools(object):
         finally:
             self._cwd = prev
 
-    def download(self, url, pathname, exceptions=True, **kwargs):
+    def download(self, url, pathname, exceptions=True, auth=None, **kwargs):
         """
         Downloads a file using HTTP.
 
+        Automatically expands any {keyword} arguments in the URL and pathname.
+
+        Basic authentication is supported by including the credentials in the URL.
+        Environment variables can be used to hide sensitive information. Specify
+        the environment variable name in the URI as e.g.
+        ``http://{environ[USER]}:{environ[PASS]}@host``.
+        Alternatively, the auth parameter can be used to provide an authentication
+        object that is passed to the requests.get() function.
+
         Throws a JoltError exception on failure.
 
         Args:
@@ -804,16 +893,32 @@ class Tools(object):
 
         url = self.expand(url)
         pathname = self.expand_path(pathname)
+
+        url_parsed = urlparse(url)
+        raise_task_error_if(
+            not url_parsed.scheme or not url_parsed.netloc,
+            self._task,
+            "Invalid URL: '{}'", url)
+
+        if auth is None and url_parsed.username and url_parsed.password:
+            auth = HTTPBasicAuth(url_parsed.username, url_parsed.password)
+
+        # Redact password from URL if present
+        if url_parsed.password:
+            url_parsed = url_parsed._replace(netloc=url_parsed.netloc.replace(url_parsed.password, "****"))
+
+        url_cleaned = urlunparse(url_parsed)
+
         try:
-            response = http_session.get(url, stream=True, **kwargs)
+            response = http_session.get(url, stream=True, auth=auth, **kwargs)
             raise_error_if(
                 exceptions and response.status_code not in [200],
-                f"Download from '{url}' failed with status '{response.status_code}'")
+                f"Download from '{url_cleaned}' failed with status '{response.status_code}'")
 
             name = fs.path.basename(pathname)
             size = int(response.headers.get('content-length', 0))
             with log.progress("Downloading {0}".format(utils.shorten(name)), size, "B") as pbar:
-                log.verbose("{} -> {}", url, pathname)
+                log.verbose("{} -> {}", url_cleaned, pathname)
                 with open(pathname, 'wb') as out_file:
                     chunk_size = 4096
                     for data in response.iter_content(chunk_size=chunk_size):
@@ -821,7 +926,7 @@ class Tools(object):
                         pbar.update(len(data))
                 actual_size = self.file_size(pathname)
                 raise_error_if(
-                    size != 0 and size != actual_size,
+                    size != 0 and size > actual_size,
                     f"Downloaded file was truncated to {actual_size}/{size} bytes: {name}")
 
             return response.status_code == 200
@@ -850,16 +955,29 @@ class Tools(object):
                 with tools.environ(CC="clang"):
                     tools.run("make all")
         """
-        for key, value in kwargs.items():
-            kwargs[key] = self.expand(value)
-
         restore = {key: value for key, value in self._env.items()}
-        self._env.update(kwargs)
-        yield self._env
+
         for key, value in kwargs.items():
-            if key not in restore:
-                del self._env[key]
-        self._env.update(restore)
+            if value is not None:
+                self._env[key] = self.expand(value)
+            else:
+                self._env.pop(key, None)
+
+        try:
+            yield self._env
+        finally:
+            self._env = restore
+
+    def exists(self, pathname):
+        """ Checks if a file or directory exists.
+
+        Args:
+            pathname (str): Path to file or directory.
+
+        Returns:
+            bool: True if the file or directory exists, False otherwise.
+        """
+        return fs.path.exists(self.expand_path(pathname))
 
     def expand(self, string, *args, **kwargs):
         """ Expands keyword arguments/macros in a format string.
@@ -967,10 +1085,12 @@ class Tools(object):
 
         Supported formats are:
 
+        - 7z
         - tar
         - tar.bz2
         - tar.gz
         - tar.xz
+        - tar.zst
         - zip
 
         Args:
@@ -1024,6 +1144,18 @@ class Tools(object):
                             tar.extract(file, filepath)
                     else:
                         tar.extractall(filepath)
+            elif filename.endswith(".tar.zst"):
+                try:
+                    self._extract_tarzstd(filename, filepath, files)
+                except tarfile.StreamError as e:
+                    raise_task_error(self._task, "failed to extract archive '{0}': {1}", filename, str(e))
+            elif filename.endswith(".7z"):
+                with py7zr.SevenZipFile(filename, 'r') as archive:
+                    if files:
+                        for file in files:
+                            archive.extract(file, filepath)
+                    else:
+                        archive.extractall(filepath)
             else:
                 raise_task_error(self._task, "unknown archive type '{0}'", fs.path.basename(filename))
         except Exception:
@@ -1053,7 +1185,7 @@ class Tools(object):
         """ Returns the current working directory. """
         return fs.path.normpath(self._cwd)
 
-    def getenv(self, key, default=""):
+    def getenv(self, key, default=None):
         """ Returns the value of an environment variable.
 
         Only child processes spawned by the same tools object can see
@@ -1090,7 +1222,19 @@ class Tools(object):
             files = [self.expand_path(file) for file in files]
         elif not fs.path.isabs(pathname):
             files = [self.expand_relpath(file, self.getcwd()) for file in files]
-        return files
+        return list(sorted(files))
+
+    def isdir(self, pathname):
+        """ Determines if a path is a directory.
+
+        Args:
+            pathname (str): Path to a file or directory.
+
+        Returns:
+            boolean: True if the path is a directory, False otherwise.
+        """
+        pathname = self.expand_path(pathname)
+        return fs.path.isdir(pathname)
 
     def mkdir(self, pathname, recursively=True):
         """ Create directory. """
@@ -1163,9 +1307,80 @@ class Tools(object):
         """
         return utils.map_concurrent(callable, iterable, max_workers)
 
-    def meson(self, deps=None):
+    def meson(self, deps=None, incremental=False):
         """ Creates a Meson invokation helper """
-        return _Meson(deps, self)
+        return _Meson(deps, self, incremental=incremental)
+
+    @contextmanager
+    def nixpkgs(self, nixfile=None, packages=None, pure=False, path=None, options=None):
+        """
+        Creates a Nix environment with the specified packages.
+
+        Args:
+            nixfile (str): Path to a Nix expression file.
+            packages (list): List of Nix packages to include in environment.
+            pure (boolean): Create a pure environment.
+            path (list): List of Nix expression paths.
+            options (dict): Nix configuration options.
+
+        Example:
+
+            .. code-block:: python
+
+                def run(self, deps, tools):
+                    with tools.nixpkgs(packages=["gcc13"]):
+                        tools.run("gcc --version")
+
+        """
+
+        # Check if Nix is available
+        raise_task_error_if(
+            not self.which("nix-shell"),
+            self._task,
+            "Nix not available on this system")
+
+        nixfile = self.expand_path(nixfile) if nixfile else ""
+        pathflags = " ".join([f"-I {path}" for path in path or []])
+        options = " ".join([f"--option {k} {v}" for k, v in (options or {}).items()])
+        pureflag = "--pure" if pure else ""
+        packages = "-p " + " ".join(packages) if packages else ""
+
+        # Expand all placeholders
+        options = self.expand(options)
+        packages = self.expand(packages)
+        pathflags = self.expand(pathflags)
+
+        # Use cached-nix-shell is available
+        nixshell = "cached-nix-shell" if self.which("cached-nix-shell") else "nix-shell"
+
+        # Run nix-shell to stage packages and environment
+        env = self.run(
+            "{} {} {} {} --run 'env -0' {}",
+            nixshell,
+            pathflags,
+            pureflag,
+            packages,
+            nixfile,
+            output_on_error=True)
+        env = env.strip().strip("\x00")
+        env = dict(line.split("=", 1) for line in env.split('\x00'))
+
+        # Add host path first to environment PATH
+        host_path = env.get("HOST_PATH", None)
+        if host_path:
+            env["PATH"] = host_path + os.pathsep + env["PATH"]
+
+        # Enter the environment
+        old_env = self._env
+        try:
+            if pure:
+                self._env = env
+            else:
+                self._env = copy.deepcopy(env)
+                self._env.update(env)
+            yield
+        finally:
+            self._env = old_env
 
     def render(self, template, **kwargs):
         """ Render a Jinja template string.
@@ -1370,6 +1585,9 @@ class Tools(object):
                 refuses to terminate, it will be killed after an additional
                 10 seconds have passed. Default: None.
 
+        Returns:
+           str: stdout from command unless output=False
+
         Example:
 
             .. code-block:: python
@@ -1390,6 +1608,11 @@ class Tools(object):
             else:
                 cmd = " ".join(self._chroot_prefix + self._run_prefix) + " " + cmd
 
+        if self._deadline is not None:
+            remaining = int(self._deadline - time.time() + 0.5)
+            timeout = kwargs.get("timeout", remaining)
+            kwargs["timeout"] = min(remaining, timeout)
+
         cmd = self.expand(cmd, *args, **kwargs)
 
         stdi, stdo, stde = None, None, None
@@ -1404,7 +1627,7 @@ class Tools(object):
             except Exception:
                 pass
 
-            return _run(cmd, self._cwd, self._env, self._preexec_fn, *args, **kwargs)
+            return _run(cmd, self._cwd, self._env, *args, **kwargs)
 
         finally:
             if stdi:
@@ -1484,7 +1707,7 @@ class Tools(object):
         """
 
         raise_error_if(
-            type(artifact) is not cache.Artifact,
+            type(artifact) not in [cache.Artifact, cache.ArtifactToolsProxy],
             "non-artifact passed as argument to Tools.sandbox()")
 
         suffix = utils.canonical(artifact.task.short_qualified_name)
@@ -1518,11 +1741,15 @@ class Tools(object):
             fs.makedirs(path)
             for relsrcpath, reldstpath in artifact.files.items():
                 srcpath = fs.path.normpath(fs.path.join(artifact.task.joltdir, relsrcpath))
+                srcpath = self.expand_path(srcpath)
                 dstpath = fs.path.normpath(fs.path.join(path, reldstpath))
+                dstpath = self.expand_path(dstpath)
+
                 if dstpath != fs.path.realpath(dstpath):
                     log.debug("Cannot symlink '{} -> {}', parent directory already symlinked",
                               srcpath, dstpath)
                     continue
+
                 if fs.path.isdir(dstpath):
                     files = fs.scandir(srcpath)
                     for file in files:
@@ -1532,7 +1759,8 @@ class Tools(object):
                     self.symlink(srcpath, dstpath)
 
                 # Restore missing srcfiles if they resided in a build directory
-                if srcpath.startswith(artifact.tools.buildroot) and \
+                buildroot_abs = self.expand_path(artifact.tools.buildroot)
+                if srcpath.startswith(buildroot_abs) and \
                    not fs.path.exists(srcpath):
                     fs.copy(fs.path.join(artifact.path, reldstpath), srcpath, symlinks=True)
             self.write_file(meta, artifact.path)
@@ -1579,7 +1807,41 @@ class Tools(object):
             fs.makedirs(dstdir)
         fs.symlink(src, dst)
 
-    def tmpdir(self, name):
+    @contextmanager
+    def timeout(self, seconds):
+        """ Context manager to set a timeout for a block of code.
+
+        A TimeoutError exception is raised if the block of code does not
+        complete within the specified time.
+
+        Args:
+            seconds (int): Timeout in seconds.
+
+        Example:
+
+                .. code-block:: python
+
+                    with tools.timeout(5):
+                        tools.run("sleep 10")
+
+        """
+        if seconds is None:
+            yield
+            return
+
+        with utils.timeout(seconds, JoltTimeoutError):
+            old_deadline = self._deadline
+            try:
+                if old_deadline is None:
+                    self._deadline = time.time() + seconds
+                else:
+                    self._deadline = min(old_deadline, time.time() + seconds)
+                yield
+            finally:
+                self._deadline = old_deadline
+
+    @contextmanager
+    def tmpdir(self, name=None):
         """ Creates a temporary directory.
 
         The directory is only valid within a context and it is removed
@@ -1594,11 +1856,18 @@ class Tools(object):
 
             .. code-block:: python
 
-                with tools.tmpdir("temp") as tmp, tools.cwd(tmp.path):
+                with tools.tmpdir() as tmp, tools.cwd(tmp):
                     tools.write_file("tempfile", "tempdata")
 
         """
-        return _tmpdir(name, cwd=self._cwd)
+        dirname = None
+        try:
+            self.mkdir(self.buildroot)
+            dirname = fs.mkdtemp(prefix=(name or "tmpdir") + "-", dir=self.buildroot)
+            yield fs.path.normpath(dirname)
+        finally:
+            if dirname:
+                self.rmtree(dirname, ignore_errors=True)
 
     def unlink(self, pathname, *args, **kwargs):
         """Removes a file from disk.
@@ -1613,7 +1882,8 @@ class Tools(object):
         return fs.unlink(pathname, ignore_errors=kwargs.get("ignore_errors", False))
 
     @contextmanager
-    def chroot(self, chroot, *args, **kwargs):
+    @utils.deprecated
+    def chroot(self, chroot, *args, path=None, **kwargs):
         """
         Experimental: Use chroot as root filesystem when running commands.
 
@@ -1625,6 +1895,9 @@ class Tools(object):
         Args:
             chroot (str, artifact): Path to rootfs directory, or an artifact
                 with a 'rootfs' metadata path (artifact.paths.rootfs).
+            path (list): List of directory paths within the chroot to add to
+                the PATH environment variable, e.g. ["/usr/bin", "/bin"].
+                By default, the current PATH is used also within the chroot.
 
         Example:
 
@@ -1637,16 +1910,16 @@ class Tools(object):
         raise_error_if(platform.system() != "Linux", "Tools.chroot() is only supported on Linux")
 
         raise_task_error_if(
-            not self.which("newuidmap"), self._task,
+            not self.which("newuidmap") and not self.which("/usr/bin/newuidmap"), self._task,
             "No usable 'newuidmap' found in PATH")
 
         raise_task_error_if(
-            not self.which("newgidmap"), self._task,
+            not self.which("newgidmap") and not self.which("/usr/bin/newuidmap"), self._task,
             "No usable 'newgidmap' found in PATH")
 
-        if type(chroot) is cache.Artifact:
+        if type(chroot) in [cache.Artifact, cache.ArtifactToolsProxy]:
             raise_task_error_if(
-                not str(chroot.paths.rootfs), self._task,
+                not chroot.paths.rootfs, self._task,
                 "No 'rootfs' path in artifact")
             chroot = chroot.paths.rootfs
 
@@ -1703,22 +1976,34 @@ class Tools(object):
         unshare = os.path.join(os.path.dirname(__file__), "chroot.py")
 
         old_chroot = self._chroot
+        old_chroot_path = self._chroot_path
         old_chroot_prefix = self._chroot_prefix
         self._chroot = chroot
-        self._chroot_prefix = [
-            sys.executable,
-            unshare,
-            "-b",
-        ] + bind + [
-            "-c",
-            chroot,
-            "--shell={shell}",
-            "--",
-        ]
+
+        if path:
+            self._chroot_path = path
+        else:
+            self._chroot_path = self._env.get("PATH")
+            self._chroot_path = self._chroot_path.split(fs.pathsep) if self._chroot_path else []
+
         try:
-            yield
+            with self.tmpdir("chroot") as bindroot:
+                self._chroot_prefix = [
+                    sys.executable,
+                    unshare,
+                    "-b",
+                ] + bind + [
+                    "-c",
+                    chroot,
+                    "-t",
+                    bindroot,
+                    "--shell={shell}",
+                    "--",
+                ]
+                yield
         finally:
             self._chroot = old_chroot
+            self._chroot_path = old_chroot_path
             self._chroot_prefix = old_chroot_prefix
 
     def _unshare(self, uidmap, gidmap):
@@ -1846,6 +2131,15 @@ class Tools(object):
         """
         Uploads a file using HTTP (PUT).
 
+        Automatically expands any {keyword} arguments in the URL and pathname.
+
+        Basic authentication is supported by including the credentials in the URL.
+        Environment variables can be used to hide sensitive information. Specify
+        the environment variable name in the URI as e.g.
+        ``http://{environ[USER]}:{environ[PASS]}@host``.
+        Alternatively, the auth parameter can be used to provide an authentication
+        object that is passed to the requests.get() function.
+
         Throws a JoltError exception on failure.
 
         Args:
@@ -1858,11 +2152,28 @@ class Tools(object):
 
         """
         pathname = self.expand_path(pathname)
+        url = self.expand(url)
         name = fs.path.basename(pathname)
         size = self.file_size(pathname)
+
+        url_parsed = urlparse(url)
+        raise_task_error_if(
+            not url_parsed.scheme or not url_parsed.netloc,
+            self._task,
+            "Invalid URL: '{}'", url)
+
+        if auth is None and url_parsed.username and url_parsed.password:
+            auth = HTTPBasicAuth(url_parsed.username, url_parsed.password)
+
+        # Redact password from URL if present
+        if url_parsed.password:
+            url_parsed = url_parsed._replace(netloc=url_parsed.netloc.replace(url_parsed.password, "****"))
+
+        url_cleaned = urlunparse(url_parsed)
+
         with log.progress("Uploading " + utils.shorten(name), size, "B") as pbar, \
              open(pathname, 'rb') as fileobj:
-            log.verbose("{} -> {}", pathname, url)
+            log.verbose("{} -> {}", pathname, url_cleaned)
 
             def read():
                 data = fileobj.read(4096)
@@ -1872,7 +2183,7 @@ class Tools(object):
             response = http_session.put(url, data=iter(read, b''), auth=auth, **kwargs)
             raise_error_if(
                 exceptions and response.status_code not in [201, 204],
-                f"Upload to '{url}' failed with status '{response.status_code}'")
+                f"Upload to '{url_cleaned}' failed with status '{response.status_code}'")
         return response.status_code in [201, 204]
 
     def read_file(self, pathname, binary=False):
@@ -1921,6 +2232,20 @@ class Tools(object):
 
         return deps
 
+    def read_json(self, pathname):
+        """
+        Reads a JSON file.
+
+        Args:
+            pathname (str): Name/path of file to be read.
+
+        Returns:
+            dict: Dictionary of JSON data.
+        """
+        pathname = self.expand_path(pathname)
+        with open(pathname) as f:
+            return json.load(f)
+
     def which(self, executable):
         """ Find executable in PATH.
 
@@ -1934,8 +2259,9 @@ class Tools(object):
         path = self._env.get("PATH")
 
         if self._chroot:
-            path = fs.pathsep.join(
-                [self._chroot + p for p in path.split(fs.pathsep)]) + fs.pathsep + path
+            path = path.split(fs.pathsep) if path else []
+            path = [os.path.join(self._chroot, p.lstrip(fs.sep)) for p in self._chroot_path] + path
+            path = fs.pathsep.join(path)
 
         result = shutil.which(executable, path=path)
         if result and self._chroot and result.startswith(self._chroot):
@@ -1964,6 +2290,19 @@ class Tools(object):
         with open(pathname, "wb") as f:
             f.write(content.encode())
 
+    def write_json(self, pathname, data, indent=4, sort_keys=True):
+        """
+        Writes a JSON file.
+
+        Args:
+            pathname (str): Name/path of file to be written.
+            data (dict): Dictionary of JSON data.
+            indent (int): Indentation level for JSON output.
+        """
+        pathname = self.expand_path(pathname)
+        with open(pathname, "w") as f:
+            json.dump(data, f, indent=indent, sort_keys=sort_keys)
+
     @property
     def wsroot(self):
         """ Return the root path of all build directories """