jolt 0.9.342__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/plugins/pkgconfig.py

@@ -0,0 +1,219 @@
+from jolt import attributes
+from jolt import filesystem as fs
+from jolt import log
+from jolt import utils
+from jolt.error import raise_error_if
+
+
+class PkgConfigHelper(object):
+
+    TEMPLATE_PKGCONFIG = """
+prefix={{ artifact.final_path }}
+
+Name: {{ pkgname }}
+Description: {{ pkgname }} package
+Version: {{ artifact.identity }}
+
+Cflags: {% for flag in cxxflags %}{{ flag }} {% endfor %}{% for inc in incpaths %}-I${prefix}/{{ inc }} {% endfor %}{% for macro in macros %}-D{{ macro }} {% endfor %}
+
+Libs: {% for flag in ldflags %}{{ flag }} {% endfor %}{% for libpath in libpaths %}-L${prefix}/{{ libpath }} {% endfor %}{% for library in libraries %}-l{{ library }} {% endfor %}
+
+"""
+
+    def __init__(self, artifact, tools):
+        self.artifact = artifact
+        self.tools = tools
+
+    def _mkpath(self, path):
+        if fs.path.commonpath([path, self.artifact.path]) != self.artifact.path:
+            return path
+        return fs.path.relpath(path, self.artifact.path)
+
+    def cflags(self, package):
+        package = " ".join(utils.as_list(package))
+        try:
+            with self.tools.environ(**self.environ):
+                output = self.tools.run("{} --maximum-traverse-depth=1 --cflags-only-other {}", self.pkgconfig, package, output=False)
+                return output.strip().split()
+        except Exception as e:
+            log.debug("PkgConfig.cflags: {}", str(e))
+            return []
+
+    def incpaths(self, package):
+        package = " ".join(utils.as_list(package))
+        try:
+            with self.tools.environ(**self.environ):
+                output = self.tools.run("{} --maximum-traverse-depth=1 --cflags-only-I {}", self.pkgconfig, package, output=False)
+                return [self._mkpath(inc[2:]) for inc in output.strip().split()]
+        except Exception as e:
+            log.debug("PkgConfig.incpaths: {}", str(e))
+            return []
+
+    def linkflags(self, package):
+        package = " ".join(utils.as_list(package))
+        try:
+            with self.tools.environ(**self.environ):
+                output = self.tools.run("{} --maximum-traverse-depth=1 --libs-only-other {}", self.pkgconfig, package, output=False)
+                return output.strip().split()
+        except Exception as e:
+            log.debug("PkgConfig.linkflags: {}", str(e))
+            return []
+
+    def libpaths(self, package):
+        package = " ".join(utils.as_list(package))
+        try:
+            with self.tools.environ(**self.environ):
+                output = self.tools.run("{} --maximum-traverse-depth=1 --libs-only-L {}", self.pkgconfig, package, output=False)
+                return [self._mkpath(lib[2:]) for lib in output.strip().split()]
+        except Exception as e:
+            log.debug("PkgConfig.libpaths: {}", str(e))
+            return []
+
+    def libraries(self, package):
+        package = " ".join(utils.as_list(package))
+        try:
+            with self.tools.environ(**self.environ):
+                output = self.tools.run("{} --maximum-traverse-depth=1 --libs-only-l {}", self.pkgconfig, package, output=False)
+                return [lib[2:] for lib in output.strip().split()]
+        except Exception as e:
+            log.debug("PkgConfig.libraries: {}", str(e))
+            return []
+
+    def write_pc(self, package):
+        with self.tools.tmpdir() as tmpdir, self.tools.cwd(tmpdir):
+            content = self.tools.render(
+                self.TEMPLATE_PKGCONFIG,
+                artifact=self.artifact,
+                pkgname=package,
+                macros=list(self.artifact.cxxinfo.macros),
+                incpaths=list(self.artifact.cxxinfo.incpaths),
+                libpaths=list(self.artifact.cxxinfo.libpaths),
+                libraries=list(self.artifact.cxxinfo.libraries),
+            )
+            print(content)
+            self.tools.write_file(f"{package}.pc", content, expand=False)
+            self.artifact.collect(f"{package}.pc", "lib/pkgconfig/")
+
+    @property
+    def pkgconfig(self):
+        pkgconf = self.tools.which(self.tools.getenv("PKG_CONFIG", "pkg-config"))
+        raise_error_if(not pkgconf, "No pkg-config tool found in PATH")
+        return pkgconf
+
+    @property
+    def environ(self):
+        path = self.artifact.environ.get("PKG_CONFIG_PATH")
+        if path is None:
+            self.tools._task.verbose("No PKG_CONFIG_PATH in artifact environment")
+
+        # Path from the artifact environment
+        path = str(path).split(fs.pathsep)
+        path = fs.pathsep.join(fs.path.join(self.artifact.path, p) for p in path)
+
+        # Append existing PKG_CONFIG_PATH from the tools environment
+        if self.tools.getenv("PKG_CONFIG_PATH"):
+            path = path + fs.pathsep + self.tools.getenv("PKG_CONFIG_PATH")
+
+        return {"PKG_CONFIG_PATH": path}
+
+
+def to_cxxinfo(
+    pkg: list | str,
+    cflags: bool = True,
+    cxxflags: bool = True,
+    incpaths: bool = True,
+    ldflags: bool = True,
+    libpaths: bool = True,
+    libraries: bool = True,
+):
+    """
+    Decorator to add pkg-config information to cxxinfo metadata of an artifact.
+
+    The decorator enables interoperability between libraries built with Jolt's
+    Ninja plugin and external packages that provide pkg-config files.
+
+    It uses the pkg-config tool to query for compiler and linker flags,
+    include paths, library paths, and libraries associated with a given package.
+    If the relevant flags are found, they are appended to the corresponding fields
+    in the artifact's cxxinfo metadata.
+
+    Args:
+        pkg (str): The name of the pkg-config package to be added to cxxinfo.
+        cflags (bool): Whether to add C compiler flags from pkg-config.
+        cxxflags (bool): Whether to add C++ compiler flags from pkg-config.
+        incpaths (bool): Whether to add include paths from pkg-config.
+        ldflags (bool): Whether to add linker flags from pkg-config.
+        libpaths (bool): Whether to add library paths from pkg-config.
+        libraries (bool): Whether to add libraries from pkg-config.
+    """
+
+    def decorate(cls):
+        original_publish = cls.publish
+
+        def publish(self, artifact, tools):
+            original_publish(self, artifact, tools)
+
+            pc = PkgConfigHelper(artifact, tools)
+            if not pc.environ:
+                self.verbose("Skipping pkg-config cxxinfo addition due to missing PKG_CONFIG_PATH.")
+                return
+
+            if cflags:
+                artifact.cxxinfo.cflags.extend(pc.cflags(pkg))
+            if cxxflags:
+                artifact.cxxinfo.cxxflags.extend(pc.cflags(pkg))
+            if incpaths:
+                artifact.cxxinfo.incpaths.extend(pc.incpaths(pkg))
+            if ldflags:
+                artifact.cxxinfo.ldflags.extend(pc.linkflags(pkg))
+            if libpaths:
+                artifact.cxxinfo.libpaths.extend(pc.libpaths(pkg))
+            if libraries:
+                artifact.cxxinfo.libraries.extend(pc.libraries(pkg))
+
+        cls.publish = publish
+        return cls
+
+    return decorate
+
+
+def from_cxxinfo(package):
+    """
+    Decorator to write a pkg-config file for the given package
+    based on the cxxinfo metadata of the artifact.
+    """
+
+    def decorate(cls):
+        original_publish = cls.publish
+        original_unpack = cls.unpack
+
+        def publish(self, artifact, tools):
+            original_publish(self, artifact, tools)
+
+            pc = PkgConfigHelper(artifact, tools)
+            pc.write_pc(package)
+
+        def unpack(self, artifact, tools):
+            original_unpack(self, artifact, tools)
+
+            pc = PkgConfigHelper(artifact, tools)
+            pc.write_pc(package)
+
+        cls.publish = publish
+        cls.unpack = unpack
+        return cls
+
+    return decorate
+
+
+def requires():
+    """ Decorator to add pkg-config requirements to a task. """
+
+    import jolt.pkgs.pkgconfig
+
+    def decorate(cls):
+        cls = attributes.requires("requires_pkgconf")(cls)
+        cls.requires_pkgconf = ["pkgconf"]
+        return cls
+
+    return decorate

jolt/plugins/podman.py

@@ -1,4 +1,4 @@
-from jolt import Download, Parameter, Resource, Task
+from jolt import Parameter, Resource, Task
 from jolt.error import raise_task_error_if
 from jolt.tasks import TaskRegistry
 from jolt import attributes
@@ -15,7 +15,6 @@ from jolt.cache import ArtifactAttributeSetProvider
 import contextlib
 import json
 from os import path
-from platform import system
 import tarfile
 
 
@@ -100,35 +99,6 @@ class PodmanAttributeProvider(ArtifactAttributeSetProvider):
         artifact.podman.unapply(task, artifact)
 
 
-class PodmanClient(Download):
-    """ Task: Downloads and publishes the Podman command line client.
-
-    The task will be automatically made available after importing
-    ``jolt.plugins.podman``.
-    """
-
-    name = "podman/cli"
-    """ Name of the task """
-
-    arch = Parameter("x86_64", help="Host architecture")
-    """ Host architecture [x86_64] """
-
-    collect = ["podman/podman"]
-
-    host = Parameter(system().lower(), help="Host operating system")
-    """ Host operating system [autodetected] """
-
-    url = "https://download.podman.com/{host}/static/stable/{arch}/podman-{version}.tgz"
-    """ URL of binaries """
-
-    version = Parameter("20.10.13", help="Podman version")
-    """ Podman version [20.10.13] """
-
-    def publish(self, artifact, tools):
-        super().publish(artifact, tools)
-        artifact.environ.PATH.append("podman")
-
-
 @attributes.requires("_image")
 class Container(Resource):
     """
@@ -206,8 +176,8 @@ class Container(Resource):
     """
 
     volumes_default = [
-        "{joltdir}:{joltdir}",
-        "{joltcachedir}:{joltcachedir}",
+        "{joltdir}",
+        "{joltcachedir}",
     ]
     """
     A list of default volumes to mount.
@@ -247,7 +217,7 @@ class Container(Resource):
     def _image(self):
         registry = TaskRegistry.get()
         tool = tools.Tools(self)
-        if registry.get_task_class(tool.expand(self.image)):
+        if registry.has_task(tool.expand(self.image)):
             return [self.image]
         return []
 
@@ -284,7 +254,7 @@ class Container(Resource):
 
     @property
     def _volumes(self):
-        return " ".join([utils.option("-v ", self.tools.expand(vol))
+        return " ".join([utils.option("-v ", self.tools.expand_path(vol))
                          for vol in self.volumes_default + self.volumes])
 
     def acquire(self, artifact, deps, tools, owner):
@@ -344,8 +314,6 @@ class PodmanLogin(Resource):
     name = "podman/login"
     """ Name of the resource """
 
-    requires = ["podman/cli"]
-
     user = Parameter("", help="Podman Registry username")
     """
     Podman Registry username.
@@ -385,7 +353,6 @@ class PodmanLogin(Resource):
         tools.run("podman logout {server}")
 
 
-TaskRegistry.get().add_task_class(PodmanClient)
 TaskRegistry.get().add_task_class(PodmanLogin)
 
 
@@ -445,7 +412,6 @@ class ContainerImage(Task):
 
     Optionally add requirements to:
 
-      - ``podman/cli`` to provision the Podman client, if none is available on the host.
       - ``podman/login`` to automatically login to the Podman registry.
 
     This class must be subclassed.
@@ -468,7 +434,6 @@ class ContainerImage(Task):
         class Busybox(ContainerImage):
             \"\"\" Publishes Busybox image as gzip-compressed tarball \"\"\"
             compression = "gz"
-            requires = ["podman/cli"]
             tags = ["busybox:{identity}"]
 
     """
@@ -518,6 +483,7 @@ class ContainerImage(Task):
         - custom
         - directory
         - docker-archive
+        - ext4
         - oci-archive
         - oci-directory
         - squashfs
@@ -551,6 +517,9 @@ class ContainerImage(Task):
     The ``podman/login`` Jolt resource can be used for that purpose.
     """
 
+    size = None
+    """ Size of the image, e.g. "64M" (for certain output formats). """
+
     squash = False
     """ Squash image layers """
 
@@ -629,7 +598,7 @@ class ContainerImage(Task):
                         tools.run("podman image save --format={output} {} -o {}", self.tags[0], "image.tar")
                     if output == "oci-directory":
                         tools.run("podman image save --format=oci-dir {} -o {}", self.tags[0], "image.dir")
-                    if output in ["archive", "cpio", "custom", "directory", "squashfs"]:
+                    if output in ["archive", "cpio", "custom", "directory", "ext4", "squashfs"]:
                         ctr = tools.run("podman create {}", self.tags[0])
                         try:
                             with tools.runprefix("podman unshare "):
@@ -641,6 +610,9 @@ class ContainerImage(Task):
                                 elif output == "cpio":
                                     with tools.cwd(mount_path):
                                         tools.run("find | podman unshare cpio -o -F {}/image.cpio -H newc", outdir, output_on_error=True)
+                                elif output == "ext4":
+                                    assert self.size, "Size must be set for ext4 output"
+                                    tools.run("mke2fs -t ext4 -F -L rootfs -d {} image.ext4 {size}", mount_path, output_on_error=True)
                                 elif output == "squashfs":
                                     tools.run("mksquashfs {} image.squashfs", mount_path, output_on_error=True)
                                 else:
@@ -695,6 +667,8 @@ class ContainerImage(Task):
                     self.publish_custom(artifact, tools)
                 if output in ["directory"]:
                     artifact.paths.rootfs = output
+                if output in ["ext4"]:
+                    artifact.collect("image.ext4", output + "/{_imagefile}.ext4")
                 if output in ["squashfs"]:
                     artifact.collect("image.squashfs", output + "/{_imagefile}.squashfs")
 

jolt/plugins/python.py

@@ -1,9 +1,12 @@
 import sys
 
+from jolt import Task
+from jolt import attributes
 from jolt import filesystem as fs
 from jolt.cache import ArtifactStringAttribute
 from jolt.cache import ArtifactAttributeSet
 from jolt.cache import ArtifactAttributeSetProvider
+from jolt.error import raise_task_error_if
 
 
 class PythonVariable(ArtifactStringAttribute):
@@ -98,3 +101,137 @@ class PythonProvider(ArtifactAttributeSetProvider):
 
     def unapply(self, task, artifact):
         artifact.python.unapply(task, artifact)
+
+
+@attributes.system
+@attributes.requires("requires_python")
+class PythonEnv(Task):
+    """
+    Base class for Python virtual environment tasks.
+
+    Builds a Python virtual environment and installs specified packages.
+
+    The venv module from the Python standard library must be available in the
+    Python installation used to run the task.
+    """
+
+    abstract = True
+    """ This is an abstract base class that should be inherited by concrete tasks. """
+
+    executable = "python3"
+    """ Python executable to use for creating the virtual environment. """
+
+    requirements = []
+    """
+    List of Python packages to install in the virtual environment.
+
+    Each entry should be a string suitable for pip, e.g., "package==version".
+    """
+
+    def _verify_influence(self, deps, artifact, tools, sources=None):
+        # No influence to verify
+        return
+
+    def relocate_scripts(self, artifact, tools, frompath, topath):
+        bindir = "Scripts" if self.system == "windows" else "bin"
+
+        with tools.cwd(artifact.path, bindir):
+            for script in tools.glob("*"):
+                if script.startswith("python"):
+                    continue
+                tools.replace_in_file(script, frompath, topath)
+
+        with tools.cwd(artifact.path):
+            if not tools.exists("local/bin"):
+                return
+            with tools.cwd("local", "bin"):
+                for script in tools.glob("*"):
+                    tools.replace_in_file(script, frompath, topath)
+
+    def publish(self, artifact, tools):
+        # Create a parallel installation by copying a Python installation
+
+        # First locate the Python executable to copy
+        py_exe = tools.which(self.executable)
+        raise_task_error_if(
+            py_exe is None, self,
+            f"Python executable '{self.executable}' not found in PATH.",
+        )
+
+        # Follow symlinks to get the real executable
+        py_exe = fs.path.realpath(py_exe)
+
+        # Determine the Python home directory
+        py_home = fs.path.dirname(fs.path.dirname(py_exe))
+
+        # Determine the Python version
+        self.version_major = tools.run(
+            [py_exe, "-c", "import sys; print(\"{{}}.{{}}\".format(sys.version_info[0], sys.version_info[1]))"],
+            shell=False,
+            output_on_error=True).strip()
+
+        self.info("Python executable: {0}", py_exe)
+        self.info("Python home: {0}", py_home)
+        self.info("Python version: {0}", self.version_major)
+
+        # Copy the Python installation to the artifact path
+        with tools.cwd(py_home):
+            artifact.collect(py_exe, "bin/python3")
+            artifact.collect("lib/python3")
+            artifact.collect("lib/python{version_major}")
+            artifact.collect("lib/libpython{version_major}.*")
+
+        # Create common symlinks
+        if self.system != "windows":
+            with tools.cwd(artifact.path, "bin"):
+                tools.symlink("python3", "python")
+                tools.symlink("python3", "python{version_major}")
+
+        # Install required packages into the artifact using pip
+        with tools.environ(PYTHONHOME=artifact.path):
+            py_exe = fs.path.join(artifact.path, "bin", "python3")
+            with tools.tmpdir() as tmp, tools.cwd(tmp):
+                tools.write_file(
+                    "requirements.txt",
+                    "\n".join(self.requirements) + "\n"
+                )
+
+                pip_cmd = [
+                    py_exe,
+                    "-m",
+                    "pip",
+                    "--isolated",
+                    "--no-cache-dir",
+                    "install",
+                    "-r",
+                    "requirements.txt",
+                    "--break-system-packages",
+                ]
+                tools.run(pip_cmd, shell=False)
+
+        artifact.environ.PATH.append("bin")
+        artifact.environ.PATH.append("local/bin")
+        artifact.strings.install_prefix = artifact.path
+
+    def unpack(self, artifact, tools):
+        # Relocate the virtual environment by adjusting script paths
+        frompath = artifact.strings.install_prefix
+        topath = artifact.final_path
+        self.relocate_scripts(artifact, tools, frompath, topath)
+
+        artifact.strings.install_prefix = artifact.final_path
+
+
+def requires(python=True):
+    """ Decorator to add Python requirements to a task. """
+
+    import jolt.pkgs.cpython
+
+    def decorate(cls):
+        if python:
+            cls = attributes.requires("requires_python")(cls)
+            cls.requires_python = ["cpython"]
+
+        return cls
+
+    return decorate

jolt/plugins/rust.py

@@ -0,0 +1,25 @@
+from jolt import attributes, Task
+
+
+@attributes.common_metadata()
+class Rust(Task):
+    """ Base class for Rust-based build tasks. """
+
+    abstract = True
+
+    srcdir = None
+    """
+    Source directory for the Rust project.
+
+    If None, defaults to the task work directory (joltdir).
+    """
+
+    def run(self, deps, tools):
+        self.builddir = tools.builddir(incremental=True)
+        self.installdir = tools.builddir("install")
+        with tools.cwd(self.srcdir or self.joltdir):
+            tools.run("cargo install --path . --target-dir={builddir} --root={installdir}")
+
+    def publish(self, artifact, tools):
+        with tools.cwd(self.installdir):
+            artifact.collect("*", symlinks=True)

jolt/plugins/scheduler.py

@@ -13,7 +13,6 @@ from jolt import config
 from jolt import hooks
 from jolt import loader
 from jolt import log
-from jolt import manifest
 from jolt import common_pb2 as common_pb
 from jolt import scheduler
 from jolt import utils
@@ -145,6 +144,15 @@ class RemoteExecutor(NetworkExecutor):
         self.session = session
         self.task = task
 
+    def schedule(self, env):
+        """
+        Schedule the task for execution.
+
+        The task is marked as in progress before scheduling.
+        """
+        self.task.set_in_progress()
+        return super().schedule(env)
+
     def cancel(self):
         """
         Cancel the build session.
@@ -436,21 +444,11 @@ class RemoteSession(object):
         if self.build:
             return
 
-        registry = ExecutorRegistry.get()
-
         if not self.buildenv:
-            # Create a list of parameters to send to the scheduler.
-            parameters = []
-            for key, value in registry.get_network_parameters(None).items():
-                parameters.append(common_pb.Property(key=key, value=value))
-
-            # Add parameters from the config / command line (-c params.key).
-            parameters.extend(config.export_params())
-
             # Create the build environment.
             self.buildenv = common_pb.BuildEnvironment(
                 client=selfdeploy.get_client(),
-                parameters=parameters,
+                parameters=config.export_params(),
                 task_default_parameters=scheduler.export_task_default_params(self.tasks),
                 tasks=scheduler.export_tasks(self.tasks + self.pruned),
                 workspace=loader.export_workspace(self.tasks),
@@ -580,8 +578,14 @@ def executor(ctx, worker, build, request):
     loglevel = request.environment.loglevel
     log.set_level_pb(loglevel)
 
-    # Import protobuf build description
-    manifest.ManifestExtensionRegistry.import_protobuf(request.environment)
+    # Import workspace
+    loader.import_workspace(request.environment)
+
+    # Import configuration snippet
+    config.import_config(request.environment.config)
+
+    # Import configuration parameters (-c params.key)
+    config.import_params({param.key: param.value for param in request.environment.parameters})
 
     options = JoltOptions(
         network=True,

jolt/plugins/selfdeploy/setup.py

@@ -85,6 +85,7 @@ setup(
         "protobuf",
         "psutil",
         "pygit2",
+        "py7zr",
         "requests",
         "zstandard",
         "tqdm",
@@ -92,7 +93,7 @@ setup(
     dependency_links=[],
     extras_require={
         "allure": ["allure-python-commons"],
-        "conan": ["conan<2.0"],
+        "conan": ["conan>=2.0"],
         "dev": ["check-manifest"],
         "doc": ["sphinx-click", "sphinx-rtd-theme"],
         "test": ["coverage"],