jolt 0.9.342__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/plugins/cmake.py

@@ -1,3 +1,4 @@
+from jolt import attributes
 from jolt import influence
 from jolt import Task
 from jolt import utils
@@ -6,31 +7,98 @@ from jolt.error import raise_task_error_if
 import os
 
 
+def options(attrib):
+    """
+    Decorates a CMake task with an alternative ``options`` attribute.
+
+    The new attribute will be concatenated with the regular
+    ``options`` attribute.
+
+    Args:
+        attrib (str): Name of alternative attribute.
+            Keywords are expanded.
+    """
+    return utils.concat_attributes("options", attrib)
+
+
+def requires(version=None):
+    """ Decorator to add CMake requirements to a task. """
+
+    import jolt.pkgs.cmake
+
+    def decorate(cls):
+        cls = attributes.requires("requires_cmake")(cls)
+        cls.requires_cmake = ["cmake" + (f":version={version}" if version else "")]
+        return cls
+
+    return decorate
+
+
+def use_ninja():
+    """
+    Decorator to add Ninja dependencies to CMake task.
+
+    It also selects Ninja as the CMake generator and builder.
+    """
+
+    import jolt.pkgs.ninja
+
+    def decorate(cls):
+        cls = attributes.requires("requires_ninja")(cls)
+        cls.generator = "Ninja"
+        cls.requires_ninja = ["ninja"]
+        return cls
+
+    return decorate
+
+
+@attributes.common_metadata()
+@options("options")
 class CMake(Task):
     """ Builds and publishes a CMake project """
-
     abstract = True
 
     cmakelists = "CMakeLists.txt"
     """ Path to CMakeLists.txt or directory containing CMakelists.txt """
 
+    config = "Release"
+    """ The default build configuration to use """
+
     generator = None
+    """ The build file generator that CMake should use """
 
     incremental = True
+    """ Keep build directories between Jolt invocations """
 
     options = []
     """ List of options and their values (``option[:type]=value``) """
 
+    srcdir = None
+    """ Source directory. If not specified, the task working directory is used. """
+
+    install = ["install"]
+    """ List of install build targets. If empty, the default install target is used. """
+
+    def clean(self, tools):
+        cmake = tools.cmake(incremental=self.incremental)
+        cmake.clean()
+
     def run(self, deps, tools):
+        self.deps = deps
         raise_task_error_if(not self.cmakelists, self, "cmakelists attribute has not been defined")
 
-        cmake = tools.cmake(incremental=self.incremental)
-        cmake.configure(tools.expand(self.cmakelists), *["-D" + tools.expand(option) for option in self.options], generator=self.generator)
-        cmake.build()
-        cmake.install()
+        config = tools.expand(str(self.config))
+        options = self._options()
+        options += ["CMAKE_BUILD_TYPE=" + config]
+
+        with tools.cwd(self.srcdir or self.joltdir):
+            cmake = tools.cmake(deps, incremental=self.incremental)
+            cmake.configure(tools.expand(self.cmakelists), *["-D" + tools.expand(option) for option in options], generator=self.generator)
+            for target in self.install or ["install"]:
+                cmake.install(config=config, target=target)
 
     def publish(self, artifact, tools):
-        cmake = tools.cmake()
+        cmake = tools.cmake(incremental=self.incremental)
         cmake.publish(artifact)
 
 

jolt/plugins/conan.py

@@ -3,6 +3,7 @@ from os import path
 
 from jolt import Task
 from jolt import influence
+from jolt import utils
 from jolt.error import raise_task_error_if
 
 
@@ -239,3 +240,240 @@ class Conan(Task):
             artifact.cxxinfo.cflags.append(dep["cflags"])
             artifact.cxxinfo.cxxflags.append(dep["cxxflags"])
             artifact.cxxinfo.ldflags.append(dep["exelinkflags"])
+
+
+@influence.attribute("conanfile")
+@influence.attribute("generators")
+@influence.attribute("incremental")
+@influence.attribute("options")
+@influence.attribute("packages")
+class Conan2(Task):
+    """
+    Conan package installer task.
+
+    This task base class can be used to fetch, build and publish Conan packages
+    as Jolt artifacts. All package metadata is transfered from the Conan package
+    manifest to the Jolt artifact so that no manual configuration of include
+    paths, library paths, macros, etc is required.
+
+    An existing installation of Conan is required. Please visit https://conan.io/
+    for installation instructions and documentation.
+
+    A minimal task to download and publish the Boost C++ libraries can look like this:
+
+    .. code-block:: python
+
+        from jolt.plugins.conan import Conan
+
+        class Boost(Conan):
+            packages = ["boost/1.74.0"]
+
+    Boost may then be used from Ninja tasks by declaring a requirement:
+
+    .. code-block:: python
+
+        from jolt.plugins.ninja import CXXExecutable
+
+        class BoostApplication(CXXExecutable):
+            requires = ["boost"]
+            sources = ["main.cpp"]
+
+    The task supports using an existing conanfile.txt, but it is not required.
+    Packages are installed into and collected from Jolt build directories. The
+    user's regular Conan cache will not be affected.
+
+    """
+
+    abstract = True
+
+    conanfile = None
+    """
+    An existing conanfile.txt file to use.
+
+    Instead of generating the conanfile.txt file on-demand, an external
+    file may be used. If this attribute is set, the ``generators``, ``options``
+    and ``packages`` attributes must not be set.
+
+    See Conan documentation for further details.
+    """
+
+    packages = []
+    """
+    A list of Conan package references to collect and publish.
+
+    The reference format is ``PkgName/<version>@user/channel``. See Conan
+    documentation for further details.
+
+    Any {keyword} arguments, or macros, found in the strings are automatically
+    expanded to the value of the associated task's parameters and properties.
+
+    Example:
+
+    .. code-block:: python
+
+        sdl_version = Parameter("2.0.12")
+
+        packages = [
+            "boost/1.74.0",
+            "sdl2/{sdl_version}@bincrafters/stable",
+        ]
+
+    """
+
+    options = []
+    """
+    A list of Conan package options to apply
+
+    The option format is ``PkgName:Option=Value``. See Conan
+    documentation for further details.
+
+    Any {keyword} arguments, or macros, found in the strings are automatically
+    expanded to the value of the associated task's parameters and properties.
+
+    Example:
+
+    .. code-block:: python
+
+        options = [
+            "boost:shared=True",
+            "zlib:shared=True",
+        ]
+
+    """
+
+    settings = []
+    """
+    A list of Conan settings to apply
+
+    The settings format is ``Option=Value``. See Conan
+    documentation for further details.
+
+    Any {keyword} arguments, or macros, found in the strings are automatically
+    expanded to the value of the associated task's parameters and properties.
+
+    Example:
+
+    .. code-block:: python
+
+        settings = [
+            "compiler.libcxx=libstdc++11",
+        ]
+
+    """
+
+    remotes = {}
+    """
+    A dictionary with Conan remotes to use when fetching packages.
+
+    The dictionary key is the name of remote and its value is the URL.
+
+    Example:
+
+    .. code-block:: python
+
+        remotes = {
+            "bincrafters": "https://api.bintray.com/conan/bincrafters/public-conan"
+        }
+
+    """
+
+    incremental = True
+    """
+    Keep installed packages in the Conan cache between Jolt invokations.
+
+    If incremental build is disabled, the Jolt Conan cache is removed
+    before execution begins.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.conanfile:
+            self.influence.append(influence.FileInfluence(self.conanfile))
+
+    def _options(self):
+        return [] + self.options
+
+    def _settings(self):
+        return [] + self.settings
+
+    def _packages(self):
+        return [] + self.packages
+
+    def _remotes(self):
+        return self.remotes
+
+    def run(self, deps, tools):
+        raise_task_error_if(
+            not tools.which("conan"), self,
+            "Conan: Conan is not installed in the PATH")
+        raise_task_error_if(
+            self.conanfile and (self._packages() or self._options()), self,
+            "Conan: 'conanfile' attribute cannot be used with other attributes")
+
+        conanfile = tools.expand_path(self.conanfile) if self.conanfile else None
+
+        with tools.cwd(tools.builddir()):
+            if conanfile is None or not path.exists(conanfile):
+                conanfile = "conanfile.txt"
+                self.info("Creating conanfile.txt")
+                self.tools.write_file(conanfile, "[requires]\n")
+                for pkg in self._packages():
+                    self.tools.append_file(conanfile, pkg + "\n")
+
+            with tools.environ(CONAN_USER_HOME=tools.builddir("conan", incremental=self.incremental)):
+                for remote, url in self._remotes().items():
+                    self.info("Registering remote '{}'", remote)
+                    tools.run("conan remote add -f {} {}", remote, url, output_on_error=True)
+
+                self.info("Installing packages into the Conan cache")
+                options = " ".join(["-o " + opt for opt in self._options()])
+                settings = " ".join(["-s " + opt for opt in self._settings()])
+                output = tools.run("conan install --build=missing --output-folder . -u --format=json {} {} {}", options, settings, conanfile, output_stdout=False)
+
+            self.info("Parsing manifest")
+            self._manifest = json.loads(output)
+
+            for dep in self._manifest["graph"]["nodes"].values():
+                if dep["package_folder"]:
+                    self.info("Collecting '{}' files from: {}", dep["name"], dep["package_folder"])
+                    tools.copy(dep["package_folder"], dep["name"])
+
+    def publish(self, artifact, tools):
+        self.info("Publishing package files")
+        with tools.cwd(tools.builddir()):
+            artifact.collect("*")
+
+        self.info("Publishing metadata")
+        for dep in self._manifest["graph"]["nodes"].values():
+            if not dep["package_folder"]:
+                continue
+
+            for node in dep["cpp_info"]:
+                for incpath in dep["cpp_info"][node]["includedirs"]:
+                    artifact.cxxinfo.incpaths.append(path.join(dep["name"], path.relpath(incpath, dep["package_folder"])))
+                for libpath in dep["cpp_info"][node]["libdirs"]:
+                    artifact.cxxinfo.libpaths.append(path.join(dep["name"], path.relpath(libpath, dep["package_folder"])))
+                for binpath in dep["cpp_info"][node]["bindirs"]:
+                    artifact.environ.PATH.append(path.join(dep["name"], path.relpath(binpath, dep["package_folder"])))
+                if dep["cpp_info"][node]["libs"]:
+                    artifact.cxxinfo.libraries.extend(dep["cpp_info"][node]["libs"])
+                if dep["cpp_info"][node]["system_libs"]:
+                    artifact.cxxinfo.libraries.extend(dep["cpp_info"][node]["system_libs"])
+                if dep["cpp_info"][node]["defines"]:
+                    artifact.cxxinfo.macros.extend(dep["cpp_info"][node]["defines"])
+                if dep["cpp_info"][node]["cflags"]:
+                    artifact.cxxinfo.cflags.extend(dep["cpp_info"][node]["cflags"])
+                if dep["cpp_info"][node]["cxxflags"]:
+                    artifact.cxxinfo.cxxflags.extend(dep["cpp_info"][node]["cxxflags"])
+                if dep["cpp_info"][node]["exelinkflags"]:
+                    artifact.cxxinfo.ldflags.extend(dep["cpp_info"][node]["exelinkflags"])
+
+        # Make list of unique values
+        artifact.cxxinfo.incpaths = utils.unique_list(artifact.cxxinfo.incpaths)
+        artifact.cxxinfo.libpaths = utils.unique_list(artifact.cxxinfo.libpaths)
+        artifact.cxxinfo.libraries = utils.unique_list(artifact.cxxinfo.libraries)
+        artifact.cxxinfo.macros = utils.unique_list(artifact.cxxinfo.macros)
+        artifact.cxxinfo.cflags = utils.unique_list(artifact.cxxinfo.cflags)
+        artifact.cxxinfo.cxxflags = utils.unique_list(artifact.cxxinfo.cxxflags)
+        artifact.cxxinfo.ldflags = utils.unique_list(artifact.cxxinfo.ldflags)
+        artifact.environ.PATH = path.pathsep.join(utils.unique_list(str(artifact.environ.PATH).split(path.pathsep)))

jolt/plugins/cxxinfo.py

@@ -1,4 +1,5 @@
 from jolt.cache import ArtifactListAttribute
+from jolt.cache import ArtifactStringAttribute
 from jolt.cache import ArtifactAttributeSet
 from jolt.cache import ArtifactAttributeSetProvider
 
@@ -7,6 +8,10 @@ class CppInfoListVariable(ArtifactListAttribute):
     pass
 
 
+class CppInfoStringVariable(ArtifactStringAttribute):
+    pass
+
+
 class CppInfoDictVariable(ArtifactListAttribute):
     def __init__(self, artifact, name):
         super(CppInfoDictVariable, self).__init__(artifact, name)
@@ -26,6 +31,8 @@ class CppInfo(ArtifactAttributeSet):
             return CppInfoListVariable(self._artifact, "asflags")
         if name == "cflags":
             return CppInfoListVariable(self._artifact, "cflags")
+        if name == "crt":
+            return CppInfoStringVariable(self._artifact, "crt")
         if name == "cxxflags":
             return CppInfoListVariable(self._artifact, "cxxflags")
         if name == "incpaths":

jolt/plugins/docker.py

@@ -103,8 +103,8 @@ class DockerClient(Download):
     url = "https://download.docker.com/{host}/static/stable/{arch}/docker-{version}.tgz"
     """ URL of binaries """
 
-    version = Parameter("27.3.1", help="Docker version")
-    """ Docker version [27.3.1] """
+    version = Parameter("27.4.1", help="Docker version")
+    """ Docker version [27.4.1] """
 
     def publish(self, artifact, tools):
         super().publish(artifact, tools)
@@ -254,7 +254,7 @@ class DockerContainer(Resource):
     def _image(self):
         registry = TaskRegistry.get()
         tool = tools.Tools(self)
-        if registry.get_task_class(tool.expand(self.image)):
+        if registry.has_task(tool.expand(self.image)):
             return [self.image]
         return []
 

jolt/plugins/environ.py

@@ -55,6 +55,17 @@ class PathEnvironmentVariable(EnvironmentVariable):
 
 
 class EnvironmentVariableSet(ArtifactAttributeSet):
+    """ A set of environment variables for an artifact.
+
+    Example:
+
+    .. code-block:: python
+
+        artifact.environ.FOO = "bar"
+        artifact.environ.PATH.append("bin")
+
+    """
+
     def __init__(self, artifact):
         super(EnvironmentVariableSet, self).__init__()
         super(ArtifactAttributeSet, self).__setattr__("_artifact", artifact)

jolt/plugins/fetch.py

@@ -0,0 +1,141 @@
+import hashlib
+from jolt import BooleanParameter, Parameter
+from jolt.plugins.git import ErrorDict
+from jolt.tasks import Resource, TaskRegistry
+from jolt import filesystem as fs
+from jolt import utils
+from jolt.error import raise_error_if
+from jolt.loader import JoltLoader
+from jolt.tools import SUPPORTED_ARCHIVE_TYPES
+
+
+class Fetch(Resource):
+    """
+    Fetch a file from a URL and extract it.
+
+    By default, the fetched file is extracted if it is a supported archive type.
+    Extraction may be disable by assigning the `extract` parameter to false.
+
+    The fetched content is placed in a build directory named after the resource.
+    The `path` parameter can be used to specify a different location relative to
+    the workspace root.
+
+    The path of the fetched content is made available to consuming tasks through
+    the `fetch` attribute. The `fetch` attribute is a dictionary where the key
+    is the name of the fetched file and the value is a path relative to the
+    workspace root. You can specify an alias for the filename / key through the
+    `alias` parameter.
+
+    The plugin must be loaded before it can be used. This is done by importing
+    the module, or by adding the following line to the configuration file:
+
+    .. code-block:: ini
+
+            [fetch]
+
+
+    Example:
+
+    .. code-block:: python
+
+            from jolt.plugins import fetch
+
+            class Example(Task):
+                requires = ["fetch:alias=zlib,url=https://zlib.net/zlib-1.3.1.tar.gz"]
+
+                def run(self, deps, tools):
+                    self.info("The source tree is located at: {fetch[zlib]}")
+                    with tools.cwd(self.fetch["zlib"]):
+                        tools.run("make")
+
+            """
+
+    name = "fetch"
+    alias = Parameter(required=False, help="Name of the task used when referencing content. Defaults to the filename of the fetched file.")
+    extract = BooleanParameter(default=True, help="Whether to extract the fetched file.")
+    path = Parameter(required=False, help="Destination directory.")
+    url = Parameter(help="URL to fetch from.")
+    md5 = Parameter(required=False, help="Expected MD5 hash of the fetched file.")
+    sha256 = Parameter(required=False, help="Expected SHA256 hash of the fetched file.")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.joltdir = JoltLoader.get().joltdir
+
+        # Set the path to the extraction directory
+        if self.path.is_unset():
+            self.abspath = self.tools.builddir(utils.canonical(self.short_qualified_name), incremental="always", unique=False)
+            if not self._extract():
+                # Join with filename if not extracting
+                self.abspath = fs.path.join(self.abspath, self._get_filename())
+        else:
+            self.abspath = fs.path.join(self.joltdir, str(self.path) or self._get_name())
+
+        self.relpath = fs.path.relpath(self.abspath, self.tools.wsroot)
+
+    def _extract(self):
+        """ Check if the fetched file should/can be extracted. """
+        filename = self._get_filename()
+
+        if not any([filename.endswith(ext) for ext in SUPPORTED_ARCHIVE_TYPES]):
+            return False
+
+        return bool(self.extract)
+
+    def _acquire_ws(self):
+        # Create the destination directory if it does not exist
+        self.tools.rmtree(self.abspath, ignore_errors=True)
+        self.tools.mkdir(fs.path.dirname(self.abspath), recursively=True)
+
+        if self._extract():
+            with self.tools.tmpdir() as tmpdir, self.tools.cwd(tmpdir):
+                filename = self._get_filename()
+                self.tools.download(self.url, filename)
+                self._verify_sha256(filename)
+                self.tools.extract(filename, self.abspath)
+        else:
+            self.tools.download(self.url, self.abspath)
+            self._verify_sha256(self.abspath)
+
+    def acquire(self, artifact, deps, tools, owner):
+        self._acquire_ws()
+        self._assign_fetch(owner)
+        artifact.worktree = fs.path.relpath(self.abspath, owner.joltdir)
+
+    def _assign_fetch(self, task, none=False):
+        if not hasattr(task, "fetch"):
+            task.fetch = ErrorDict(self)
+        if none:
+            # None means the git repo is not cloned or checked out
+            # and should not be included in the git dictionary
+            # of the consuming task yet. If the consuming task
+            # requires the git repo for its influence collection,
+            # the dict will raise an error. The solution is to
+            # assign hash=true to the git requirement which
+            # will cause the git repo to be cloned and checked out
+            # before the influence collection is performed.
+            task.fetch[self._get_name()] = None
+        else:
+            # Assign the git repo to the consuming task.
+            # The git repo is cloned and checked out before
+            # any influence collection is performed.
+            task.fetch[self._get_name()] = fs.path.relpath(self.abspath, task.joltdir)
+
+    def _get_name(self):
+        return str(self.alias) if self.alias.is_set() else self._get_filename()
+
+    def _get_filename(self):
+        return fs.path.basename(str(self.url))
+
+    def _verify_sha256(self, filepath):
+        if not self.sha256.is_set():
+            return
+        actual_hash = self.tools.checksum_file(filepath, hashfn=hashlib.sha256)
+        expected_hash = str(self.sha256)
+        raise_error_if(
+            actual_hash != expected_hash,
+            f"SHA256 hash mismatch for fetched file '{filepath}': expected {expected_hash}, got {actual_hash}"
+        )
+
+
+TaskRegistry.get().add_task_class(Fetch)

jolt/plugins/gdb.py

@@ -70,31 +70,34 @@ def gdb(ctx, task, default, machine_interface, no_binary, gdb_args):
     if machine_interface:
         log.enable_gdb()
 
-    manifest = ctx.obj["manifest"]
     options = JoltOptions(default=default)
     acache = cache.ArtifactCache.get(options)
     TaskHookRegistry.get(options)
     executors = scheduler.ExecutorRegistry.get(options)
     registry = TaskRegistry.get()
     strategy = scheduler.DownloadStrategy(executors, acache)
-    queue = scheduler.TaskQueue(strategy, acache, {})
+    queue = scheduler.TaskQueue()
 
     for params in default:
         registry.set_default_parameters(params)
 
-    gb = graph.GraphBuilder(registry, acache, manifest, options, progress=True)
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
     dag = gb.build([task])
 
     try:
         with log.progress("Progress", dag.number_of_tasks(), " tasks", estimates=False, debug=False) as p:
+            in_progress = set()
+
             while dag.has_tasks() or not queue.empty():
-                leafs = dag.select(lambda graph, task: task.is_ready())
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(task)
+                    executor = strategy.create_executor({}, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
 
-                task, error = queue.wait()
+                task, _ = queue.wait()
 
                 # Materialize workspace resources so that
                 # source code is available to the debugger.
@@ -113,6 +116,7 @@ def gdb(ctx, task, default, machine_interface, no_binary, gdb_args):
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()