jolt 0.9.76__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/plugins/cache.py

@@ -0,0 +1,133 @@
+from requests.exceptions import ConnectTimeout, RequestException
+from urllib.parse import urlparse, urlunparse
+
+from jolt import utils
+from jolt import cache
+from jolt import log
+from jolt import config
+from jolt import filesystem as fs
+from jolt import tools
+from jolt.error import raise_error_if, JoltError
+
+
+NAME = "cache"
+NAME_LOG = "Cache Service"
+TIMEOUT = (3.5, 27)
+TIMEOUT_HEAD = (27, 27)
+
+
+class Cache(cache.StorageProvider):
+    def __init__(self, cache):
+        super().__init__()
+        self._cache = cache
+        self._uri = config.get(NAME, "uri", "http://cache")
+        self._uri = self._uri.rstrip("/")
+        raise_error_if(not self._uri, "Cache Service URI not configured")
+        self._file_uri = self._uri + "/files"
+        self._upload = config.getboolean(NAME, "upload", True)
+        self._download = config.getboolean(NAME, "download", True)
+        self._disabled = False
+
+    def _get_path(self, artifact):
+        return artifact.tools.expand(
+            "{name}/{file}",
+            name=artifact.task.name,
+            file=fs.path.basename(artifact.get_archive_path()))
+
+    def _get_url(self, artifact):
+        return artifact.tools.expand(
+            "{uri}/{name}/{file}",
+            uri=self._file_uri,
+            name=artifact.task.name,
+            file=fs.path.basename(artifact.get_archive_path()))
+
+    @utils.retried.on_exception((RequestException, JoltError))
+    def download(self, artifact, force=False):
+        if self._disabled:
+            return False
+        if not self._download and not force:
+            return False
+        url = self._get_url(artifact)
+        return artifact.tools.download(
+            url,
+            artifact.get_archive_path(),
+            exceptions=False,
+            timeout=TIMEOUT)
+
+    def download_enabled(self):
+        return not self._disabled and self._download
+
+    @utils.retried.on_exception((RequestException))
+    def upload(self, artifact, force=False):
+        if self._disabled:
+            return True
+        if not self._upload and not force:
+            return True
+        url = self._get_url(artifact)
+        archive = artifact.get_archive()
+        return artifact.tools.upload(
+            archive, url,
+            exceptions=False,
+            timeout=TIMEOUT)
+
+    def upload_enabled(self):
+        return not self._disabled and self._upload
+
+    @utils.retried.on_exception((RequestException))
+    def availability(self, artifacts):
+        if self._disabled:
+            return [], artifacts
+
+        file_map = {self._get_path(artifact): artifact for artifact in artifacts}
+        try:
+            data = {"files": list(file_map.keys())}
+            response = tools.http_session.post(self._file_uri, json=data, stream=True, timeout=TIMEOUT_HEAD)
+        except ConnectTimeout:
+            self._disabled = True
+            log.warning(NAME_LOG + " failed to establish server connection, disabled")
+            return [], []
+
+        if response.status_code != 200:
+            log.debug(NAME_LOG + " POST ({}): {}", response.status_code, self._file_uri)
+            return [], artifacts
+
+        present = []
+        missing = []
+        data = response.json()
+
+        for file in data.get("present", []):
+            present.append(file_map[file])
+
+        for file in data.get("missing", []):
+            missing.append(file_map[file])
+
+        return present, missing
+
+    @utils.retried.on_exception((RequestException))
+    def location(self, artifact):
+        if self._disabled:
+            return False
+
+        url = self._get_url(artifact)
+        try:
+            response = tools.http_session.head(url, stream=True, timeout=TIMEOUT_HEAD)
+
+        except ConnectTimeout:
+            self._disabled = True
+            log.warning(NAME_LOG + " failed to establish server connection, disabled")
+            return False
+
+        if response.status_code != 200:
+            log.debug(NAME_LOG + " HEAD ({}): {}", response.status_code, url)
+            return None
+
+        url = urlunparse(urlparse(url))
+        return url
+
+
+@cache.RegisterStorage
+class HttpFactory(cache.StorageProviderFactory):
+    @staticmethod
+    def create(cache):
+        log.verbose(NAME_LOG + " Loaded")
+        return Cache(cache)

jolt/plugins/cmake.py

@@ -1,3 +1,4 @@
+from jolt import attributes
 from jolt import influence
 from jolt import Task
 from jolt import utils
@@ -6,31 +7,98 @@ from jolt.error import raise_task_error_if
 import os
 
 
+def options(attrib):
+    """
+    Decorates a CMake task with an alternative ``options`` attribute.
+
+    The new attribute will be concatenated with the regular
+    ``options`` attribute.
+
+    Args:
+        attrib (str): Name of alternative attribute.
+            Keywords are expanded.
+    """
+    return utils.concat_attributes("options", attrib)
+
+
+def requires(version=None):
+    """ Decorator to add CMake requirements to a task. """
+
+    import jolt.pkgs.cmake
+
+    def decorate(cls):
+        cls = attributes.requires("requires_cmake")(cls)
+        cls.requires_cmake = ["cmake" + (f":version={version}" if version else "")]
+        return cls
+
+    return decorate
+
+
+def use_ninja():
+    """
+    Decorator to add Ninja dependencies to CMake task.
+
+    It also selects Ninja as the CMake generator and builder.
+    """
+
+    import jolt.pkgs.ninja
+
+    def decorate(cls):
+        cls = attributes.requires("requires_ninja")(cls)
+        cls.generator = "Ninja"
+        cls.requires_ninja = ["ninja"]
+        return cls
+
+    return decorate
+
+
+@attributes.common_metadata()
+@options("options")
 class CMake(Task):
     """ Builds and publishes a CMake project """
-
     abstract = True
 
     cmakelists = "CMakeLists.txt"
     """ Path to CMakeLists.txt or directory containing CMakelists.txt """
 
+    config = "Release"
+    """ The default build configuration to use """
+
     generator = None
+    """ The build file generator that CMake should use """
 
     incremental = True
+    """ Keep build directories between Jolt invocations """
 
     options = []
     """ List of options and their values (``option[:type]=value``) """
 
+    srcdir = None
+    """ Source directory. If not specified, the task working directory is used. """
+
+    install = ["install"]
+    """ List of install build targets. If empty, the default install target is used. """
+
+    def clean(self, tools):
+        cmake = tools.cmake(incremental=self.incremental)
+        cmake.clean()
+
     def run(self, deps, tools):
+        self.deps = deps
         raise_task_error_if(not self.cmakelists, self, "cmakelists attribute has not been defined")
 
-        cmake = tools.cmake(incremental=self.incremental)
-        cmake.configure(tools.expand(self.cmakelists), *["-D" + tools.expand(option) for option in self.options], generator=self.generator)
-        cmake.build()
-        cmake.install()
+        config = tools.expand(str(self.config))
+        options = self._options()
+        options += ["CMAKE_BUILD_TYPE=" + config]
+
+        with tools.cwd(self.srcdir or self.joltdir):
+            cmake = tools.cmake(deps, incremental=self.incremental)
+            cmake.configure(tools.expand(self.cmakelists), *["-D" + tools.expand(option) for option in options], generator=self.generator)
+            for target in self.install or ["install"]:
+                cmake.install(config=config, target=target)
 
     def publish(self, artifact, tools):
-        cmake = tools.cmake()
+        cmake = tools.cmake(incremental=self.incremental)
         cmake.publish(artifact)
 
 

jolt/plugins/conan.py

@@ -3,6 +3,7 @@ from os import path
 
 from jolt import Task
 from jolt import influence
+from jolt import utils
 from jolt.error import raise_task_error_if
 
 
@@ -239,3 +240,240 @@ class Conan(Task):
             artifact.cxxinfo.cflags.append(dep["cflags"])
             artifact.cxxinfo.cxxflags.append(dep["cxxflags"])
             artifact.cxxinfo.ldflags.append(dep["exelinkflags"])
+
+
+@influence.attribute("conanfile")
+@influence.attribute("generators")
+@influence.attribute("incremental")
+@influence.attribute("options")
+@influence.attribute("packages")
+class Conan2(Task):
+    """
+    Conan package installer task.
+
+    This task base class can be used to fetch, build and publish Conan packages
+    as Jolt artifacts. All package metadata is transfered from the Conan package
+    manifest to the Jolt artifact so that no manual configuration of include
+    paths, library paths, macros, etc is required.
+
+    An existing installation of Conan is required. Please visit https://conan.io/
+    for installation instructions and documentation.
+
+    A minimal task to download and publish the Boost C++ libraries can look like this:
+
+    .. code-block:: python
+
+        from jolt.plugins.conan import Conan
+
+        class Boost(Conan):
+            packages = ["boost/1.74.0"]
+
+    Boost may then be used from Ninja tasks by declaring a requirement:
+
+    .. code-block:: python
+
+        from jolt.plugins.ninja import CXXExecutable
+
+        class BoostApplication(CXXExecutable):
+            requires = ["boost"]
+            sources = ["main.cpp"]
+
+    The task supports using an existing conanfile.txt, but it is not required.
+    Packages are installed into and collected from Jolt build directories. The
+    user's regular Conan cache will not be affected.
+
+    """
+
+    abstract = True
+
+    conanfile = None
+    """
+    An existing conanfile.txt file to use.
+
+    Instead of generating the conanfile.txt file on-demand, an external
+    file may be used. If this attribute is set, the ``generators``, ``options``
+    and ``packages`` attributes must not be set.
+
+    See Conan documentation for further details.
+    """
+
+    packages = []
+    """
+    A list of Conan package references to collect and publish.
+
+    The reference format is ``PkgName/<version>@user/channel``. See Conan
+    documentation for further details.
+
+    Any {keyword} arguments, or macros, found in the strings are automatically
+    expanded to the value of the associated task's parameters and properties.
+
+    Example:
+
+    .. code-block:: python
+
+        sdl_version = Parameter("2.0.12")
+
+        packages = [
+            "boost/1.74.0",
+            "sdl2/{sdl_version}@bincrafters/stable",
+        ]
+
+    """
+
+    options = []
+    """
+    A list of Conan package options to apply
+
+    The option format is ``PkgName:Option=Value``. See Conan
+    documentation for further details.
+
+    Any {keyword} arguments, or macros, found in the strings are automatically
+    expanded to the value of the associated task's parameters and properties.
+
+    Example:
+
+    .. code-block:: python
+
+        options = [
+            "boost:shared=True",
+            "zlib:shared=True",
+        ]
+
+    """
+
+    settings = []
+    """
+    A list of Conan settings to apply
+
+    The settings format is ``Option=Value``. See Conan
+    documentation for further details.
+
+    Any {keyword} arguments, or macros, found in the strings are automatically
+    expanded to the value of the associated task's parameters and properties.
+
+    Example:
+
+    .. code-block:: python
+
+        settings = [
+            "compiler.libcxx=libstdc++11",
+        ]
+
+    """
+
+    remotes = {}
+    """
+    A dictionary with Conan remotes to use when fetching packages.
+
+    The dictionary key is the name of remote and its value is the URL.
+
+    Example:
+
+    .. code-block:: python
+
+        remotes = {
+            "bincrafters": "https://api.bintray.com/conan/bincrafters/public-conan"
+        }
+
+    """
+
+    incremental = True
+    """
+    Keep installed packages in the Conan cache between Jolt invokations.
+
+    If incremental build is disabled, the Jolt Conan cache is removed
+    before execution begins.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.conanfile:
+            self.influence.append(influence.FileInfluence(self.conanfile))
+
+    def _options(self):
+        return [] + self.options
+
+    def _settings(self):
+        return [] + self.settings
+
+    def _packages(self):
+        return [] + self.packages
+
+    def _remotes(self):
+        return self.remotes
+
+    def run(self, deps, tools):
+        raise_task_error_if(
+            not tools.which("conan"), self,
+            "Conan: Conan is not installed in the PATH")
+        raise_task_error_if(
+            self.conanfile and (self._packages() or self._options()), self,
+            "Conan: 'conanfile' attribute cannot be used with other attributes")
+
+        conanfile = tools.expand_path(self.conanfile) if self.conanfile else None
+
+        with tools.cwd(tools.builddir()):
+            if conanfile is None or not path.exists(conanfile):
+                conanfile = "conanfile.txt"
+                self.info("Creating conanfile.txt")
+                self.tools.write_file(conanfile, "[requires]\n")
+                for pkg in self._packages():
+                    self.tools.append_file(conanfile, pkg + "\n")
+
+            with tools.environ(CONAN_USER_HOME=tools.builddir("conan", incremental=self.incremental)):
+                for remote, url in self._remotes().items():
+                    self.info("Registering remote '{}'", remote)
+                    tools.run("conan remote add -f {} {}", remote, url, output_on_error=True)
+
+                self.info("Installing packages into the Conan cache")
+                options = " ".join(["-o " + opt for opt in self._options()])
+                settings = " ".join(["-s " + opt for opt in self._settings()])
+                output = tools.run("conan install --build=missing --output-folder . -u --format=json {} {} {}", options, settings, conanfile, output_stdout=False)
+
+            self.info("Parsing manifest")
+            self._manifest = json.loads(output)
+
+            for dep in self._manifest["graph"]["nodes"].values():
+                if dep["package_folder"]:
+                    self.info("Collecting '{}' files from: {}", dep["name"], dep["package_folder"])
+                    tools.copy(dep["package_folder"], dep["name"])
+
+    def publish(self, artifact, tools):
+        self.info("Publishing package files")
+        with tools.cwd(tools.builddir()):
+            artifact.collect("*")
+
+        self.info("Publishing metadata")
+        for dep in self._manifest["graph"]["nodes"].values():
+            if not dep["package_folder"]:
+                continue
+
+            for node in dep["cpp_info"]:
+                for incpath in dep["cpp_info"][node]["includedirs"]:
+                    artifact.cxxinfo.incpaths.append(path.join(dep["name"], path.relpath(incpath, dep["package_folder"])))
+                for libpath in dep["cpp_info"][node]["libdirs"]:
+                    artifact.cxxinfo.libpaths.append(path.join(dep["name"], path.relpath(libpath, dep["package_folder"])))
+                for binpath in dep["cpp_info"][node]["bindirs"]:
+                    artifact.environ.PATH.append(path.join(dep["name"], path.relpath(binpath, dep["package_folder"])))
+                if dep["cpp_info"][node]["libs"]:
+                    artifact.cxxinfo.libraries.extend(dep["cpp_info"][node]["libs"])
+                if dep["cpp_info"][node]["system_libs"]:
+                    artifact.cxxinfo.libraries.extend(dep["cpp_info"][node]["system_libs"])
+                if dep["cpp_info"][node]["defines"]:
+                    artifact.cxxinfo.macros.extend(dep["cpp_info"][node]["defines"])
+                if dep["cpp_info"][node]["cflags"]:
+                    artifact.cxxinfo.cflags.extend(dep["cpp_info"][node]["cflags"])
+                if dep["cpp_info"][node]["cxxflags"]:
+                    artifact.cxxinfo.cxxflags.extend(dep["cpp_info"][node]["cxxflags"])
+                if dep["cpp_info"][node]["exelinkflags"]:
+                    artifact.cxxinfo.ldflags.extend(dep["cpp_info"][node]["exelinkflags"])
+
+        # Make list of unique values
+        artifact.cxxinfo.incpaths = utils.unique_list(artifact.cxxinfo.incpaths)
+        artifact.cxxinfo.libpaths = utils.unique_list(artifact.cxxinfo.libpaths)
+        artifact.cxxinfo.libraries = utils.unique_list(artifact.cxxinfo.libraries)
+        artifact.cxxinfo.macros = utils.unique_list(artifact.cxxinfo.macros)
+        artifact.cxxinfo.cflags = utils.unique_list(artifact.cxxinfo.cflags)
+        artifact.cxxinfo.cxxflags = utils.unique_list(artifact.cxxinfo.cxxflags)
+        artifact.cxxinfo.ldflags = utils.unique_list(artifact.cxxinfo.ldflags)
+        artifact.environ.PATH = path.pathsep.join(utils.unique_list(str(artifact.environ.PATH).split(path.pathsep)))