jolt 0.9.123__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/plugins/selfdeploy/setup.py

@@ -74,28 +74,32 @@ setup(
         "click>=8.1",
         "colorama",
         "fasteners",
+        "grpcio>=1.62.2",
         "jinja2",
         "keyring",
         "keyrings.alt",
+        "importlib_metadata",
         "lxml",
         "multi_key_dict",
-        "ninja-syntax",
+        "ninja",
+        "protobuf",
         "psutil",
         "pygit2",
+        "py7zr",
         "requests",
+        "zstandard",
         "tqdm",
     ],
     dependency_links=[],
     extras_require={
         "allure": ["allure-python-commons"],
-        "amqp": ["pika"],
-        "conan": ["conan<2.0"],
+        "conan": ["conan>=2.0"],
         "dev": ["check-manifest"],
         "doc": ["sphinx-click", "sphinx-rtd-theme"],
         "test": ["coverage"],
     },
     package_data={
-        "jolt": ["**/*.sh", "**/*.xslt", "**/*.template"],
+        "jolt": ["**/*.sh", "**/*.xslt", "**/*.template", "**/fstree-*-x86_64"],
     },
     entry_points={
         "console_scripts": [

jolt/plugins/selfdeploy.py

@@ -1,19 +1,21 @@
-from jolt.tasks import Task, TaskRegistry
-from jolt.cache import ArtifactCache
-from jolt.graph import GraphBuilder
-from jolt.error import raise_error_if
-from jolt.manifest import JoltManifest
-from jolt.scheduler import JoltEnvironment
-from jolt.scheduler import LocalExecutor
-from jolt.scheduler import LocalExecutorFactory
-from jolt.scheduler import NetworkExecutorExtension
-from jolt.scheduler import NetworkExecutorExtensionFactory
-from jolt.loader import JoltLoader
+import importlib_metadata
+import os
+
 from jolt import config
 from jolt import filesystem as fs
 from jolt import influence
 from jolt import log
+from jolt import common_pb2 as common_pb
 from jolt import utils
+from jolt import version
+from jolt.cache import ArtifactCache
+from jolt.error import raise_error_if
+from jolt.graph import GraphBuilder
+from jolt.loader import JoltLoader
+from jolt.scheduler import JoltEnvironment
+from jolt.scheduler import LocalExecutor
+from jolt.scheduler import LocalExecutorFactory
+from jolt.tasks import Task, TaskRegistry
 
 
 log.verbose("[SelfDeploy] Loaded")
@@ -27,6 +29,8 @@ _path = fs.path.dirname(_path)
 @influence.files(fs.path.join(_path, "**", "*.sh"))
 @influence.files(fs.path.join(_path, "**", "*.xslt"))
 @influence.files(fs.path.join(_path, "**", "*.template"))
+@influence.attribute("dependencies")
+@influence.attribute("extra_dependencies")
 class Jolt(Task):
     name = "jolt"
 
@@ -52,11 +56,7 @@ class Jolt(Task):
 
     @property
     def extra_dependencies(self):
-        req = config.get("selfdeploy", "requires", "")
-        return req.split() if req else []
-
-    def info(self, fmt, *args, **kwargs):
-        log.verbose(fmt, *args, **kwargs)
+        return get_extra_dependencies()
 
     @property
     def dependencies(self):
@@ -71,52 +71,14 @@ class Jolt(Task):
         no version pinning will be performed. Instead, workers will install
         Jolt with its default loose version requirements.
         """
-        def get_installed_distributions():
-            try:
-                from pip._internal.metadata import get_environment
-            except ImportError:
-                from pip._internal.utils import misc
-                return {
-                    dist.project_name.lower(): dist
-                    for dist in misc.get_installed_distributions()
-                }
-            else:
-                dists = get_environment(None).iter_installed_distributions()
-                return {dist._dist.project_name.lower(): dist._dist for dist in dists}
-
-        dists = get_installed_distributions()
-        reqs = ["jolt"] + [dep.lower() for dep in self.extra_dependencies]
-        pkgs = {}
-
-        while reqs:
-            req = reqs.pop()
-
-            dist = dists.get(req)
-            if dist is None:
-                self.info("[SelfDeploy] Dependency not found: {}", req)
-                req = req.partition("=")[0].partition("<")[0].partition(">")[0]
-                pkgs[req] = req
-                continue
-
-            for dep in dist.requires():
-                name = dep.project_name.lower()
-                if name not in pkgs:
-                    reqs.append(name)
-
-            pkgs[req] = f"{dist.project_name}=={dist.version}"
-
-        del pkgs["jolt"]
-        return pkgs.values()
+        return get_dependencies(["jolt"] + self.extra_dependencies)
 
     def publish(self, artifact, tools):
         with tools.cwd(tools.builddir()):
-            try:
-                pinned_reqs = self.dependencies
-                if pinned_reqs:
-                    tools.write_file("requirements.txt", "\n".join(pinned_reqs))
-                    artifact.collect("requirements.txt")
-            except Exception:
-                log.exception()
+            pinned_reqs = self.dependencies
+            if pinned_reqs:
+                tools.write_file("requirements.txt", "\n".join(pinned_reqs))
+                artifact.collect("requirements.txt")
         with tools.cwd(_path):
             artifact.collect('README.rst')
             artifact.collect('setup.py')
@@ -126,6 +88,7 @@ class Jolt(Task):
             artifact.collect('jolt/*/*/*.py')
             artifact.collect('jolt/*/*.xslt')
             artifact.collect('jolt/*/*.template')
+            artifact.collect('jolt/bin')
             artifact.collect('jolt/plugins/selfdeploy/README.rst', flatten=True)
             artifact.collect('jolt/plugins/selfdeploy/setup.py', flatten=True)
             for e in self.extra_files:
@@ -133,32 +96,119 @@ class Jolt(Task):
                     artifact.collect(fs.path.basename(e))
 
 
-class SelfDeployExtension(NetworkExecutorExtension):
-    @utils.cached.instance
-    def get_parameters(self, task):
-        registry = TaskRegistry()
-        registry.add_task_class(Jolt)
-        acache = ArtifactCache.get()
-        env = JoltEnvironment(cache=acache)
-        gb = GraphBuilder(registry, JoltManifest())
-        dag = gb.build(["jolt"])
-        task = dag.select(lambda graph, task: True)
-        assert len(task) == 1, "too many selfdeploy tasks found"
-        task = task[0]
-        if not acache.is_available_remotely(task):
-            factory = LocalExecutorFactory()
-            executor = LocalExecutor(factory, task, force_upload=True)
-            executor.run(env)
-        jolt_url = acache.location(task)
-        raise_error_if(not jolt_url, "failed to deploy jolt to a remote cache")
-        return {
-            "jolt_url": jolt_url,
-            "jolt_identity": task.identity[:8],
-            "jolt_requires": config.get("selfdeploy", "requires", "")
-        }
-
-
-@NetworkExecutorExtensionFactory.Register
-class SelfDeployExtensionFactory(NetworkExecutorExtensionFactory):
-    def create(self):
-        return SelfDeployExtension()
+@utils.cached.method
+def get_dependencies(packages=None):
+    reqs = packages or ["jolt"]
+    pkgs = {}
+    skip = set()
+
+    while reqs:
+        req = reqs.pop()
+
+        try:
+            dist = importlib_metadata.distribution(req)
+        except (ImportError, importlib_metadata.PackageNotFoundError):
+            dist = None
+        except Exception:
+            dist = None
+        if dist is None:
+            skip.add(req)
+            continue
+
+        for dep in dist.requires or []:
+            dep = dep.split(" ", 1)[0].strip()
+            dep = dep.split("[", 1)[0].strip()
+            dep = dep.split(";", 1)[0].strip()
+            dep = dep.split(">", 1)[0].strip()
+            dep = dep.split("=", 1)[0].strip()
+            dep = dep.split("<", 1)[0].strip()
+            dep = dep.split("!", 1)[0].strip()
+            if dep not in pkgs and dep not in skip:
+                reqs.append(dep)
+
+        pkgs[req] = f"{dist.name}=={dist.version}"
+
+    try:
+        del pkgs["jolt"]
+    except KeyError:
+        pass
+
+    return list(sorted(pkgs.values()))
+
+
+@utils.cached.method
+def get_extra_dependencies():
+    req = config.get("selfdeploy", "requires", "")
+    return req.split() if req else []
+
+
+@utils.cached.method
+def publish_artifact():
+    registry = TaskRegistry()
+    registry.add_task_class(Jolt)
+    acache = ArtifactCache.get()
+    env = JoltEnvironment(cache=acache, queue=None)
+    gb = GraphBuilder(registry, acache)
+    dag = gb.build(["jolt"])
+    task = dag.select(lambda graph, task: True)
+    assert len(task) == 1, "Too many selfdeploy tasks found"
+    task = task[0]
+    if not task.is_available_remotely(cache=False):
+        factory = LocalExecutorFactory()
+        executor = LocalExecutor(factory, task, force_upload=True)
+        executor.run(env)
+    jolt_url = acache.location(task.artifacts[0])
+    raise_error_if(not jolt_url, "Failed to deploy jolt to a remote cache")
+    cacheUrl = config.get("http", "uri", config.get("cache", "uri", "") + "/files")
+    substituteUrl = config.get("selfdeploy", "baseUri")
+    if cacheUrl and substituteUrl:
+        return task.identity, jolt_url.replace(cacheUrl, substituteUrl)
+    return task.identity, jolt_url
+
+
+def get_floating_version():
+    identity, url = publish_artifact()
+    return common_pb.Client(
+        identity=identity,
+        url=url,
+        version=version.__version__,
+    )
+
+
+def get_pinned_version():
+    return common_pb.Client(
+        requirements=get_extra_dependencies(),
+        version=version.__version__,
+    )
+
+
+def get_nix_version():
+    return common_pb.Client(
+        requirements=get_extra_dependencies(),
+        version=version.__version__,
+        nix=True,
+    )
+
+
+def get_client():
+    # Floating version is a special case where we want to deploy the Jolt
+    # source code to a remote cache and use that URL as the client URL
+    # for workers.
+    floating = config.getboolean("selfdeploy", "floating", False)
+    if floating:
+        return get_floating_version()
+
+    # If Nix has been explicitly disabled, we want to pin versions.
+    if not config.getboolean("selfdeploy", "nix", True):
+        return get_pinned_version()
+
+    # If Nix has been explicitly enabled, we want to use the Nix shell.
+    if config.getboolean("selfdeploy", "nix", False):
+        return get_nix_version()
+
+    # If we are in a Nix shell, we want to use the Nix shell.
+    if os.environ.get("IN_NIX_SHELL"):
+        return get_nix_version()
+
+    # Default to pinned version
+    return get_pinned_version()

jolt/plugins/strings.py

@@ -1,31 +1,50 @@
-from jolt.cache import ArtifactAttributeSet
 from jolt.cache import ArtifactAttributeSetProvider
-from jolt.cache import ArtifactStringAttribute
 
 
-class StringVariable(ArtifactStringAttribute):
-    def __init__(self, artifact, name):
-        super(StringVariable, self).__init__(artifact, name)
-        self._old_value = None
+class StringVariableSet(object):
+    """
+    A set of string variables for an artifact.
 
-    def apply(self, task, artifact):
-        pass
+    Example:
 
-    def unapply(self, task, artifact):
-        pass
+    .. code-block:: python
 
+        artifact.strings.foo = "bar"
+        print(artifact.strings.foo)
+
+    """
 
-class StringVariableSet(ArtifactAttributeSet):
     def __init__(self, artifact):
-        super(StringVariableSet, self).__init__()
-        super(ArtifactAttributeSet, self).__setattr__("_artifact", artifact)
+        super(StringVariableSet, self).__setattr__("_attributes", {})
+        super(StringVariableSet, self).__setattr__("_artifact", artifact)
+
+    def _get_attributes(self):
+        return self._attributes
+
+    def __getattr__(self, name):
+        attributes = self._get_attributes()
+        return attributes.get(name, None)
+
+    def __setattr__(self, name, value):
+        if not isinstance(value, str):
+            raise ValueError(f"Value assigned to artifact.strings.{name} must be a string, got {type(value)}")
+        attributes = self._get_attributes()
+        attributes[name] = self._artifact.tools.expand(value)
+        return value
 
-    def create(self, name):
-        return StringVariable(self._artifact, name)
+    def __dict__(self):
+        return {key: str(value) for key, value in self.items()}
+
+    def items(self):
+        return self._get_attributes().items()
 
 
 @ArtifactAttributeSetProvider.Register
 class StringVariableSetProvider(ArtifactAttributeSetProvider):
+    """
+    A provider of string variable sets.
+    """
+
     def create(self, artifact):
         setattr(artifact, "strings", StringVariableSet(artifact))
 
@@ -34,7 +53,7 @@ class StringVariableSetProvider(ArtifactAttributeSetProvider):
             return
 
         for key, value in content["strings"].items():
-            getattr(artifact.strings, key).set_value(value, expand=False)
+            setattr(artifact.strings, key, value)
 
     def format(self, artifact, content):
         if "strings" not in content:
@@ -42,9 +61,3 @@ class StringVariableSetProvider(ArtifactAttributeSetProvider):
 
         for key, value in artifact.strings.items():
             content["strings"][key] = str(value)
-
-    def apply(self, task, artifact):
-        artifact.strings.apply(task, artifact)
-
-    def unapply(self, task, artifact):
-        artifact.strings.unapply(task, artifact)

jolt/plugins/symlinks.py

@@ -1,5 +1,4 @@
 
-from jolt import cache
 from jolt import config
 from jolt import filesystem as fs
 from jolt import loader
@@ -19,20 +18,36 @@ class SymlinkTaskHooks(TaskHook):
         self._path = config.get("symlinks", "path", "artifacts")
         raise_error_if(not self._path, "symlinks.path not configured")
 
+    @property
+    def rootpath(self):
+        return fs.path.normpath(
+            fs.path.join(
+                fs.path.dirname(loader.JoltLoader.get().build_path),
+                self._path
+            )
+        )
+
     def task_finished(self, task):
         if not task.has_artifact():
             return
 
-        srcpath = cache.ArtifactCache.get().get_path(task)
-        destpath = fs.path.join(
-            loader.get_workspacedir(),
-            self._path,
-            utils.canonical(task.short_qualified_name))
-
-        if fs.path.exists(srcpath):
-            fs.unlink(destpath, ignore_errors=True)
-            fs.makedirs(fs.path.dirname(destpath))
-            fs.symlink(srcpath, destpath)
+        for artifact in task.artifacts:
+            srcpath = artifact.final_path
+            if artifact.name == "main":
+                destpath = fs.path.join(
+                    self.rootpath,
+                    utils.canonical(task.short_qualified_name),
+                )
+            else:
+                destpath = fs.path.join(
+                    self.rootpath,
+                    artifact.name + "@" + utils.canonical(task.short_qualified_name),
+                )
+
+            if fs.path.exists(srcpath):
+                fs.unlink(destpath, ignore_errors=True)
+                fs.makedirs(fs.path.dirname(destpath))
+                fs.symlink(srcpath, destpath)
 
     def task_pruned(self, task):
         self.task_finished(task)

jolt/plugins/telemetry.py

@@ -1,4 +1,3 @@
-from socket import gethostname
 from requests.exceptions import RequestException
 from requests.api import post
 
@@ -38,9 +37,10 @@ class TelemetryHooks(TaskHook):
             "name": task.short_qualified_name,
             "identity": task.identity,
             "instance": task.task._instance.value,
-            "hostname": gethostname(),
+            "hostname": utils.hostname(),
             "role": "client" if client else "worker",
             "event": event,
+            "routing_key": getattr(task.task, "routing_key", "default")
         }
         if hasattr(task, "logstash"):
             data["log"] = task.logstash
@@ -74,6 +74,9 @@ class TelemetryHooks(TaskHook):
             if self._network and self._failed:
                 self.post(task, "failed", client=True)
 
+    def task_unstable(self, task):
+        self.task_failed(task)
+
     def task_finished(self, task):
         if not self._finished:
             return

jolt/plugins/timeline.py

@@ -1,15 +1,25 @@
 from datetime import datetime
 import os
 
+from jolt import log
+from jolt import tools
 from jolt import utils
 from jolt.hooks import TaskHook, TaskHookFactory
-from jolt.config import get_workdir
+from jolt import config
+
+
+NAME_LOG = "Timeline"
 
 
 class TimelineHooks(TaskHook):
     def __init__(self):
+        self.path = os.path.join(
+            config.get_workdir(),
+            config.get("timeline", "path", "timeline.html")
+        )
         self.tasks = []
         self.task_ids = {}
+        self.tools = tools.Tools()
 
     def started(self, task):
         task._timeline_started = datetime.now().isoformat()
@@ -27,8 +37,7 @@ class TimelineHooks(TaskHook):
             enumerate=enumerate,
             tasks=self.tasks)
 
-        with open(os.path.join(get_workdir(), "timeline.html"), "w") as f:
-            f.write(timeline)
+        self.tools.write_file(self.path, timeline, expand=False)
 
     def deps(self, task):
         ids = []
@@ -59,4 +68,5 @@ class TimelineHooks(TaskHook):
 @TaskHookFactory.register
 class TimelineFactory(TaskHookFactory):
     def create(self, env):
+        log.verbose(NAME_LOG + " Loaded")
         return TimelineHooks()

jolt/plugins/volume.py

@@ -28,78 +28,76 @@ class DiskVolume(cache.StorageProvider):
         self._upload = config.getboolean(NAME, "upload", True)
         self._download = config.getboolean(NAME, "download", True)
 
-    def _get_path(self, node, artifact):
-        return "{path}/{name}/{file}".format(
+    def _get_path(self, artifact):
+        return artifact.tools.expand(
+            "{path}/{name}/{file}",
             path=self._path,
-            name=node.name,
+            name=artifact.task.name,
             file=fs.path.basename(artifact.get_archive_path()))
 
-    def _get_temp(self, node, artifact):
-        return "{path}/{name}/{file}".format(
+    def _get_temp(self, artifact):
+        return artifact.tools.expand(
+            "{path}/{name}/{file}",
             path=self._path,
-            name=node.name,
             file=uuid.uuid4())
 
     @utils.retried.on_exception(StaleFileHandleError)
-    def download(self, node, force=False):
+    def download(self, artifact, force=False):
         if not self._download and not force:
             return False
 
-        with self._cache.get_artifact(node) as artifact:
-            path = self._get_path(node, artifact)
-            try:
-                log.verbose("[VOLUME] Copying {}", path)
-                fs.copy(path, artifact.get_archive_path())
-                return True
-            except OSError as e:
-                if e.errno == errno.ESTALE:
-                    log.verbose("[VOLUME] got stale file handle, retrying...")
-                    raise StaleFileHandleError(e)
-                else:
-                    log.exception()
-            except Exception:
+        path = self._get_path(artifact)
+        try:
+            log.verbose("[VOLUME] Copying {}", path)
+            fs.copy(path, artifact.get_archive_path())
+            return True
+        except OSError as e:
+            if e.errno == errno.ESTALE:
+                log.verbose("[VOLUME] got stale file handle, retrying...")
+                raise StaleFileHandleError(e)
+            else:
                 log.exception()
+        except Exception:
+            log.exception()
 
         return False
 
     def download_enabled(self):
         return self._download
 
-    def upload(self, node, force=False):
+    def upload(self, artifact, force=False):
         if not self._upload and not force:
             return True
-        with self._cache.get_artifact(node) as artifact:
-            path = self._get_path(node, artifact)
-            temp = self._get_temp(node, artifact)
-            try:
-                log.verbose("[VOLUME] Copying {}", path)
-                fs.copy(artifact.get_archive_path(), temp)
-                # To avoid race-condition, make sure that the artifact still is missing before moving it into place.
-                if not fs.exists(path):
-                    fs.rename(temp, path)
-                else:
-                    fs.unlink(temp)
-                return True
-            except OSError as e:
-                if e.errno != errno.EEXIST:
-                    log.verbose("[VOLUME] Failed to copy artifact, errno={}", os.strerror(e.errno))
-                return e.errno == errno.EEXIST
-            except Exception:
-                log.exception()
-            finally:
-                fs.unlink(temp, ignore_errors=True)
+        path = self._get_path(artifact)
+        temp = self._get_temp(artifact)
+        try:
+            log.verbose("[VOLUME] Copying {}", path)
+            fs.copy(artifact.get_archive_path(), temp)
+            # To avoid race-condition, make sure that the artifact still is
+            # missing before moving it into place.
+            if not fs.exists(path):
+                fs.rename(temp, path)
+            else:
+                fs.unlink(temp)
+            return True
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                log.verbose("[VOLUME] Failed to copy artifact, errno={}", os.strerror(e.errno))
+            return e.errno == errno.EEXIST
+        except Exception:
+            log.exception()
+        finally:
+            fs.unlink(temp, ignore_errors=True)
         return False
 
     def upload_enabled(self):
         return self._upload
 
-    def location(self, node):
-        with self._cache.get_artifact(node) as artifact:
-            path = self._get_path(node, artifact)
-            avail = fs.path.exists(path)
-            log.debug("[VOLUME] {} is{} present", path, "" if avail else " not")
-            return avail
-        return False
+    def location(self, artifact):
+        path = self._get_path(artifact)
+        avail = fs.path.exists(path)
+        log.debug("[VOLUME] {} is{} present", path, "" if avail else " not")
+        return avail
 
 
 @cache.RegisterStorage