jolt 0.9.342__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/influence.py

@@ -3,7 +3,7 @@ import hashlib
 import os
 from pathlib import Path, PurePath
 
-from jolt import config
+from jolt import config as jolt_config
 from jolt import inspection
 from jolt import utils
 from jolt import filesystem as fs
@@ -202,6 +202,71 @@ def source(name, obj=None):
     return _decorate
 
 
+class ConfigInfluence(HashInfluenceProvider):
+    name = "Config"
+
+    def __init__(self, section, key):
+        self.section = section
+        self.key = key
+
+    def get_influence(self, task):
+        value = jolt_config.get(self.section, self.key)
+        if value is None:
+            value = "<unset>"
+        return "{}.{}: {}".format(self.section, self.key, value)
+
+
+def config(section, key):
+    """ Add configuration value as hash influence.
+
+    Note that the configuration value is read at the time
+    when the task is prepared, not when it is executed.
+    This means that if the configuration value changes after
+    influence has been calculated, the task will not be re-executed.
+    This is true also in distributed mode, where the task
+    is prepared locally and executed remotely on a worker
+    where the configuration value may be different.
+
+    Args:
+        section (str): Name of the configuration section.
+        key (str): Name of the configuration key.
+
+    Example:
+    .. code-block:: python
+
+        from jolt import influence
+
+        @influence.config("jolt", "task_timeout")
+        class Example(Task):
+
+    """
+    def _decorate(cls):
+        _old_influence = cls._influence
+
+        def _influence(self, *args, **kwargs):
+            influence = _old_influence(self, *args, **kwargs)
+            influence.append(ConfigInfluence(section, key))
+            return influence
+
+        cls._influence = _influence
+        return cls
+
+    return _decorate
+
+
+def global_config(section, key):
+    """ Register a configuration influence globally.
+
+    See :py:func:`config` for more information.
+
+    Args:
+        section (str): Name of the configuration section.
+        key (str): Name of the configuration key.
+    """
+
+    HashInfluenceRegistry.get().register(ConfigInfluence(section, key))
+
+
 class TaskClassSourceInfluence(HashInfluenceProvider):
     name = "Source"
 
@@ -250,7 +315,7 @@ class CacheLocationInfluence(HashInfluenceProvider):
     name = "Cache"
 
     def get_influence(self, task):
-        return config.get_cachedir()
+        return jolt_config.get_cachedir()
 
 
 @HashInfluenceRegistry.Register

jolt/loader.py

@@ -1,4 +1,3 @@
-from collections import OrderedDict
 from contextlib import contextmanager
 import fasteners
 import json
@@ -11,20 +10,34 @@ import sys
 from types import ModuleType
 
 from jolt import inspection
-from jolt.tasks import attributes
-from jolt.tasks import Alias, Task, TaskGenerator, TaskRegistry, WorkspaceResource
-from jolt.error import raise_error_if, raise_task_error_if
+from jolt.tasks import Task, TaskGenerator
+from jolt.error import raise_error_if
 from jolt import common_pb2 as common_pb
 from jolt import config
 from jolt import filesystem as fs
 from jolt import log
 from jolt import utils
-from jolt.manifest import ManifestExtension
-from jolt.manifest import ManifestExtensionRegistry
 from jolt.tools import Tools
 
 
 class Recipe(object):
+    """
+    Abstract representation a single recipe file.
+
+    Implementations of this class are responsible for reading the recipe source
+    from the filesystem. The recipe source is then parsed and the tasks are
+    extracted and made available for execution.
+
+    The format of the recipe source is implementation defined.
+    """
+
+    tasks = []
+    """
+    List of task classes defined in the recipe.
+
+    Available after the recipe has been loaded.
+    """
+
     def __init__(self, path, joltdir=None, project=None, source=None):
         self.path = path
         self.basepath = os.path.basename(path)
@@ -33,13 +46,15 @@ class Recipe(object):
         self.source = source
         self.tasks = []
 
-    def load(self, joltdir=None):
+    def load(self):
+        """ Load the recipe source from the file system. """
         raise_error_if(self.source is not None, "recipe already loaded: {}", self.path)
 
         with open(self.path) as f:
             self.source = f.read()
 
     def save(self):
+        """ Save the recipe source to the file system. """
         raise_error_if(self.source is None, "recipe source unknown: {}", self.path)
 
         with open(self.path, "w") as f:
@@ -47,6 +62,8 @@ class Recipe(object):
 
 
 class NativeRecipe(Recipe):
+    """ Represents a Python recipe file (.jolt, .py). """
+
     @staticmethod
     def _is_abstract(cls):
         return cls.__dict__.get("abstract", False) or cls.__name__.startswith("_")
@@ -58,6 +75,13 @@ class NativeRecipe(Recipe):
             not NativeRecipe._is_abstract(cls)
 
     def load(self, joltdir=None):
+        """
+        Load the recipe source from the file system.
+
+        Python classes defined in the recipe source are extracted and made available
+        as tasks for execution. Task classes must be subclasses of Task or TaskGenerator.
+
+        """
         super(NativeRecipe, self).load()
 
         name = utils.canonical(self.path)
@@ -88,17 +112,42 @@ class NativeRecipe(Recipe):
 
 
 class Loader(object):
-    def recipes(self):
+    """
+    Base class for recipe loaders.
+
+    A Loader is responsible for finding recipes in the file system providing a list
+    of Recipe:s from which tasks can be loaded.
+    """
+
+    def recipes(self) -> list:
+        """ Return a list of Recipe:s from which tasks can be loaded. """
         pass
 
 
 class LoaderFactory(object):
+    """
+    A factory for creating Loader instances.
+
+    Factories are registered with the JoltLoader where it is used to create Loader instances.
+    """
     def create(self):
         raise NotImplementedError()
 
 
 class NativeLoader(Loader):
+    """ A loader for Python recipe files (.jolt, .py). """
+
     def __init__(self, searchpath):
+        """
+        Create a new NativeLoader instance.
+
+        Args:
+            searchpath (str): The path to search for recipe files. If the path is a file,
+                only that file will be loaded. If the path is a directory, all files with
+                the .jolt or .py extension will be loaded.
+
+        """
+
         self._files = self._find_files(searchpath)
         self._recipes = self._load_files(self._files) if self._files else []
 
@@ -109,7 +158,7 @@ class NativeLoader(Loader):
 
         _, ext = fs.path.splitext(searchpath)
         raise_error_if(not fs.path.exists(searchpath), "File does not exist: {}", searchpath)
-        raise_error_if(ext not in [".jolt", ".py"], "Invalid file extension: {}", ext)
+        raise_error_if(ext not in [".build", ".jolt", ".py"], "Invalid file extension: {}", ext)
 
         return [searchpath]
 
@@ -129,6 +178,7 @@ _loaders = []
 
 
 def register(factory):
+    """ Register a LoaderFactory with the JoltLoader. """
     raise_error_if(not issubclass(factory, LoaderFactory),
                    "{} is not a LoaderFactory", factory.__name__)
     _loaders.append(factory)
@@ -136,12 +186,22 @@ def register(factory):
 
 @register
 class NativeLoaderFactory(LoaderFactory):
+    """ A factory for creating NativeLoader instances. """
     def create(self, searchpath):
         return NativeLoader(searchpath)
 
 
 @utils.Singleton
 class JoltLoader(object):
+    """
+    The JoltLoader is responsible for loading recipes from the file system.
+
+    The JoltLoader is a singleton and is used to load recipes from the file system.
+    The recipes are loaded from the workspace directory and any project directories
+    defined in the workspace. The recipes are then made available for execution.
+
+    """
+
     filename = "*.jolt"
 
     def __init__(self):
@@ -149,61 +209,14 @@ class JoltLoader(object):
         self._recipes = []
         self._tasks = []
         self._path = None
-        self._project_modules = OrderedDict()
-        self._project_recipes = OrderedDict()
-        self._project_resources = OrderedDict()
         self._build_path = None
         self._workspace_name = None
 
-    def _add_project_module(self, project, src):
-        modules = self._project_modules.get(project, [])
-        modules.append(src)
-        self._project_modules[project] = modules
-
-    def _get_project_modules(self, project):
-        return self._project_modules.get(project, [])
-
-    def _add_project_recipe(self, project, joltdir, src):
-        recipes = self._project_recipes.get(project, [])
-        recipes.append((joltdir, src))
-        self._project_recipes[project] = recipes
-
-    def _get_project_recipes(self, project):
-        return self._project_recipes.get(project, [])
-
     def _get_first_recipe_path(self):
         for recipe in self.recipes:
             return recipe.path
-        for project, recipes in self._project_recipes.items():
-            for joltdir, src in recipes:
-                return fs.path.join(self._path, src)
         return None
 
-    def _add_project_resource(self, project, resource_name, resource_task):
-        class ProjectResource(Alias):
-            name = project + "/" + resource_name
-            requires = [resource_task]
-
-        self._tasks.append(ProjectResource)
-        resources = self._project_resources.get(project, [])
-        resources.append((resource_name, resource_task))
-        self._project_resources[project] = resources
-
-    def _get_project_resources(self, project):
-        return self._project_resources.get(project, [])
-
-    def _load_project_recipes(self):
-        for project, recipes in self._project_recipes.items():
-            resources = [resource for _, resource in self._get_project_resources(project)]
-            for joltdir, src in recipes:
-                joltdir = fs.path.join(self.joltdir, joltdir) if joltdir else self.joltdir
-                recipe = NativeRecipe(fs.path.join(self._path, src), joltdir, project)
-                recipe.load()
-                for task in recipe.tasks:
-                    task._resources = resources
-                    attributes.requires("_resources")(task)
-                self._tasks += recipe.tasks
-
     def _find_workspace_path(self, searchdir):
         for factory in _loaders:
             loader = factory().create(searchdir)
@@ -219,7 +232,16 @@ class JoltLoader(object):
     def _get_searchpaths(self):
         return [self.workspace_path]
 
-    def load(self, manifest=None):
+    def load(self, registry=None):
+        """
+        Load all recipes from the workspace directory.
+
+        Optionally populate the task registry with the tasks found in the recipes.
+
+        Returns:
+            List of Task classes found in the recipes.
+        """
+
         if not self.workspace_path:
             self.set_workspace_path(self._find_workspace_path(os.getcwd()))
 
@@ -230,21 +252,27 @@ class JoltLoader(object):
             for factory in _loaders:
                 loader = factory().create(searchpath)
                 for recipe in loader.recipes:
+                    recipe.workspace_path = os.path.relpath(recipe.path, self.workspace_path)
                     recipe.load()
                     self._recipes.append(recipe)
                     self._tasks += recipe.tasks
 
-        self._load_project_recipes()
-
         # Create workspace lock on the first loaded recipe
         if not self._lock:
             path = self._get_first_recipe_path()
             if path:
                 self._lock = fasteners.InterProcessLock(path)
 
+        # Add tasks to the registry if provided
+        if registry is not None:
+            for task in self._tasks:
+                registry.add_task_class(task)
+
         return self._tasks
 
     def load_file(self, path, joltdir=None):
+        """ Load a single recipe file. """
+
         for factory in _loaders:
             loader = factory().create(path)
             for recipe in loader.recipes:
@@ -254,6 +282,8 @@ class JoltLoader(object):
                 self._tasks += recipe.tasks
 
     def load_plugin(self, filepath):
+        """ Load a single plugin file. """
+
         plugin, ext = os.path.splitext(fs.path.basename(filepath))
         loader = SourceFileLoader("jolt.plugins." + plugin, filepath)
         module = ModuleType(loader.name)
@@ -262,6 +292,15 @@ class JoltLoader(object):
         sys.modules[loader.name] = module
 
     def load_plugins(self):
+        """
+        Load all configured plugins.
+
+        Plugins are loaded from the plugin path configured in the Jolt configuration file
+        or from the default plugin path in the Jolt package.
+
+        If a plugin is already loaded, it will not be loaded again. If a plugin is not found
+        in the plugin path, it will not be loaded.
+        """
         searchpath = config.get("jolt", "pluginpath")
         searchpath = searchpath.split(":") if searchpath else []
         searchpath.append(fs.path.join(fs.path.dirname(__file__), "plugins"))
@@ -288,14 +327,17 @@ class JoltLoader(object):
 
     @property
     def tasks(self):
+        """ Returns a list of all loaded tasks. """
         return self._tasks
 
     @property
     def joltdir(self):
+        """ Returns the path to the workspace. """
         return self._path
 
     @property
     def workspace_name(self):
+        """ Returns the name of the workspace. """
         return self._workspace_name or os.path.basename(self.workspace_path)
 
     def set_workspace_name(self, name):
@@ -303,6 +345,7 @@ class JoltLoader(object):
 
     @property
     def workspace_path(self):
+        """ Returns the path to the workspace. """
         return self._path
 
     @contextmanager
@@ -319,10 +362,12 @@ class JoltLoader(object):
 
     @property
     def build_path(self):
+        """ Returns the path to the build directory. """
         return self._build_path or os.path.join(self.workspace_path, "build")
 
     @property
     def build_path_rel(self):
+        """" Returns the path to the build directory relative to the workspace. """
         return os.path.relpath(self.build_path, self.workspace_path)
 
     def set_build_path(self, path):
@@ -330,98 +375,6 @@ class JoltLoader(object):
         log.debug("Jolt build path: {}", self._build_path)
 
 
-class RecipeExtension(ManifestExtension):
-    def export_manifest(self, manifest, tasks):
-        loader = JoltLoader.get()
-
-        for recipe in loader.recipes:
-            manifest_recipe = manifest.create_recipe()
-            manifest_recipe.path = recipe.basepath
-            manifest_recipe.source = recipe.source
-
-        projects = set([task.task.joltproject for task in tasks])
-        for project in filter(lambda x: x is not None, projects):
-            manifest_project = manifest.create_project()
-            manifest_project.name = project
-
-            for name, resource_task in loader._get_project_resources(project):
-                resource = manifest_project.create_resource()
-                resource.name = name
-                resource.text = resource_task
-
-            for joltdir, src in loader._get_project_recipes(project):
-                recipe = manifest_project.create_recipe()
-                recipe.src = src
-                if joltdir:
-                    recipe.joltdir = joltdir
-
-            for path in loader._get_project_modules(project):
-                module = manifest_project.create_module()
-                module.path = path
-
-    def import_manifest(self, manifest):
-        loader = JoltLoader.get()
-        loader.set_workspace_path(manifest.get_workspace_path() or os.getcwd())
-        loader.set_workspace_name(manifest.get_workspace_name())
-        if manifest.build:
-            loader.set_build_path(manifest.build)
-
-        for recipe in manifest.recipes:
-            recipe = Recipe(recipe.path, source=recipe.source)
-            recipe.save()
-
-        for project in manifest.projects:
-            for recipe in project.recipes:
-                loader._add_project_recipe(project.name, recipe.joltdir, recipe.src)
-
-            for resource in project.resources:
-                loader._add_project_resource(project.name, resource.name, resource.text)
-
-                # Acquire resource immediately
-                task = TaskRegistry.get().get_task(resource.text, manifest=manifest)
-                raise_task_error_if(not isinstance(task, WorkspaceResource), task,
-                                    "only workspace resources are allowed in manifest")
-                task.acquire_ws()
-
-            for module in project.modules:
-                loader._add_project_module(project.name, module.src)
-                sys.path.append(fs.path.join(manifest.get_workspace_path(), module.src))
-
-    def import_protobuf(self, buildenv):
-        loader = JoltLoader.get()
-        loader.set_workspace_path(os.getcwd())
-        loader.set_workspace_name(buildenv.workspace.name)
-        if buildenv.workspace.builddir:
-            loader.set_build_path(buildenv.workspace.builddir)
-
-        # Write .jolt files into workspace
-        for file in buildenv.workspace.files:
-            if file.content:
-                with open(file.path, "w") as f:
-                    f.write(file.content)
-
-        for project in buildenv.workspace.projects:
-            for recipe in project.recipes:
-                loader._add_project_recipe(project.name, recipe.workdir, recipe.path)
-
-            for resource in project.resources:
-                loader._add_project_resource(project.name, resource.alias, resource.name)
-
-                # Acquire resource immediately
-                task = TaskRegistry.get().get_task(resource.name, buildenv=buildenv)
-                raise_task_error_if(
-                    not isinstance(task, WorkspaceResource), task,
-                    "only workspace resources are allowed in manifest")
-                task.acquire_ws()
-
-            for path in project.paths:
-                loader._add_project_module(project.name, path.path)
-                sys.path.append(fs.path.join(loader.workspace_path, path.path))
-
-
-ManifestExtensionRegistry.add(RecipeExtension())
-
-
 def get_workspacedir():
     workspacedir = JoltLoader.get().workspace_path
     assert workspacedir is not None, "No workspace present"
@@ -441,8 +394,45 @@ def workspace_locked(func):
     return wrapper
 
 
+def import_workspace(buildenv: common_pb.BuildEnvironment):
+    """
+    Import workspace from a BuildEnvironment protobuf message.
+
+    This function will create files, recipes, resources and modules in the workspace
+    based on the information in the BuildEnvironment message.
+
+    The workspace tree is not pulled and checked out here. This is done by the
+    worker before it starts the executor.
+
+    """
+    loader = JoltLoader.get()
+    loader.set_workspace_path(os.getcwd())
+    loader.set_workspace_name(buildenv.workspace.name)
+    if buildenv.workspace.builddir:
+        loader.set_build_path(buildenv.workspace.builddir)
+
+    # Write .jolt files into workspace
+    for file in buildenv.workspace.files:
+        if file.content:
+            with open(file.path, "w") as f:
+                f.write(file.content)
+
+
 @workspace_locked
-def export_workspace(tasks=None):
+def export_workspace(tasks=None) -> common_pb.Workspace:
+    """
+    Export workspace to a Workspace protobuf message.
+
+    This function will create a Workspace protobuf message containing all the
+    recipes, resources and modules in the workspace. If tasks is provided, only
+    the projects associated with the tasks will be exported. Otherwise, all
+    projects will be exported.
+
+    If the workspace is configured to use a remote cache, the workspace will be
+    pushed to the remote cache using the fstree tool. The tree hash of the
+    workspace will be included in the returned Workspace message.
+
+    """
     loader = JoltLoader.get()
     tools = Tools()
     tree = None
@@ -461,6 +451,7 @@ def export_workspace(tasks=None):
     cache_grpc_uri = config.geturi("cache", "grpc_uri")
     if fstree_enabled:
         if not cache_grpc_uri:
+            fstree_enabled = False
             log.warning("No cache gRPC URI configured, will not push workspace to remote cache")
         else:
             raise_error_if(cache_grpc_uri.scheme not in ["tcp"], "Invalid scheme in cache gRPC URI config: {}", cache_grpc_uri.scheme)
@@ -553,40 +544,13 @@ def export_workspace(tasks=None):
         tree=tree,
     )
 
-    for recipe in loader.recipes:
-        workspace.files.append(
-            common_pb.File(
-                path=recipe.basepath,
-                content=recipe.source,
+    if not fstree_enabled:
+        for recipe in loader.recipes:
+            workspace.files.append(
+                common_pb.File(
+                    path=recipe.workspace_path,
+                    content=recipe.source,
+                )
             )
-        )
-
-    if tasks is None:
-        projects = loader._project_recipes.keys()
-    else:
-        projects = set([task.task.joltproject for task in tasks])
-
-    for project in filter(lambda x: x is not None, projects):
-        pb_project = common_pb.Project()
-        pb_project.name = project
-
-        for name, resource_task in loader._get_project_resources(project):
-            resource = common_pb.Project.Resource()
-            resource.alias = name
-            resource.name = resource_task
-            pb_project.resources.append(resource)
-
-        for joltdir, src in loader._get_project_recipes(project):
-            recipe = common_pb.Project.Recipe()
-            recipe.path = src
-            if joltdir:
-                recipe.workdir = joltdir
-            pb_project.recipes.append(recipe)
-
-        for path in loader._get_project_modules(project):
-            syspath = common_pb.Project.SystemPath(path=path)
-            pb_project.paths.append(syspath)
-
-        workspace.projects.append(pb_project)
 
     return workspace

jolt/log.py

@@ -364,6 +364,9 @@ class _Progress(object):
     def reset(self, *args, **kwargs):
         pass
 
+    def set_description(self, *args, **kwargs):
+        pass
+
 
 def progress_log(desc, count, unit):
     return _Progress(desc)
@@ -376,8 +379,15 @@ def progress(desc, count, unit, estimates=True, debug=False):
     else:
         bar_format = '{desc}{n_fmt}{unit} [{elapsed}]'
     if not debug and is_interactive() and not is_verbose():
-        p = tqdm.tqdm(total=count, unit=unit, unit_scale=True, bar_format=bar_format, dynamic_ncols=True)
-        p.set_description("[   INFO] " + desc)
+        class ProgressTqdm(tqdm.tqdm):
+            def __init__(self, *args, **kwargs):
+                super(ProgressTqdm, self).__init__(*args, **kwargs)
+
+            def set_description(self, desc=None, refresh=True):
+                return super().set_description("[   INFO] " + desc, refresh)
+
+        p = ProgressTqdm(total=count, unit=unit, unit_scale=True, bar_format=bar_format, dynamic_ncols=True)
+        p.set_description(desc)
         return p
     return progress_log(desc, count, unit)
 

jolt/manifest.py

@@ -241,49 +241,3 @@ class JoltManifest(ElementTree):
         for manifest_task in self.tasks:
             self._identities[manifest_task.name] = manifest_task.identity
         return self._identities
-
-    @staticmethod
-    def export(task):
-        manifest = JoltManifest()
-        ManifestExtensionRegistry.export_manifest(manifest, task)
-        return manifest
-
-    def process_import(self):
-        ManifestExtensionRegistry.import_manifest(self)
-
-
-class ManifestExtensionRegistry(object):
-    extensions = []
-
-    @staticmethod
-    def add(extension, priority=0):
-        ManifestExtensionRegistry.extensions.append((extension, priority))
-        ManifestExtensionRegistry.extensions.sort(key=lambda x: x[1])
-
-    @staticmethod
-    def export_manifest(manifest, task):
-        for extension, _ in ManifestExtensionRegistry.extensions:
-            extension.export_manifest(manifest, task)
-
-    @staticmethod
-    def import_manifest(manifest):
-        if not manifest.is_valid():
-            return
-        for extension, _ in ManifestExtensionRegistry.extensions:
-            extension.import_manifest(manifest)
-
-    @staticmethod
-    def import_protobuf(pb):
-        for extension, _ in ManifestExtensionRegistry.extensions:
-            extension.import_protobuf(pb)
-
-
-class ManifestExtension(object):
-    def export_manifest(self, manifest, task):
-        pass
-
-    def import_manifest(self, manifest):
-        pass
-
-    def import_protobuf(self, pb):
-        pass