jolt 0.9.355__py3-none-any.whl → 0.9.371__py3-none-any.whl

jolt/cli.py

@@ -24,7 +24,6 @@ from jolt import utils
 from jolt.influence import HashInfluenceRegistry
 from jolt.options import JoltOptions
 from jolt import hooks
-from jolt.manifest import JoltManifest
 from jolt.error import JoltError
 from jolt.error import raise_error
 from jolt.error import raise_error_if
@@ -156,23 +155,9 @@ def cli(ctx, verbose, config_file, debugger, profile,
         print(ctx.get_help())
         sys.exit(0)
 
-    manifest = JoltManifest()
-    utils.call_and_catch(manifest.parse)
-    manifest.process_import()
-    ctx.obj["manifest"] = manifest
-
-    if manifest.version:
-        from jolt.version_utils import requirement, version
-        req = requirement(manifest.version)
-        ver = version(__version__)
-        raise_error_if(not req.satisfied(ver),
-                       "This project requires Jolt version {} (running {})",
-                       req, __version__)
-
+    registry = TaskRegistry.get()
     loader = JoltLoader.get()
-    tasks = loader.load()
-    for cls in tasks:
-        TaskRegistry.get().add_task_class(cls)
+    loader.load(registry)
 
     if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir:
         ctx.obj["workspace_lock"] = utils.LockFile(
@@ -185,7 +170,7 @@ def cli(ctx, verbose, config_file, debugger, profile,
     if ctx.invoked_subcommand is None:
         task = config.get("jolt", "default", "default")
         taskname, _ = utils.parse_task_name(task)
-        if TaskRegistry.get().get_task_class(taskname) is not None:
+        if registry.get_task_class(taskname) is not None:
             ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug,
                        network=network, local=local, keep_going=keep_going, jobs=jobs)
         else:
@@ -194,11 +179,8 @@ def cli(ctx, verbose, config_file, debugger, profile,
 
 
 def _autocomplete_tasks(ctx, args, incomplete):
-    manifest = JoltManifest()
-    utils.call_and_catch(manifest.parse)
-    manifest.process_import()
-
-    tasks = JoltLoader.get().load()
+    loader = JoltLoader.get()
+    tasks = loader.load()
     tasks = [task.name for task in tasks if task.name.startswith(incomplete or '')]
     return sorted(tasks)
 
@@ -356,26 +338,20 @@ def build(ctx, task, network, keep_going, default, local,
     for params in default:
         registry.set_default_parameters(params)
 
-    manifest = ctx.obj["manifest"]
-
-    for mb in manifest.builds:
-        for mt in mb.tasks:
-            task.append(mt.name)
-        for mt in mb.defaults:
-            registry.set_default_parameters(mt.name)
-
-    if force:
-        for goal in task:
-            registry.get_task(goal, manifest=manifest).taint = uuid.uuid4()
-
     log.info("Started: {}", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
 
-    gb = graph.GraphBuilder(registry, acache, manifest, options, progress=True)
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
     dag = gb.build(task)
 
+    # If asked to force rebuild, taint all goal tasks
+    if force:
+        for goal in dag.goals:
+            goal.taint()
+
     # Collect information about artifact presence before starting prune or build
     acache.precheck(dag.persistent_artifacts, remote=not local)
 
+    # Prune the graph to remove tasks that are already available locally or remotely
     if not no_prune:
         gp = graph.GraphPruner(acache, strategy)
         dag = gp.prune(dag)
@@ -384,7 +360,7 @@ def build(ctx, task, network, keep_going, default, local,
     goal_task_duration = 0
 
     session = executors.create_session(dag) if options.network else {}
-    queue = scheduler.TaskQueue(strategy, acache, session)
+    queue = scheduler.TaskQueue()
 
     try:
         if not dag.has_tasks():
@@ -407,7 +383,8 @@ def build(ctx, task, network, keep_going, default, local,
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(task)
+                    executor = strategy.create_executor(session, task)
+                    queue.submit(executor)
 
                 task, error = queue.wait()
 
@@ -430,6 +407,7 @@ def build(ctx, task, network, keep_going, default, local,
 
                 if not keep_going and error is not None:
                     queue.abort()
+                    executors.shutdown()
                     task.raise_for_status()
                     raise error
 
@@ -448,6 +426,7 @@ def build(ctx, task, network, keep_going, default, local,
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
@@ -502,7 +481,7 @@ def clean(ctx, task, deps, expired):
     if task:
         task = [utils.stable_task_name(t) for t in task]
         registry = TaskRegistry.get()
-        dag = graph.GraphBuilder(registry, acache, ctx.obj["manifest"]).build(task)
+        dag = graph.GraphBuilder(registry, acache).build(task)
         if deps:
             tasks = dag.tasks
         else:
@@ -649,7 +628,7 @@ def display(ctx, task, reverse=None, show_cache=False, prune=False):
     registry = TaskRegistry.get()
     options = JoltOptions()
     acache = cache.ArtifactCache.get(options)
-    gb = graph.GraphBuilder(registry, acache, ctx.obj["manifest"])
+    gb = graph.GraphBuilder(registry, acache)
     dag = gb.build(task, influence=show_cache)
 
     if reverse:
@@ -746,15 +725,14 @@ def download(ctx, task, deps, copy, copy_all):
     if copy_all:
         deps = True
 
-    manifest = ctx.obj["manifest"]
     options = JoltOptions()
     acache = cache.ArtifactCache.get(options)
     hooks.TaskHookRegistry.get(options)
     executors = scheduler.ExecutorRegistry.get(options)
     registry = TaskRegistry.get()
     strategy = scheduler.DownloadStrategy(executors, acache)
-    queue = scheduler.TaskQueue(strategy, acache, {})
-    gb = graph.GraphBuilder(registry, acache, manifest, options, progress=True)
+    queue = scheduler.TaskQueue()
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
     dag = gb.build(task)
 
     if not deps:
@@ -772,7 +750,8 @@ def download(ctx, task, deps, copy, copy_all):
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(task)
+                    executor = strategy.create_executor({}, task)
+                    queue.submit(executor)
 
                 task, error = queue.wait()
                 p.update(1)
@@ -795,6 +774,7 @@ def download(ctx, task, deps, copy, copy_all):
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
@@ -809,54 +789,6 @@ def download(ctx, task, deps, copy, copy_all):
         queue.shutdown()
 
 
-@cli.command(hidden=True)
-@click.argument("task", type=str, nargs=-1, required=True)
-@click.option("-r", "--remove", is_flag=True, help="Remove tasks from existing manifest.")
-@click.option("-d", "--default", type=str, multiple=True, help="Override default parameter values.")
-@click.option("-o", "--output", type=str, default="default.joltxmanifest", help="Manifest filename.")
-@click.pass_context
-def freeze(ctx, task, default, output, remove):
-    """
-    Freeze the identity of a task.
-
-    <WIP>
-    """
-    manifest = ctx.obj["manifest"]
-
-    options = JoltOptions(default=default)
-    acache = cache.ArtifactCache.get(options)
-    scheduler.ExecutorRegistry.get(options)
-    registry = TaskRegistry.get()
-
-    for params in default:
-        registry.set_default_parameters(params)
-
-    gb = graph.GraphBuilder(registry, acache, manifest)
-    dag = gb.build(task)
-
-    available, missing = acache.availability(dag.persistent_artifacts)
-
-    for artifact in missing:
-        raise_task_error_if(
-            not remove, artifact.get_task(),
-            "Task artifact is not available in any cache, build it first")
-
-    for task in dag.tasks:
-        if task.is_resource() or not task.is_cacheable():
-            continue
-        manifest_task = manifest.find_task(task)
-        if remove and manifest_task:
-            manifest.remove_task(manifest_task)
-            continue
-        if not remove:
-            if not manifest_task:
-                manifest_task = manifest.create_task()
-            manifest_task.name = task.qualified_name
-            manifest_task.identity = task.identity
-
-    manifest.write(fs.path.join(JoltLoader.get().joltdir, output))
-
-
 @cli.command(name="list")
 @click.argument("task", type=str, nargs=-1, required=False, shell_complete=_autocomplete_tasks)
 @click.option("-a", "--all", is_flag=True, help="List all direct and indirect dependencies of TASK.")
@@ -892,7 +824,7 @@ def _list(ctx, task=None, all=False, reverse=None):
     reverse = [utils.stable_task_name(t) for t in utils.as_list(reverse or [])]
 
     try:
-        dag = graph.GraphBuilder(registry, acache, ctx.obj["manifest"]).build(task, influence=False)
+        dag = graph.GraphBuilder(registry, acache).build(task, influence=False)
     except JoltError as e:
         raise e
     except Exception:
@@ -992,9 +924,8 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
 
     print()
     print("  Requirements")
-    manifest = ctx.obj["manifest"]
     try:
-        task = task_registry.get_task(task_name, manifest=manifest)
+        task = task_registry.get_task(task_name)
         for req in sorted(utils.as_list(utils.call_or_return(task, task.requires))):
             print("    {0}".format(task.tools.expand(req)))
         if not task.requires:
@@ -1016,7 +947,7 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
     if artifact:
         options = JoltOptions(salt=salt)
         acache = cache.ArtifactCache.get()
-        builder = graph.GraphBuilder(task_registry, acache, manifest, options)
+        builder = graph.GraphBuilder(task_registry, acache, options)
         dag = builder.build([task.qualified_name])
         tasks = dag.select(lambda graph, node: node.task is task)
         assert len(tasks) == 1, "graph produced multiple tasks, one expected"
@@ -1080,7 +1011,7 @@ def _export(ctx, task):
     executors = scheduler.ExecutorRegistry.get()
     strategy = scheduler.LocalStrategy(executors, acache)
 
-    dag = graph.GraphBuilder(registry, acache, ctx.obj["manifest"])
+    dag = graph.GraphBuilder(registry, acache)
     dag = dag.build(task)
 
     gp = graph.GraphPruner(acache, strategy)
@@ -1110,7 +1041,7 @@ def _export(ctx, task):
     for task in context.tasks:
         for artifact in task.artifacts:
             raise_task_error_if(
-                artifact.is_temporary(), task,
+                not task.is_resource() and artifact.is_temporary(), task,
                 "Task artifact not found in local cache, build it first")
 
             visitor = Export()

jolt/config.py

@@ -7,7 +7,6 @@ from jolt import common_pb2 as common_pb
 from jolt import filesystem as fs
 from jolt import utils
 from jolt.error import raise_error_if
-from jolt.manifest import ManifestExtension, ManifestExtensionRegistry
 
 
 _workdir = os.getcwd()
@@ -318,39 +317,28 @@ def split(string):
     return section, key
 
 
-class ConfigExtension(ManifestExtension):
-    def export_manifest(self, manifest, _):
-        manifest.config = get("network", "config", "", expand=False)
-
-        for key, value in options("params"):
-            p = manifest.create_parameter()
-            p.key = "config." + key
-            p.value = value
-
-    def import_manifest(self, manifest):
-        if manifest.config:
-            _manifest.read_string(manifest.config)
-            from jolt.loader import JoltLoader
-            JoltLoader.get().load_plugins()
-
-        for param in manifest.parameters:
-            if param.key.startswith("config."):
-                set("params", param.key.split(".", 1)[1], param.value)
-
-    def import_protobuf(self, pb):
-        self.import_manifest(pb)
+def import_config(snippet: str):
+    """ Apply extra configuration for the worker, provided by the client. """
+    _manifest.read_string(snippet)
+    from jolt.loader import JoltLoader
+    JoltLoader.get().load_plugins()
 
 
 def export_config():
+    """ Get extra configuration for the worker. """
     return get("network", "config", "", expand=False)
 
 
+def import_params(params: dict):
+    """ Apply user-defined parameters (-c params.key=value). """
+    for key, value in params.items():
+        if key.startswith("config."):
+            set("params", key.split(".", 1)[1], value)
+
+
 def export_params():
+    """ Get user-defined parameters (-c params.key=value). """
     parameters = []
     for key, value in options("params"):
         parameters.append(common_pb.Property(key="config." + key, value=value))
     return parameters
-
-
-# High priority so that plugins are loaded before resources are acquired.
-ManifestExtensionRegistry.add(ConfigExtension(), -10)

jolt/graph.py

@@ -424,9 +424,8 @@ class TaskProxy(object):
         self._owner = owner
         self.task.exported_name = f"{self.short_qualified_name}@@{owner.short_qualified_name}"
 
-    def finalize(self, dag, manifest):
+    def finalize(self, dag):
         log.debug("Finalizing: " + self.short_qualified_name)
-        self.manifest = manifest
 
         # Find all direct and transitive dependencies
         self.ancestors = set()
@@ -468,11 +467,23 @@ class TaskProxy(object):
         self._artifacts.extend(self.task._artifacts(self.cache, self))
 
     def taint(self, salt=None):
-        self.task.taint = self.task.taint or salt or uuid.uuid4()
+        self.task.taint = salt or uuid.uuid4()
         if salt is None:
             # Only recalculate identity when build is forced, not when salted
             self.identity = None
             self.identity
+            # Recreate artifacts
+            self._artifacts = []
+            self.finalize_artifacts()
+
+        # If this is an alias, taint all children
+        if self.is_alias():
+            for child in self.children:
+                child.taint()
+
+        # Taint all extensions
+        for extension in self.extensions:
+            extension.taint()
 
     def queued(self, remote=True):
         self.task.verbose("Task queued " + self.log_name)
@@ -749,7 +760,10 @@ class TaskProxy(object):
         try:
             # Acquire resource dependencies in reverse order.
             for resource in reversed(resource_deps):
-                with resource.lock_artifacts(discard=False) if not resource.is_workspace_resource() else nullcontext():
+                # Always discard resource artifacts before acquiring the resource.
+                # They should not exist in the cache when the resource is acquired,
+                # but may exist if the resource was previously acquired by an interrupted build.
+                with resource.lock_artifacts(discard=True) if not resource.is_workspace_resource() else nullcontext():
                     resource.deps = self.cache.get_context(resource)
                     exitstack.enter_context(resource.deps)
 
@@ -779,7 +793,6 @@ class TaskProxy(object):
             self._run_task(env, force_upload, force_build)
 
     def _run_task(self, env, force_upload=False, force_build=False):
-        cache = env.cache
         queue = env.queue
 
         with self.tools:
@@ -793,7 +806,7 @@ class TaskProxy(object):
                     self.skipped()
                     return
 
-                available_remotely = cache.download_enabled() and self.is_available_remotely()
+                available_remotely = self.cache.download_enabled() and self.is_available_remotely()
                 if not available_locally and available_remotely:
                     available_locally = self.download()
 
@@ -812,7 +825,7 @@ class TaskProxy(object):
                         upload_session_artifacts = False
 
                         try:
-                            context = cache.get_context(self)
+                            context = self.cache.get_context(self)
                             exitstack.enter_context(context)
 
                             self.running_execution()
@@ -881,7 +894,7 @@ class TaskProxy(object):
                             if force_upload or force_build or not available_remotely:
                                 raise_task_error_if(
                                     not self.upload(force=force_upload, locked=False, persistent_only=True) \
-                                    and cache.upload_enabled(),
+                                    and self.cache.upload_enabled(),
                                     self, "Failed to upload task artifact")
 
                         finally:
@@ -889,14 +902,14 @@ class TaskProxy(object):
                             raise_task_error_if(
                                 upload_session_artifacts \
                                 and not self.upload(force=force_upload, locked=False, session_only=True, artifacts=upload_session_artifacts) \
-                                and cache.upload_enabled(),
+                                and self.cache.upload_enabled(),
                                 self, "Failed to upload session artifact")
 
             elif force_upload or not available_remotely:
                 self.started_upload()
                 raise_task_error_if(
                     not self.upload(force=force_upload, persistent_only=True) \
-                    and cache.upload_enabled(),
+                    and self.cache.upload_enabled(),
                     self, "Failed to upload task artifact")
                 self.finished_upload()
 
@@ -1091,12 +1104,11 @@ class Graph(object):
 
 
 class GraphBuilder(object):
-    def __init__(self, registry, cache, manifest=None, options=None, progress=False, buildenv=None):
+    def __init__(self, registry, cache, options=None, progress=False, buildenv=None):
         self.cache = cache
         self.graph = Graph()
         self.nodes = {}
         self.registry = registry
-        self.manifest = manifest
         self.buildenv = buildenv
         self.progress = progress
         self.options = options or JoltOptions()
@@ -1105,7 +1117,7 @@ class GraphBuilder(object):
         name = utils.stable_task_name(name)
         node = self.nodes.get(name)
         if not node:
-            task = self.registry.get_task(name, manifest=self.manifest, buildenv=self.buildenv)
+            task = self.registry.get_task(name, buildenv=self.buildenv)
             node = self.nodes.get(task.qualified_name, None)
             if node is not None:
                 return node
@@ -1137,7 +1149,7 @@ class GraphBuilder(object):
             parent = node
 
         for requirement in node.task.requires:
-            alias, artifact, task, name = utils.parse_aliased_task_name(requirement)
+            alias, _, task, name = utils.parse_aliased_task_name(requirement)
             child = self._get_node(progress, utils.format_task_name(task, name), parent=node)
 
             # Create direct edges from alias parents to alias children
@@ -1172,7 +1184,7 @@ class GraphBuilder(object):
             topological_nodes = self.graph.topological_nodes
             with self._progress("Collecting task influence", len(self.graph.tasks), "tasks") as p:
                 for node in reversed(topological_nodes):
-                    node.finalize(self.graph, self.manifest)
+                    node.finalize(self.graph)
                     p.update(1)
 
                 # Create artifacts in forward order so that parent identities are available