jolt 0.9.172__py3-none-any.whl → 0.9.435__py3-none-any.whl

jolt/cli.py

@@ -1,7 +1,7 @@
-from os import _exit, environ, getcwd
 import atexit
 import click
 import datetime
+import os
 import platform
 import subprocess
 import sys
@@ -18,13 +18,12 @@ from jolt import log
 from jolt import __version__
 from jolt.log import logfile
 from jolt import config
-from jolt.loader import JoltLoader
+from jolt.loader import JoltLoader, import_workspace
 from jolt import tools
 from jolt import utils
 from jolt.influence import HashInfluenceRegistry
 from jolt.options import JoltOptions
 from jolt import hooks
-from jolt.manifest import JoltManifest
 from jolt.error import JoltError
 from jolt.error import raise_error
 from jolt.error import raise_error_if
@@ -32,7 +31,7 @@ from jolt.error import raise_task_error_if
 from jolt.plugins import report
 
 debug_enabled = False
-workdir = getcwd()
+workdir = os.getcwd()
 
 
 class ArgRequiredUnless(click.Argument):
@@ -56,10 +55,12 @@ class PluginGroup(click.Group):
 
         if cmd_name in ["export", "inspect"]:
             log.set_level(log.SILENCE)
-        elif ctx.params.get("verbose", False):
-            log.set_level(log.VERBOSE)
-        elif ctx.params.get("extra_verbose", False):
+        elif ctx.params.get("verbose") >= 3:
+            log.set_level(log.EXCEPTION)
+        elif ctx.params.get("verbose") >= 2:
             log.set_level(log.DEBUG)
+        elif ctx.params.get("verbose") >= 1:
+            log.set_level(log.VERBOSE)
 
         config_files = ctx.params.get("config_file") or []
         for config_file in config_files:
@@ -74,10 +75,12 @@ class PluginGroup(click.Group):
 
 @click.group(cls=PluginGroup, invoke_without_command=True)
 @click.version_option(__version__)
-@click.option("-v", "--verbose", is_flag=True, help="Verbose output.")
-@click.option("-vv", "--extra-verbose", is_flag=True, help="Extra verbose output.")
+@click.option("-v", "--verbose", count=True, help="Verbose output (repeat to raise verbosity).")
 @click.option("-c", "--config", "config_file", multiple=True, type=str,
               help="Load a configuration file or set a configuration key.")
+@click.option("-C", "--chdir", type=str,
+              help="Change working directory before executing command.")
+@click.option("--interpreter", "machine_interface", type=str, help="Used for debugging.", hidden=True)
 @click.option("-d", "--debugger", is_flag=True,
               help="Attach debugger on exception.")
 @click.option("-p", "--profile", is_flag=True, hidden=True,
@@ -88,6 +91,7 @@ class PluginGroup(click.Group):
               help="Add salt as task influence.")
 @click.option("-g", "--debug", is_flag=True, default=False, hidden=True,
               help="Start debug shell before executing task.")
+@click.option("-m", "--mute", is_flag=True, help="Display task log only if it fails.")
 @click.option("-n", "--network", is_flag=True, default=False, hidden=True,
               help="Build on network.")
 @click.option("-l", "--local", is_flag=True, default=False, hidden=True,
@@ -98,8 +102,8 @@ class PluginGroup(click.Group):
               help="Number of tasks allowed to execute in parallel (1). ")
 @click.option("-h", "--help", is_flag=True, help="Show this message and exit.")
 @click.pass_context
-def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
-        force, salt, debug, network, local, keep_going, jobs, help):
+def cli(ctx, verbose, config_file, debugger, profile,
+        force, salt, debug, mute, network, local, keep_going, jobs, help, machine_interface, chdir):
     """
     A task execution tool.
 
@@ -123,14 +127,24 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
     global debug_enabled
     debug_enabled = debugger
 
+    if machine_interface:
+        log.enable_gdb()
+
     if ctx.invoked_subcommand not in ["log", "report"]:
         log.start_file_log()
 
+    if chdir:
+        global workdir
+        workdir = chdir
+        os.chdir(workdir)
+
+    log.verbose("Jolt version: {}", __version__)
     log.verbose("Jolt command: {}", " ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:]))
-    log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost"))
+    log.verbose("Jolt host: {}", os.environ.get("HOSTNAME", "localhost"))
     log.verbose("Jolt install path: {}", fs.path.dirname(__file__))
+    log.verbose("Jolt workdir: {}", workdir)
 
-    if ctx.invoked_subcommand in ["config", "log"]:
+    if ctx.invoked_subcommand in ["config", "executor", "log"]:
         # Don't attempt to load any task recipes as they might require
         # plugins that are not yet configured.
         return
@@ -142,27 +156,13 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
         print(ctx.get_help())
         sys.exit(0)
 
-    manifest = JoltManifest()
-    utils.call_and_catch(manifest.parse)
-    manifest.process_import()
-    ctx.obj["manifest"] = manifest
-
-    if manifest.version:
-        from jolt.version_utils import requirement, version
-        req = requirement(manifest.version)
-        ver = version(__version__)
-        raise_error_if(not req.satisfied(ver),
-                       "This project requires Jolt version {} (running {})",
-                       req, __version__)
-
+    registry = TaskRegistry.get()
     loader = JoltLoader.get()
-    tasks = loader.load()
-    for cls in tasks:
-        TaskRegistry.get().add_task_class(cls)
+    loader.load(registry)
 
     if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir:
         ctx.obj["workspace_lock"] = utils.LockFile(
-            fs.path.join(loader.joltdir, "build"),
+            loader.build_path,
             log.info, "Workspace is locked by another process, please wait...")
         atexit.register(ctx.obj["workspace_lock"].close)
 
@@ -171,8 +171,8 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
     if ctx.invoked_subcommand is None:
         task = config.get("jolt", "default", "default")
         taskname, _ = utils.parse_task_name(task)
-        if TaskRegistry.get().get_task_class(taskname) is not None:
-            ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug,
+        if registry.get_task_class(taskname) is not None:
+            ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug, mute=mute,
                        network=network, local=local, keep_going=keep_going, jobs=jobs)
         else:
             print(cli.get_help(ctx))
@@ -180,11 +180,8 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
 
 
 def _autocomplete_tasks(ctx, args, incomplete):
-    manifest = JoltManifest()
-    utils.call_and_catch(manifest.parse)
-    manifest.process_import()
-
-    tasks = JoltLoader.get().load()
+    loader = JoltLoader.get()
+    tasks = loader.load()
     tasks = [task.name for task in tasks if task.name.startswith(incomplete or '')]
     return sorted(tasks)
 
@@ -203,12 +200,13 @@ def _autocomplete_tasks(ctx, args, incomplete):
 @click.option("-n", "--network", is_flag=True, default=False, help="Distribute tasks to network workers.")
 @click.option("-s", "--salt", type=str, help="Add salt as hash influence for all tasks in dependency tree.", metavar="SALT")
 @click.option("-m", "--mute", is_flag=True, help="Display task log only if it fails.")
-@click.option("-v", "--verbose", is_flag=True, help="Verbose output.")
-@click.option("-vv", "--extra-verbose", is_flag=True, help="Extra verbose output.")
+@click.option("-v", "--verbose", count=True, help="Verbose output (repeat to raise verbosity).")
 @click.option("--result", type=click.Path(), hidden=True,
               help="Write result manifest to this file.")
 @click.option("--no-download", is_flag=True, default=False,
-              help="Don't download artifacts from remote storage")
+              help="Don't download any artifacts from remote storage")
+@click.option("--no-download-persistent", is_flag=True, default=False,
+              help="Don't download persistent artifacts from remote storage (only session artifacts)")
 @click.option("--no-upload", is_flag=True, default=False,
               help="Don't upload artifacts to remote storage")
 @click.option("--download", is_flag=True, default=False,
@@ -219,12 +217,13 @@ def _autocomplete_tasks(ctx, args, incomplete):
               help="Don't prune cached artifacts from the build graph. This option can be used to populate the local cache with remotely cached dependency artifacts.")
 @click.option("--worker", is_flag=True, default=False,
               help="Run with the worker build strategy", hidden=True)
+@click.option("--environ", type=click.Path(), help="Import build environment from protobuf", hidden=True)
 @click.pass_context
 @hooks.cli_build
 def build(ctx, task, network, keep_going, default, local,
-          no_download, no_upload, download, upload, worker, force,
-          salt, copy, debug, result, jobs, no_prune, verbose, extra_verbose,
-          mute):
+          no_download, no_download_persistent, no_upload, download, upload, worker, force,
+          salt, copy, debug, result, jobs, no_prune, verbose,
+          mute, environ):
     """
     Build task artifact.
 
@@ -256,6 +255,7 @@ def build(ctx, task, network, keep_going, default, local,
     are removed before execution starts.
 
     """
+
     raise_error_if(network and local,
                    "The -n and -l flags are mutually exclusive")
 
@@ -268,10 +268,12 @@ def build(ctx, task, network, keep_going, default, local,
     raise_error_if(no_upload and upload,
                    "The --upload and --no-upload flags are mutually exclusive")
 
-    if verbose:
-        log.set_level(log.VERBOSE)
-    elif extra_verbose:
+    if verbose >= 3:
+        log.set_level(log.EXCEPTION)
+    elif verbose >= 2:
         log.set_level(log.DEBUG)
+    elif verbose >= 1:
+        log.set_level(log.VERBOSE)
 
     ts_start = utils.duration()
     task = list(task)
@@ -283,6 +285,7 @@ def build(ctx, task, network, keep_going, default, local,
     else:
         _download = config.getboolean("jolt", "download", True)
         _upload = config.getboolean("jolt", "upload", True)
+    _download_session = _download
 
     if local:
         _download = False
@@ -290,17 +293,52 @@ def build(ctx, task, network, keep_going, default, local,
     else:
         if no_download:
             _download = False
+            _download_session = False
+        if no_download_persistent:
+            _download = False
         if no_upload:
             _upload = False
         if download:
             _download = True
+            _download_session = True
         if upload:
             _upload = True
 
+    if keep_going:
+        config.set_keep_going(True)
+
+    # Import build environment from protobuf if provided
+    buildenv = None
+    if environ:
+        with open(environ, "rb") as f:
+            from jolt import common_pb2 as common_pb
+            buildenv = common_pb.BuildEnvironment()
+            try:
+                buildenv.ParseFromString(f.read())
+            except Exception as e:
+                raise_error("Failed to parse build environment protobuf: {}", e)
+
+        # Import log level
+        log.set_level_pb(buildenv.loglevel)
+
+        # Import workspace
+        import_workspace(buildenv)
+
+        # Import configuration snippet
+        config.import_config(buildenv.config)
+
+        # Import configuration parameters (-c params.key)
+        config.import_params({param.key: param.value for param in buildenv.parameters})
+
+        # Import default parameters (-d taskname:param=value)
+        default = utils.as_list(default)
+        default += buildenv.task_default_parameters
+
     options = JoltOptions(
         network=network,
         local=local,
         download=_download,
+        download_session=_download_session,
         upload=_upload,
         keep_going=keep_going,
         default=default,
@@ -330,26 +368,20 @@ def build(ctx, task, network, keep_going, default, local,
     for params in default:
         registry.set_default_parameters(params)
 
-    manifest = ctx.obj["manifest"]
-
-    for mb in manifest.builds:
-        for mt in mb.tasks:
-            task.append(mt.name)
-        for mt in mb.defaults:
-            registry.set_default_parameters(mt.name)
-
-    if force:
-        for goal in task:
-            registry.get_task(goal, manifest=manifest).taint = uuid.uuid4()
-
     log.info("Started: {}", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
 
-    gb = graph.GraphBuilder(registry, acache, manifest, options, progress=True)
+    gb = graph.GraphBuilder(registry, acache, options, progress=True, buildenv=buildenv)
     dag = gb.build(task)
 
+    # If asked to force rebuild, taint all goal tasks
+    if force:
+        for goal in dag.goals:
+            goal.get_extended_task().taint()
+
     # Collect information about artifact presence before starting prune or build
-    acache.precheck(dag.persistent_artifacts)
+    acache.precheck(dag.persistent_artifacts, remote=not local)
 
+    # Prune the graph to remove tasks that are already available locally or remotely
     if not no_prune:
         gp = graph.GraphPruner(acache, strategy)
         dag = gp.prune(dag)
@@ -358,7 +390,7 @@ def build(ctx, task, network, keep_going, default, local,
     goal_task_duration = 0
 
     session = executors.create_session(dag) if options.network else {}
-    queue = scheduler.TaskQueue(strategy, acache, session)
+    queue = scheduler.TaskQueue()
 
     try:
         if not dag.has_tasks():
@@ -372,16 +404,20 @@ def build(ctx, task, network, keep_going, default, local,
             debug=debug)
 
         with progress:
+            in_progress = set()
+
             while dag.has_tasks() or not queue.empty():
                 # Find all tasks ready to be executed
-                leafs = dag.select(lambda graph, task: task.is_ready())
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
                 # Order the tasks by their weights to improve build times
                 leafs.sort(key=lambda x: x.weight)
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(task)
+                    executor = strategy.create_executor(session, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
 
                 task, error = queue.wait()
 
@@ -391,6 +427,7 @@ def build(ctx, task, network, keep_going, default, local,
                 elif task.is_goal() and task.duration_running:
                     goal_task_duration += task.duration_running.seconds
 
+                # Unpack tasks with overridden unpack() method
                 if not task.is_resource():
                     if no_prune and task.task.unpack.__func__ is not Task.unpack:
                         with acache.get_context(task):
@@ -398,8 +435,13 @@ def build(ctx, task, network, keep_going, default, local,
 
                     progress.update(1)
 
+                if no_prune and task.is_workspace_resource():
+                    task.task.acquire_ws(force=True)
+
                 if not keep_going and error is not None:
                     queue.abort()
+                    executors.shutdown()
+                    task.raise_for_status()
                     raise error
 
         if dag.failed or dag.unstable:
@@ -407,6 +449,8 @@ def build(ctx, task, network, keep_going, default, local,
             for failed in dag.failed + dag.unstable:
                 log.error("- {}", failed.log_name.strip("()"))
 
+        for failed_task in dag.failed:
+            failed_task.raise_for_status()
         if dag.failed:
             raise_error("No more tasks could be executed")
 
@@ -415,12 +459,15 @@ def build(ctx, task, network, keep_going, default, local,
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
             log.warning("Interrupted again, exiting")
-            _exit(1)
+            os._exit(1)
     finally:
+        queue.shutdown()
+
         for task in goal_tasks:
             for artifact in task.artifacts:
                 if acache.is_available_locally(artifact):
@@ -467,7 +514,7 @@ def clean(ctx, task, deps, expired):
     if task:
         task = [utils.stable_task_name(t) for t in task]
         registry = TaskRegistry.get()
-        dag = graph.GraphBuilder(registry, acache, ctx.obj["manifest"]).build(task)
+        dag = graph.GraphBuilder(registry, acache).build(task)
         if deps:
             tasks = dag.tasks
         else:
@@ -614,7 +661,7 @@ def display(ctx, task, reverse=None, show_cache=False, prune=False):
     registry = TaskRegistry.get()
     options = JoltOptions()
     acache = cache.ArtifactCache.get(options)
-    gb = graph.GraphBuilder(registry, acache, ctx.obj["manifest"])
+    gb = graph.GraphBuilder(registry, acache)
     dag = gb.build(task, influence=show_cache)
 
     if reverse:
@@ -679,7 +726,14 @@ def docs():
     """
     Opens the Jolt documentation in the default webbrowser.
     """
-    webbrowser.open(config.get("jolt", "docs", "http://jolt.readthedocs.io/"))
+    url = config.get("jolt", "docs", "http://jolt.readthedocs.io/")
+    success = False
+    try:
+        success = webbrowser.open(url)
+    except Exception:
+        pass
+    if not success:
+        print(f"Failed to open web browser. Visit {url} manually.")
 
 
 @cli.command()
@@ -704,15 +758,14 @@ def download(ctx, task, deps, copy, copy_all):
     if copy_all:
         deps = True
 
-    manifest = ctx.obj["manifest"]
     options = JoltOptions()
     acache = cache.ArtifactCache.get(options)
     hooks.TaskHookRegistry.get(options)
     executors = scheduler.ExecutorRegistry.get(options)
     registry = TaskRegistry.get()
     strategy = scheduler.DownloadStrategy(executors, acache)
-    queue = scheduler.TaskQueue(strategy, acache, {})
-    gb = graph.GraphBuilder(registry, acache, manifest, options, progress=True)
+    queue = scheduler.TaskQueue()
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
     dag = gb.build(task)
 
     if not deps:
@@ -725,12 +778,16 @@ def download(ctx, task, deps, copy, copy_all):
 
     try:
         with log.progress("Progress", dag.number_of_tasks(), " tasks", estimates=False, debug=False) as p:
+            in_progress = set()
+
             while dag.has_tasks() or not queue.empty():
-                leafs = dag.select(lambda graph, task: task.is_ready())
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(task)
+                    executor = strategy.create_executor({}, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
 
                 task, error = queue.wait()
                 p.update(1)
@@ -753,62 +810,19 @@ def download(ctx, task, deps, copy, copy_all):
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
             log.warning("Interrupted again, exiting")
-            _exit(1)
+            os._exit(1)
+
     except Exception as e:
         log.set_interactive(True)
         raise e
 
-
-@cli.command(hidden=True)
-@click.argument("task", type=str, nargs=-1, required=True)
-@click.option("-r", "--remove", is_flag=True, help="Remove tasks from existing manifest.")
-@click.option("-d", "--default", type=str, multiple=True, help="Override default parameter values.")
-@click.option("-o", "--output", type=str, default="default.joltxmanifest", help="Manifest filename.")
-@click.pass_context
-def freeze(ctx, task, default, output, remove):
-    """
-    Freeze the identity of a task.
-
-    <WIP>
-    """
-    manifest = ctx.obj["manifest"]
-
-    options = JoltOptions(default=default)
-    acache = cache.ArtifactCache.get(options)
-    scheduler.ExecutorRegistry.get(options)
-    registry = TaskRegistry.get()
-
-    for params in default:
-        registry.set_default_parameters(params)
-
-    gb = graph.GraphBuilder(registry, acache, manifest)
-    dag = gb.build(task)
-
-    available, missing = acache.availability(dag.persistent_artifacts)
-
-    for artifact in missing:
-        raise_task_error_if(
-            not remove, artifact.get_task(),
-            "Task artifact is not available in any cache, build it first")
-
-    for task in dag.tasks:
-        if task.is_resource() or not task.is_cacheable():
-            continue
-        manifest_task = manifest.find_task(task)
-        if remove and manifest_task:
-            manifest.remove_task(manifest_task)
-            continue
-        if not remove:
-            if not manifest_task:
-                manifest_task = manifest.create_task()
-            manifest_task.name = task.qualified_name
-            manifest_task.identity = task.identity
-
-    manifest.write(fs.path.join(JoltLoader.get().joltdir, output))
+    finally:
+        queue.shutdown()
 
 
 @cli.command(name="list")
@@ -846,7 +860,7 @@ def _list(ctx, task=None, all=False, reverse=None):
     reverse = [utils.stable_task_name(t) for t in utils.as_list(reverse or [])]
 
     try:
-        dag = graph.GraphBuilder(registry, acache, ctx.obj["manifest"]).build(task, influence=False)
+        dag = graph.GraphBuilder(registry, acache).build(task, influence=False)
     except JoltError as e:
         raise e
     except Exception:
@@ -891,7 +905,7 @@ def _log(follow, delete):
             fs.unlink(file)
     else:
         t = tools.Tools()
-        configured_pager = config.get("jolt", "pager", environ.get("PAGER", None))
+        configured_pager = config.get("jolt", "pager", os.environ.get("PAGER", None))
         for pager in [configured_pager, "less", "more", "cat"]:
             if pager and t.which(pager):
                 return subprocess.call("{1} {0}".format(log.logfiles[-1], pager), shell=True)
@@ -946,9 +960,8 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
 
     print()
     print("  Requirements")
-    manifest = ctx.obj["manifest"]
     try:
-        task = task_registry.get_task(task_name, manifest=manifest)
+        task = task_registry.get_task(task_name)
         for req in sorted(utils.as_list(utils.call_or_return(task, task.requires))):
             print("    {0}".format(task.tools.expand(req)))
         if not task.requires:
@@ -970,7 +983,7 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
     if artifact:
         options = JoltOptions(salt=salt)
         acache = cache.ArtifactCache.get()
-        builder = graph.GraphBuilder(task_registry, acache, manifest, options)
+        builder = graph.GraphBuilder(task_registry, acache, options)
         dag = builder.build([task.qualified_name])
         tasks = dag.select(lambda graph, node: node.task is task)
         assert len(tasks) == 1, "graph produced multiple tasks, one expected"
@@ -1034,7 +1047,7 @@ def _export(ctx, task):
     executors = scheduler.ExecutorRegistry.get()
     strategy = scheduler.LocalStrategy(executors, acache)
 
-    dag = graph.GraphBuilder(registry, acache, ctx.obj["manifest"])
+    dag = graph.GraphBuilder(registry, acache)
     dag = dag.build(task)
 
     gp = graph.GraphPruner(acache, strategy)
@@ -1064,7 +1077,7 @@ def _export(ctx, task):
     for task in context.tasks:
         for artifact in task.artifacts:
             raise_task_error_if(
-                artifact.is_temporary(), task,
+                not task.is_resource() and artifact.is_temporary(), task,
                 "Task artifact not found in local cache, build it first")
 
             visitor = Export()
@@ -1105,15 +1118,15 @@ def _report(ctx):
             session=True,
             tools=t)
 
-        with t.tmpdir("report") as tmp, t.cwd(tmp.path):
+        with t.tmpdir("report") as tmp, t.cwd(tmp):
             log.info("Collecting environment")
             env = ""
-            for key, val in environ.items():
+            for key, val in os.environ.items():
                 env += f"{key} = {val}\n"
             t.write_file("environ.txt", env, expand=False)
 
             log.info("Collecting configuration")
-            config.save(tmp.path)
+            config.save(tmp)
             artifact.collect("*.conf", "configs/")
 
             log.info("Collecting platform information")