jolt 0.9.76__py3-none-any.whl → 0.9.429__py3-none-any.whl

jolt/cli.py

@@ -1,10 +1,12 @@
 import atexit
 import click
+import datetime
+import os
+import platform
 import subprocess
 import sys
 import uuid
 import webbrowser
-from os import _exit, environ, getcwd
 
 from jolt.tasks import Task, TaskRegistry, Parameter
 from jolt import scheduler
@@ -16,13 +18,12 @@ from jolt import log
 from jolt import __version__
 from jolt.log import logfile
 from jolt import config
-from jolt.loader import JoltLoader
+from jolt.loader import JoltLoader, import_workspace
 from jolt import tools
 from jolt import utils
 from jolt.influence import HashInfluenceRegistry
 from jolt.options import JoltOptions
 from jolt import hooks
-from jolt.manifest import JoltManifest
 from jolt.error import JoltError
 from jolt.error import raise_error
 from jolt.error import raise_error_if
@@ -30,7 +31,7 @@ from jolt.error import raise_task_error_if
 from jolt.plugins import report
 
 debug_enabled = False
-workdir = getcwd()
+workdir = os.getcwd()
 
 
 class ArgRequiredUnless(click.Argument):
@@ -54,10 +55,12 @@ class PluginGroup(click.Group):
 
         if cmd_name in ["export", "inspect"]:
             log.set_level(log.SILENCE)
-        elif ctx.params.get("verbose", False):
-            log.set_level(log.VERBOSE)
-        elif ctx.params.get("extra_verbose", False):
+        elif ctx.params.get("verbose") >= 3:
+            log.set_level(log.EXCEPTION)
+        elif ctx.params.get("verbose") >= 2:
             log.set_level(log.DEBUG)
+        elif ctx.params.get("verbose") >= 1:
+            log.set_level(log.VERBOSE)
 
         config_files = ctx.params.get("config_file") or []
         for config_file in config_files:
@@ -72,10 +75,12 @@ class PluginGroup(click.Group):
 
 @click.group(cls=PluginGroup, invoke_without_command=True)
 @click.version_option(__version__)
-@click.option("-v", "--verbose", is_flag=True, help="Verbose output.")
-@click.option("-vv", "--extra-verbose", is_flag=True, help="Extra verbose output.")
+@click.option("-v", "--verbose", count=True, help="Verbose output (repeat to raise verbosity).")
 @click.option("-c", "--config", "config_file", multiple=True, type=str,
               help="Load a configuration file or set a configuration key.")
+@click.option("-C", "--chdir", type=str,
+              help="Change working directory before executing command.")
+@click.option("--interpreter", "machine_interface", type=str, help="Used for debugging.", hidden=True)
 @click.option("-d", "--debugger", is_flag=True,
               help="Attach debugger on exception.")
 @click.option("-p", "--profile", is_flag=True, hidden=True,
@@ -86,6 +91,7 @@ class PluginGroup(click.Group):
               help="Add salt as task influence.")
 @click.option("-g", "--debug", is_flag=True, default=False, hidden=True,
               help="Start debug shell before executing task.")
+@click.option("-m", "--mute", is_flag=True, help="Display task log only if it fails.")
 @click.option("-n", "--network", is_flag=True, default=False, hidden=True,
               help="Build on network.")
 @click.option("-l", "--local", is_flag=True, default=False, hidden=True,
@@ -96,8 +102,8 @@ class PluginGroup(click.Group):
               help="Number of tasks allowed to execute in parallel (1). ")
 @click.option("-h", "--help", is_flag=True, help="Show this message and exit.")
 @click.pass_context
-def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
-        force, salt, debug, network, local, keep_going, jobs, help):
+def cli(ctx, verbose, config_file, debugger, profile,
+        force, salt, debug, mute, network, local, keep_going, jobs, help, machine_interface, chdir):
     """
     A task execution tool.
 
@@ -121,11 +127,24 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
     global debug_enabled
     debug_enabled = debugger
 
+    if machine_interface:
+        log.enable_gdb()
+
+    if ctx.invoked_subcommand not in ["log", "report"]:
+        log.start_file_log()
+
+    if chdir:
+        global workdir
+        workdir = chdir
+        os.chdir(workdir)
+
+    log.verbose("Jolt version: {}", __version__)
     log.verbose("Jolt command: {}", " ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:]))
-    log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost"))
+    log.verbose("Jolt host: {}", os.environ.get("HOSTNAME", "localhost"))
     log.verbose("Jolt install path: {}", fs.path.dirname(__file__))
+    log.verbose("Jolt workdir: {}", workdir)
 
-    if ctx.invoked_subcommand in ["config"]:
+    if ctx.invoked_subcommand in ["config", "executor", "log"]:
         # Don't attempt to load any task recipes as they might require
         # plugins that are not yet configured.
         return
@@ -137,27 +156,13 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
         print(ctx.get_help())
         sys.exit(0)
 
-    manifest = JoltManifest()
-    utils.call_and_catch(manifest.parse)
-    manifest.process_import()
-    ctx.obj["manifest"] = manifest
-
-    if manifest.version:
-        from jolt.version_utils import requirement, version
-        req = requirement(manifest.version)
-        ver = version(__version__)
-        raise_error_if(not req.satisfied(ver),
-                       "This project requires Jolt version {} (running {})",
-                       req, __version__)
-
+    registry = TaskRegistry.get()
     loader = JoltLoader.get()
-    tasks = loader.load()
-    for cls in tasks:
-        TaskRegistry.get().add_task_class(cls)
+    loader.load(registry)
 
     if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir:
         ctx.obj["workspace_lock"] = utils.LockFile(
-            fs.path.join(loader.joltdir, "build"),
+            loader.build_path,
             log.info, "Workspace is locked by another process, please wait...")
         atexit.register(ctx.obj["workspace_lock"].close)
 
@@ -166,8 +171,8 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
     if ctx.invoked_subcommand is None:
         task = config.get("jolt", "default", "default")
         taskname, _ = utils.parse_task_name(task)
-        if TaskRegistry.get().get_task_class(taskname) is not None:
-            ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug,
+        if registry.get_task_class(taskname) is not None:
+            ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug, mute=mute,
                        network=network, local=local, keep_going=keep_going, jobs=jobs)
         else:
             print(cli.get_help(ctx))
@@ -175,11 +180,8 @@ def cli(ctx, verbose, extra_verbose, config_file, debugger, profile,
 
 
 def _autocomplete_tasks(ctx, args, incomplete):
-    manifest = JoltManifest()
-    utils.call_and_catch(manifest.parse)
-    manifest.process_import()
-
-    tasks = JoltLoader.get().load()
+    loader = JoltLoader.get()
+    tasks = loader.load()
     tasks = [task.name for task in tasks if task.name.startswith(incomplete or '')]
     return sorted(tasks)
 
@@ -197,10 +199,14 @@ def _autocomplete_tasks(ctx, args, incomplete):
 @click.option("-l", "--local", is_flag=True, default=False, help="Disable remote cache access.")
 @click.option("-n", "--network", is_flag=True, default=False, help="Distribute tasks to network workers.")
 @click.option("-s", "--salt", type=str, help="Add salt as hash influence for all tasks in dependency tree.", metavar="SALT")
+@click.option("-m", "--mute", is_flag=True, help="Display task log only if it fails.")
+@click.option("-v", "--verbose", count=True, help="Verbose output (repeat to raise verbosity).")
 @click.option("--result", type=click.Path(), hidden=True,
               help="Write result manifest to this file.")
 @click.option("--no-download", is_flag=True, default=False,
-              help="Don't download artifacts from remote storage")
+              help="Don't download any artifacts from remote storage")
+@click.option("--no-download-persistent", is_flag=True, default=False,
+              help="Don't download persistent artifacts from remote storage (only session artifacts)")
 @click.option("--no-upload", is_flag=True, default=False,
               help="Don't upload artifacts to remote storage")
 @click.option("--download", is_flag=True, default=False,
@@ -211,11 +217,13 @@ def _autocomplete_tasks(ctx, args, incomplete):
               help="Don't prune cached artifacts from the build graph. This option can be used to populate the local cache with remotely cached dependency artifacts.")
 @click.option("--worker", is_flag=True, default=False,
               help="Run with the worker build strategy", hidden=True)
+@click.option("--environ", type=click.Path(), help="Import build environment from protobuf", hidden=True)
 @click.pass_context
 @hooks.cli_build
 def build(ctx, task, network, keep_going, default, local,
-          no_download, no_upload, download, upload, worker, force,
-          salt, copy, debug, result, jobs, no_prune):
+          no_download, no_download_persistent, no_upload, download, upload, worker, force,
+          salt, copy, debug, result, jobs, no_prune, verbose,
+          mute, environ):
     """
     Build task artifact.
 
@@ -247,6 +255,7 @@ def build(ctx, task, network, keep_going, default, local,
     are removed before execution starts.
 
     """
+
     raise_error_if(network and local,
                    "The -n and -l flags are mutually exclusive")
 
@@ -259,8 +268,14 @@ def build(ctx, task, network, keep_going, default, local,
     raise_error_if(no_upload and upload,
                    "The --upload and --no-upload flags are mutually exclusive")
 
-    duration = utils.duration()
+    if verbose >= 3:
+        log.set_level(log.EXCEPTION)
+    elif verbose >= 2:
+        log.set_level(log.DEBUG)
+    elif verbose >= 1:
+        log.set_level(log.VERBOSE)
 
+    ts_start = utils.duration()
     task = list(task)
     task = [utils.stable_task_name(t) for t in task]
 
@@ -270,6 +285,7 @@ def build(ctx, task, network, keep_going, default, local,
     else:
         _download = config.getboolean("jolt", "download", True)
         _upload = config.getboolean("jolt", "upload", True)
+    _download_session = _download
 
     if local:
         _download = False
@@ -277,23 +293,60 @@ def build(ctx, task, network, keep_going, default, local,
     else:
         if no_download:
             _download = False
+            _download_session = False
+        if no_download_persistent:
+            _download = False
         if no_upload:
             _upload = False
         if download:
             _download = True
+            _download_session = True
         if upload:
             _upload = True
 
-    options = JoltOptions(network=network,
-                          local=local,
-                          download=_download,
-                          upload=_upload,
-                          keep_going=keep_going,
-                          default=default,
-                          worker=worker,
-                          debug=debug,
-                          salt=salt,
-                          jobs=jobs)
+    if keep_going:
+        config.set_keep_going(True)
+
+    # Import build environment from protobuf if provided
+    buildenv = None
+    if environ:
+        with open(environ, "rb") as f:
+            from jolt import common_pb2 as common_pb
+            buildenv = common_pb.BuildEnvironment()
+            try:
+                buildenv.ParseFromString(f.read())
+            except Exception as e:
+                raise_error("Failed to parse build environment protobuf: {}", e)
+
+        # Import log level
+        log.set_level_pb(buildenv.loglevel)
+
+        # Import workspace
+        import_workspace(buildenv)
+
+        # Import configuration snippet
+        config.import_config(buildenv.config)
+
+        # Import configuration parameters (-c params.key)
+        config.import_params({param.key: param.value for param in buildenv.parameters})
+
+        # Import default parameters (-d taskname:param=value)
+        default = utils.as_list(default)
+        default += buildenv.task_default_parameters
+
+    options = JoltOptions(
+        network=network,
+        local=local,
+        download=_download,
+        download_session=_download_session,
+        upload=_upload,
+        keep_going=keep_going,
+        default=default,
+        worker=worker,
+        debug=debug,
+        salt=salt,
+        jobs=jobs,
+        mute=mute)
 
     acache = cache.ArtifactCache.get(options)
 
@@ -315,29 +368,29 @@ def build(ctx, task, network, keep_going, default, local,
     for params in default:
         registry.set_default_parameters(params)
 
-    manifest = ctx.obj["manifest"]
+    log.info("Started: {}", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
 
-    for mb in manifest.builds:
-        for mt in mb.tasks:
-            task.append(mt.name)
-        for mt in mb.defaults:
-            registry.set_default_parameters(mt.name)
+    gb = graph.GraphBuilder(registry, acache, options, progress=True, buildenv=buildenv)
+    dag = gb.build(task)
 
+    # If asked to force rebuild, taint all goal tasks
     if force:
-        for goal in task:
-            registry.get_task(goal, manifest=manifest).taint = uuid.uuid4()
+        for goal in dag.goals:
+            goal.get_extended_task().taint()
 
-    gb = graph.GraphBuilder(registry, manifest, options, progress=True)
-    dag = gb.build(task)
+    # Collect information about artifact presence before starting prune or build
+    acache.precheck(dag.persistent_artifacts, remote=not local)
 
+    # Prune the graph to remove tasks that are already available locally or remotely
     if not no_prune:
-        gp = graph.GraphPruner(strategy)
+        gp = graph.GraphPruner(acache, strategy)
         dag = gp.prune(dag)
 
     goal_tasks = dag.goals
     goal_task_duration = 0
 
-    queue = scheduler.TaskQueue(strategy)
+    session = executors.create_session(dag) if options.network else {}
+    queue = scheduler.TaskQueue()
 
     try:
         if not dag.has_tasks():
@@ -351,16 +404,20 @@ def build(ctx, task, network, keep_going, default, local,
             debug=debug)
 
         with progress:
-            while dag.has_tasks():
+            in_progress = set()
+
+            while dag.has_tasks() or not queue.empty():
                 # Find all tasks ready to be executed
-                leafs = dag.select(lambda graph, task: task.is_ready())
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
                 # Order the tasks by their weights to improve build times
                 leafs.sort(key=lambda x: x.weight)
 
                 while leafs:
                     task = leafs.pop()
-                    queue.submit(acache, task)
+                    executor = strategy.create_executor(session, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
 
                 task, error = queue.wait()
 
@@ -370,6 +427,7 @@ def build(ctx, task, network, keep_going, default, local,
                 elif task.is_goal() and task.duration_running:
                     goal_task_duration += task.duration_running.seconds
 
+                # Unpack tasks with overridden unpack() method
                 if not task.is_resource():
                     if no_prune and task.task.unpack.__func__ is not Task.unpack:
                         with acache.get_context(task):
@@ -377,35 +435,50 @@ def build(ctx, task, network, keep_going, default, local,
 
                     progress.update(1)
 
+                if no_prune and task.is_workspace_resource():
+                    task.task.acquire_ws(force=True)
+
                 if not keep_going and error is not None:
                     queue.abort()
+                    executors.shutdown()
+                    task.raise_for_status()
                     raise error
 
-        if dag.failed:
+        if dag.failed or dag.unstable:
             log.error("List of failed tasks")
-            for failed in dag.failed:
+            for failed in dag.failed + dag.unstable:
                 log.error("- {}", failed.log_name.strip("()"))
+
+        for failed_task in dag.failed:
+            failed_task.raise_for_status()
+        if dag.failed:
             raise_error("No more tasks could be executed")
 
-        for goal in goal_tasks:
-            if acache.is_available_locally(goal):
-                with acache.get_artifact(goal) as artifact:
-                    log.info("Location: {0}", artifact.path)
-                    if copy:
-                        artifact.copy("*", utils.as_dirpath(fs.path.join(workdir, click.format_filename(copy))), symlinks=True)
     except KeyboardInterrupt:
         print()
         log.warning("Interrupted by user")
         try:
             queue.abort()
+            executors.shutdown()
             sys.exit(1)
         except KeyboardInterrupt:
             print()
             log.warning("Interrupted again, exiting")
-            _exit(1)
+            os._exit(1)
     finally:
+        queue.shutdown()
+
+        for task in goal_tasks:
+            for artifact in task.artifacts:
+                if acache.is_available_locally(artifact):
+                    log.info("Location: {0}", artifact.path)
+                    if copy:
+                        dst = utils.as_dirpath(fs.path.join(workdir, click.format_filename(copy)))
+                        artifact.copy("*", dst, symlinks=True)
+
+        log.info("Ended: {}", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
         log.info("Total execution time: {0} {1}",
-                 str(duration),
+                 str(ts_start),
                  str(queue.duration_acc) if network else '')
         if result:
             with report.update() as manifest:
@@ -441,7 +514,7 @@ def clean(ctx, task, deps, expired):
     if task:
         task = [utils.stable_task_name(t) for t in task]
         registry = TaskRegistry.get()
-        dag = graph.GraphBuilder(registry, ctx.obj["manifest"]).build(task)
+        dag = graph.GraphBuilder(registry, acache).build(task)
         if deps:
             tasks = dag.tasks
         else:
@@ -576,20 +649,20 @@ def _config(ctx, list, delete, global_, user, key, value):
 
 @cli.command()
 @click.argument("task", type=str, nargs=-1, required=False, shell_complete=_autocomplete_tasks)
-@click.option("-r", "--reverse", type=str, help="Display consumers of REVERSE if TASK is executed.")
 @click.option("-c", "--cached", "show_cache", is_flag=True, help="Highlight cache presence with colors.")
+@click.option("-r", "--reverse", type=str, help="Display consumers of REVERSE if TASK is executed.")
+@click.option("-p", "--prune", "prune", is_flag=True, help="Do not repeat tasks. An already visited task's name is printed inside [square brackets] and its deps are omitted.")
 @click.pass_context
-def display(ctx, task, reverse=None, show_cache=False):
+def display(ctx, task, reverse=None, show_cache=False, prune=False):
     """
     Display a task and its dependencies visually.
 
     """
     registry = TaskRegistry.get()
-    gb = graph.GraphBuilder(registry, ctx.obj["manifest"])
-    dag = gb.build(task, influence=show_cache)
-
     options = JoltOptions()
     acache = cache.ArtifactCache.get(options)
+    gb = graph.GraphBuilder(registry, acache)
+    dag = gb.build(task, influence=show_cache)
 
     if reverse:
         def iterator(task):
@@ -604,6 +677,8 @@ def display(ctx, task, reverse=None, show_cache=False):
         tasklist = dag.requested_goals
 
     if dag.has_tasks():
+        processed = set()
+
         def _display(task, indent=0, last=None):
             header = ""
             if indent > 0:
@@ -624,10 +699,22 @@ def display(ctx, task, reverse=None, show_cache=False):
             else:
                 colorize = colors.green
 
-            print(header + colorize(task.short_qualified_name))
+            if prune and task.short_qualified_name in processed:
+                def prune_marker(n):
+                    return "[" + n + "]"
+            else:
+                def prune_marker(n):
+                    return n
+
+            print(header + colorize(prune_marker(task.short_qualified_name)))
             children = iterator(task)
-            for i in range(0, len(children)):
-                _display(children[i], indent + 1, last=(last or []) + [i + 1 != len(children)])
+            if not prune or task.short_qualified_name not in processed:
+                for i in range(0, len(children)):
+                    _display(children[i], indent + 1, last=(last or []) + [i + 1 != len(children)])
+
+            if prune:
+                processed.add(task.short_qualified_name)
+
         for task in tasklist:
             _display(task)
     else:
@@ -639,58 +726,103 @@ def docs():
     """
     Opens the Jolt documentation in the default webbrowser.
     """
-    webbrowser.open(config.get("jolt", "docs", "http://jolt.readthedocs.io/"))
+    url = config.get("jolt", "docs", "http://jolt.readthedocs.io/")
+    success = False
+    try:
+        success = webbrowser.open(url)
+    except Exception:
+        pass
+    if not success:
+        print(f"Failed to open web browser. Visit {url} manually.")
 
 
-@cli.command(hidden=True)
-@click.argument("task", type=str, nargs=-1, required=True)
-@click.option("-r", "--remove", is_flag=True, help="Remove tasks from existing manifest.")
-@click.option("-d", "--default", type=str, multiple=True, help="Override default parameter values.")
-@click.option("-o", "--output", type=str, default="default.joltxmanifest", help="Manifest filename.")
+@cli.command()
+@click.argument("task", type=str, nargs=-1, required=True, shell_complete=_autocomplete_tasks)
+@click.option("-c", "--copy", type=click.Path(),
+              help="Copy artifact content to directory PATH.")
+@click.option("-ca", "--copy-all", type=click.Path(), help="Copy artifacts, including dependency artifacts, to directory PATH. Implies --deps.")
+@click.option("-d", "--deps", is_flag=True, help="Download dependencies.")
 @click.pass_context
-def freeze(ctx, task, default, output, remove):
+@hooks.cli_download
+def download(ctx, task, deps, copy, copy_all):
     """
-    Freeze the identity of a task.
+    Download task artifacts from remote caches.
+
+    No attempt to build the artifact is made if it is not present
+    in configured remote caches.
 
-    <WIP>
+    Task dependencies may optionally be downloaded by passing the -d option.
     """
-    manifest = ctx.obj["manifest"]
 
-    options = JoltOptions(default=default)
+    raise_error_if(copy and copy_all, "--copy and --copy-all are mutually exclusive")
+    if copy_all:
+        deps = True
+
+    options = JoltOptions()
     acache = cache.ArtifactCache.get(options)
-    scheduler.ExecutorRegistry.get(options)
+    hooks.TaskHookRegistry.get(options)
+    executors = scheduler.ExecutorRegistry.get(options)
     registry = TaskRegistry.get()
+    strategy = scheduler.DownloadStrategy(executors, acache)
+    queue = scheduler.TaskQueue()
+    gb = graph.GraphBuilder(registry, acache, options, progress=True)
+    dag = gb.build(task)
 
-    for params in default:
-        registry.set_default_parameters(params)
+    if not deps:
+        for task in dag.tasks:
+            if not task.is_goal():
+                task.pruned()
 
-    gb = graph.GraphBuilder(registry, manifest)
-    dag = gb.build(task)
+    all_tasks = dag.tasks
+    goal_tasks = dag.goals
 
-    available_in_cache = [
-        (t.is_available_locally(acache) or (
-            t.is_available_remotely(acache) and acache.download_enabled()), t)
-        for t in dag.tasks if t.is_cacheable()]
+    try:
+        with log.progress("Progress", dag.number_of_tasks(), " tasks", estimates=False, debug=False) as p:
+            in_progress = set()
 
-    for available, task in available_in_cache:
-        raise_task_error_if(
-            not remove and not available, task,
-            "Task artifact is not available in any cache, build it first")
+            while dag.has_tasks() or not queue.empty():
+                leafs = dag.select(lambda graph, task: task.is_ready() and task not in in_progress)
 
-    for task in dag.tasks:
-        if task.is_resource() or not task.is_cacheable():
-            continue
-        manifest_task = manifest.find_task(task)
-        if remove and manifest_task:
-            manifest.remove_task(manifest_task)
-            continue
-        if not remove:
-            if not manifest_task:
-                manifest_task = manifest.create_task()
-            manifest_task.name = task.qualified_name
-            manifest_task.identity = task.identity
+                while leafs:
+                    task = leafs.pop()
+                    executor = strategy.create_executor({}, task)
+                    queue.submit(executor)
+                    in_progress.add(task)
 
-    manifest.write(fs.path.join(JoltLoader.get().joltdir, output))
+                task, error = queue.wait()
+                p.update(1)
+
+        copy_tasks = goal_tasks if not copy_all else all_tasks
+        for goal in copy_tasks:
+            if goal.is_available_locally():
+                for artifact in goal.artifacts:
+                    if copy:
+                        log.info("Copying: {0}", artifact.path)
+                        artifact.copy("*", utils.as_dirpath(fs.path.join(workdir, click.format_filename(copy))), symlinks=True)
+                    elif copy_all:
+                        log.info("Copying: {0}", artifact.path)
+                        artifact.copy("*", utils.as_dirpath(fs.path.join(workdir, click.format_filename(copy_all))), symlinks=True)
+                    elif goal.is_goal():
+                        log.info("Location: {0}", artifact.path)
+
+    except KeyboardInterrupt:
+        print()
+        log.warning("Interrupted by user")
+        try:
+            queue.abort()
+            executors.shutdown()
+            sys.exit(1)
+        except KeyboardInterrupt:
+            print()
+            log.warning("Interrupted again, exiting")
+            os._exit(1)
+
+    except Exception as e:
+        log.set_interactive(True)
+        raise e
+
+    finally:
+        queue.shutdown()
 
 
 @cli.command(name="list")
@@ -713,6 +845,8 @@ def _list(ctx, task=None, all=False, reverse=None):
 
     raise_error_if(not task and reverse, "TASK required with --reverse")
 
+    options = JoltOptions()
+    acache = cache.ArtifactCache.get(options)
     registry = TaskRegistry.get()
 
     if not task:
@@ -726,7 +860,7 @@ def _list(ctx, task=None, all=False, reverse=None):
     reverse = [utils.stable_task_name(t) for t in utils.as_list(reverse or [])]
 
     try:
-        dag = graph.GraphBuilder(registry, ctx.obj["manifest"]).build(task, influence=False)
+        dag = graph.GraphBuilder(registry, acache).build(task, influence=False)
     except JoltError as e:
         raise e
     except Exception:
@@ -760,16 +894,21 @@ def _log(follow, delete):
     Display the Jolt log file.
 
     """
+    if not log.logfiles:
+        print("No logs exist")
+        return
+
     if follow:
-        subprocess.call("tail -f {0}".format(logfile), shell=True)
+        subprocess.call("tail -f {0}".format(log.logfiles[-1]), shell=True)
     elif delete:
-        fs.unlink(logfile)
+        for file in log.logfiles:
+            fs.unlink(file)
     else:
         t = tools.Tools()
-        configured_pager = config.get("jolt", "pager", environ.get("PAGER", None))
+        configured_pager = config.get("jolt", "pager", os.environ.get("PAGER", None))
         for pager in [configured_pager, "less", "more", "cat"]:
             if pager and t.which(pager):
-                return subprocess.call("{1} {0}".format(logfile, pager), shell=True)
+                return subprocess.call("{1} {0}".format(log.logfiles[-1], pager), shell=True)
         print(t.read_file(logfile))
 
 
@@ -821,9 +960,8 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
 
     print()
     print("  Requirements")
-    manifest = ctx.obj["manifest"]
     try:
-        task = task_registry.get_task(task_name, manifest=manifest)
+        task = task_registry.get_task(task_name)
         for req in sorted(utils.as_list(utils.call_or_return(task, task.requires))):
             print("    {0}".format(task.tools.expand(req)))
         if not task.requires:
@@ -843,8 +981,9 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
         task.taint = salt
 
     if artifact:
+        options = JoltOptions(salt=salt)
         acache = cache.ArtifactCache.get()
-        builder = graph.GraphBuilder(task_registry, manifest)
+        builder = graph.GraphBuilder(task_registry, acache, options)
         dag = builder.build([task.qualified_name])
         tasks = dag.select(lambda graph, node: node.task is task)
         assert len(tasks) == 1, "graph produced multiple tasks, one expected"
@@ -853,14 +992,14 @@ def inspect(ctx, task, influence=False, artifact=False, salt=None):
 
         print("  Cache")
         print("    Identity          {0}".format(proxy.identity))
-        if acache.is_available_locally(proxy):
-            with acache.get_artifact(proxy) as artifact:
+        if proxy.is_available_locally():
+            for artifact in filter(lambda a: not a.is_session(), proxy.artifacts):
                 print("    Location          {0}".format(artifact.path))
             print("    Local             True ({0})".format(
-                utils.as_human_size(acache.get_artifact(proxy).get_size())))
+                utils.as_human_size(sum([artifact.get_size() for artifact in proxy.artifacts]))))
         else:
             print("    Local             False")
-        print("    Remote            {0}".format(acache.is_available_remotely(proxy)))
+        print("    Remote            {0}".format(proxy.is_available_remotely(cache=False)))
         print()
 
     if influence:
@@ -908,10 +1047,10 @@ def _export(ctx, task):
     executors = scheduler.ExecutorRegistry.get()
     strategy = scheduler.LocalStrategy(executors, acache)
 
-    dag = graph.GraphBuilder(registry, ctx.obj["manifest"])
+    dag = graph.GraphBuilder(registry, acache)
     dag = dag.build(task)
 
-    gp = graph.GraphPruner(strategy)
+    gp = graph.GraphPruner(acache, strategy)
     dag = gp.prune(dag)
 
     class Export(object):
@@ -936,17 +1075,97 @@ def _export(ctx, task):
     context = Context(tasks)
 
     for task in context.tasks:
-        artifact = acache.get_artifact(task)
-        raise_task_error_if(
-            artifact.is_temporary(), task,
-            "Task artifact not found in local cache, build it first")
+        for artifact in task.artifacts:
+            raise_task_error_if(
+                not task.is_resource() and artifact.is_temporary(), task,
+                "Task artifact not found in local cache, build it first")
 
-        visitor = Export()
-        cache.visit_artifact(task, artifact, visitor)
-        context.add_export(task, visitor)
+            visitor = Export()
+            cache.visit_artifact(task, artifact, visitor)
+            context.add_export(task, visitor)
 
     script = utils.render(
         "export.sh.template",
         ctx=context)
 
     print(script)
+
+
+@cli.command(name="report")
+@click.pass_context
+@hooks.cli_report
+def _report(ctx):
+    """Create and publish a report with system information and logs.
+
+    The purpose of the report command is to aid troubleshooting by
+    providing users a method to quickly generate and share a report
+    with information about the environment and previous activity.
+
+    The report is uploaded to remote caches if configured, otherwise
+    a report archive is created in the current working directory.
+    """
+
+    options = JoltOptions()
+    acache = cache.ArtifactCache.get(options)
+
+    with tools.Tools() as t:
+
+        artifact = cache.Artifact(
+            acache,
+            None,
+            identity=str(uuid.uuid4()),
+            name="jolt",
+            session=True,
+            tools=t)
+
+        with t.tmpdir("report") as tmp, t.cwd(tmp):
+            log.info("Collecting environment")
+            env = ""
+            for key, val in os.environ.items():
+                env += f"{key} = {val}\n"
+            t.write_file("environ.txt", env, expand=False)
+
+            log.info("Collecting configuration")
+            config.save(tmp)
+            artifact.collect("*.conf", "configs/")
+
+            log.info("Collecting platform information")
+            uname = platform.uname()
+            libc = " ".join(platform.libc_ver())
+            pyver = platform.python_version()
+            pyimpl = platform.python_implementation()
+            pfm = f"""System: {uname.system}
+Node: {uname.node}
+Release: {uname.release}
+Version: {uname.version}
+Machine: {uname.machine}
+Libc: {libc}
+Python: {pyimpl} {pyver}
+"""
+            t.write_file("platform.txt", pfm, expand=False)
+
+            def collect_command(what, cmd):
+                try:
+                    log.info("Collecting {}", what)
+                    t.run(cmd, output=False)
+                except Exception:
+                    pass
+
+            collect_command("pip modules", "pip list > packages-pip.txt")
+            if uname.system == "Linux":
+                collect_command("apt packages", "apt list --installed > packages-apt.txt")
+                artifact.collect("/etc/os-release", "os-release.txt")
+            artifact.collect("*.txt")
+
+        with t.cwd(log.logpath):
+            artifact.collect("*.log", "logs/")
+
+        acache.commit(artifact)
+        if acache.upload_enabled():
+            assert acache.upload(artifact)
+            url = acache.location(artifact)
+        else:
+            url = f"{artifact.identity}.tgz"
+            t.archive(artifact.path, url)
+
+        log.info("Location: {}", url)