ob-metaflow 2.11.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/cli.py

@@ -1,55 +1,48 @@
+import os
+import functools
 import inspect
+import os
 import sys
 import traceback
 from datetime import datetime
-from functools import wraps
-import metaflow.tracing as tracing
 
+import metaflow.tracing as tracing
 from metaflow._vendor import click
 
-from . import lint
-from . import plugins
-from . import parameters
-from . import decorators
-from . import metaflow_version
-from . import namespace
-from .metaflow_current import current
+from . import decorators, lint, metaflow_version, parameters, plugins
 from .cli_args import cli_args
-from .tagging_util import validate_tags
-from .util import (
-    resolve_identity,
-    decompress_list,
-    write_latest_run_id,
-    get_latest_run_id,
-)
-from .task import MetaflowTask
+from .cli_components.utils import LazyGroup, LazyPluginCommandCollection
+from .datastore import FlowDataStore, TaskDataStoreSet
+from .debug import debug
 from .exception import CommandException, MetaflowException
+from .flowspec import FlowStateItems
 from .graph import FlowGraph
-from .datastore import FlowDataStore, TaskDataStoreSet, TaskDataStore
-
-from .runtime import NativeRuntime
-from .package import MetaflowPackage
-from .plugins import (
-    DATASTORES,
-    ENVIRONMENTS,
-    LOGGING_SIDECARS,
-    METADATA_PROVIDERS,
-    MONITOR_SIDECARS,
-)
 from .metaflow_config import (
     DEFAULT_DATASTORE,
+    DEFAULT_DECOSPECS,
     DEFAULT_ENVIRONMENT,
     DEFAULT_EVENT_LOGGER,
     DEFAULT_METADATA,
     DEFAULT_MONITOR,
     DEFAULT_PACKAGE_SUFFIXES,
 )
+from .metaflow_current import current
+from .metaflow_profile import from_start
+from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
+from .packaging_sys import MetaflowCodeContent
+from .plugins import (
+    DATASTORES,
+    ENVIRONMENTS,
+    LOGGING_SIDECARS,
+    METADATA_PROVIDERS,
+    MONITOR_SIDECARS,
+)
 from .pylint_wrapper import PyLint
-from .R import use_r, metaflow_r_version
-from .mflog import mflog, LOG_SOURCES
-from .unbounded_foreach import UBF_CONTROL, UBF_TASK
-
+from .R import metaflow_r_version, use_r
+from .util import get_latest_run_id, resolve_identity, decompress_list
+from .user_configs.config_options import LocalFileInput, config_options
+from .user_configs.config_parameters import ConfigValue
 
 ERASE_TO_EOL = "\033[K"
 HIGHLIGHT = "red"
@@ -59,19 +52,21 @@ LOGGER_TIMESTAMP = "magenta"
 LOGGER_COLOR = "green"
 LOGGER_BAD_COLOR = "red"
 
-try:
-    # Python 2
-    import cPickle as pickle
-except ImportError:
-    # Python 3
-    import pickle
-
 
 def echo_dev_null(*args, **kwargs):
     pass
 
 
 def echo_always(line, **kwargs):
+    if kwargs.pop("wrap", False):
+        import textwrap
+
+        indent_str = INDENT if kwargs.get("indent", None) else ""
+        effective_width = 80 - len(indent_str)
+        wrapped = textwrap.wrap(line, width=effective_width, break_long_words=False)
+        line = "\n".join(indent_str + l for l in wrapped)
+        kwargs["indent"] = False
+
     kwargs["err"] = kwargs.get("err", True)
     if kwargs.pop("indent", None):
         line = "\n".join(INDENT + x for x in line.splitlines())
@@ -124,7 +119,18 @@ def logger(body="", system_msg=False, head="", bad=False, timestamp=True, nl=Tru
     click.secho(body, bold=system_msg, fg=LOGGER_BAD_COLOR if bad else None, nl=nl)
 
 
-@click.group()
+@click.group(
+    cls=LazyGroup,
+    lazy_subcommands={
+        "init": "metaflow.cli_components.init_cmd.init",
+        "dump": "metaflow.cli_components.dump_cmd.dump",
+        "step": "metaflow.cli_components.step_cmd.step",
+        "run": "metaflow.cli_components.run_cmds.run",
+        "resume": "metaflow.cli_components.run_cmds.resume",
+        "spin": "metaflow.cli_components.run_cmds.spin",
+        "spin-step": "metaflow.cli_components.step_cmd.spin_step",
+    },
+)
 def cli(ctx):
     pass
 
@@ -138,7 +144,13 @@ def cli(ctx):
 )
 @click.pass_obj
 def check(obj, warnings=False):
-    _check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint, warnings=warnings)
+    if obj.is_quiet:
+        echo = echo_dev_null
+    else:
+        echo = echo_always
+    _check(
+        echo, obj.graph, obj.flow, obj.environment, pylint=obj.pylint, warnings=warnings
+    )
     fname = inspect.getfile(obj.flow.__class__)
     echo(
         "\n*'{cmd} show'* shows a description of this flow.\n"
@@ -153,8 +165,14 @@ def check(obj, warnings=False):
 @click.pass_obj
 def show(obj):
     echo_always("\n%s" % obj.graph.doc)
-    for _, node in sorted((n.func_lineno, n) for n in obj.graph):
-        echo_always("\nStep *%s*" % node.name, err=False)
+    for node_name in obj.graph.sorted_nodes:
+        echo_always("")
+        node = obj.graph[node_name]
+        for deco in node.decorators:
+            echo_always("@%s" % deco.name, err=False)
+        for deco in node.wrappers:
+            echo_always("@%s" % deco.decorator_name, err=False)
+        echo_always("Step *%s*" % node.name, err=False)
         echo_always(node.doc if node.doc else "?", indent=True, err=False)
         if node.type != "end":
             echo_always(
@@ -188,15 +206,15 @@ def output_raw(obj, json):
     else:
         _graph = str(obj.graph)
         _msg = "Internal representation of the flow:"
-    echo(_msg, fg="magenta", bold=False)
+    echo_always(_msg, fg="magenta", bold=False)
     echo_always(_graph, err=False)
 
 
 @cli.command(help="Visualize the flow with Graphviz.")
 @click.pass_obj
 def output_dot(obj):
-    echo("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False)
-    echo(
+    echo_always("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False)
+    echo_always(
         "Try piping the output to 'dot -Tpng -o graph.png' to produce "
         "an actual image.",
         indent=True,
@@ -204,712 +222,31 @@ def output_dot(obj):
     echo_always(obj.graph.output_dot(), err=False)
 
 
-@cli.command(
-    help="Get data artifacts of a task or all tasks in a step. "
-    "The format for input-path is either <run_id>/<step_name> or "
-    "<run_id>/<step_name>/<task_id>."
-)
-@click.argument("input-path")
-@click.option(
-    "--private/--no-private",
-    default=False,
-    show_default=True,
-    help="Show also private attributes.",
-)
-@click.option(
-    "--max-value-size",
-    default=1000,
-    show_default=True,
-    type=int,
-    help="Show only values that are smaller than this number. "
-    "Set to 0 to see only keys.",
-)
-@click.option(
-    "--include",
-    type=str,
-    default="",
-    help="Include only artifacts in the given comma-separated list.",
-)
-@click.option(
-    "--file", type=str, default=None, help="Serialize artifacts in the given file."
-)
-@click.pass_obj
-def dump(obj, input_path, private=None, max_value_size=None, include=None, file=None):
-    output = {}
-    kwargs = {
-        "show_private": private,
-        "max_value_size": max_value_size,
-        "include": {t for t in include.split(",") if t},
-    }
-
-    # Pathspec can either be run_id/step_name or run_id/step_name/task_id.
-    parts = input_path.split("/")
-    if len(parts) == 2:
-        run_id, step_name = parts
-        task_id = None
-    elif len(parts) == 3:
-        run_id, step_name, task_id = parts
-    else:
-        raise CommandException(
-            "input_path should either be run_id/step_name or run_id/step_name/task_id"
-        )
-
-    datastore_set = TaskDataStoreSet(
-        obj.flow_datastore,
-        run_id,
-        steps=[step_name],
-        prefetch_data_artifacts=kwargs.get("include"),
-    )
-    if task_id:
-        ds_list = [datastore_set.get_with_pathspec(input_path)]
-    else:
-        ds_list = list(datastore_set)  # get all tasks
-
-    for ds in ds_list:
-        echo(
-            "Dumping output of run_id=*{run_id}* "
-            "step=*{step}* task_id=*{task_id}*".format(
-                run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id
-            ),
-            fg="magenta",
-        )
-
-        if file is None:
-            echo_always(
-                ds.format(**kwargs), highlight="green", highlight_bold=False, err=False
-            )
-        else:
-            output[ds.pathspec] = ds.to_dict(**kwargs)
-
-    if file is not None:
-        with open(file, "wb") as f:
-            pickle.dump(output, f, protocol=pickle.HIGHEST_PROTOCOL)
-        echo("Artifacts written to *%s*" % file)
-
-
-@cli.command(
-    help="Show stdout/stderr produced by a task or all tasks in a step. "
-    "The format for input-path is either <run_id>/<step_name> or "
-    "<run_id>/<step_name>/<task_id>."
-)
-@click.argument("input-path")
-@click.option(
-    "--stdout/--no-stdout",
-    default=False,
-    show_default=True,
-    help="Show stdout of the task.",
-)
-@click.option(
-    "--stderr/--no-stderr",
-    default=False,
-    show_default=True,
-    help="Show stderr of the task.",
-)
-@click.option(
-    "--both/--no-both",
-    default=True,
-    show_default=True,
-    help="Show both stdout and stderr of the task.",
-)
-@click.option(
-    "--timestamps/--no-timestamps",
-    default=False,
-    show_default=True,
-    help="Show timestamps.",
-)
-@click.pass_obj
-def logs(obj, input_path, stdout=None, stderr=None, both=None, timestamps=False):
-    types = set()
-    if stdout:
-        types.add("stdout")
-        both = False
-    if stderr:
-        types.add("stderr")
-        both = False
-    if both:
-        types.update(("stdout", "stderr"))
-
-    streams = list(sorted(types, reverse=True))
-
-    # Pathspec can either be run_id/step_name or run_id/step_name/task_id.
-    parts = input_path.split("/")
-    if len(parts) == 2:
-        run_id, step_name = parts
-        task_id = None
-    elif len(parts) == 3:
-        run_id, step_name, task_id = parts
-    else:
-        raise CommandException(
-            "input_path should either be run_id/step_name "
-            "or run_id/step_name/task_id"
-        )
-
-    datastore_set = TaskDataStoreSet(
-        obj.flow_datastore, run_id, steps=[step_name], allow_not_done=True
-    )
-    if task_id:
-        ds_list = [
-            TaskDataStore(
-                obj.flow_datastore,
-                run_id=run_id,
-                step_name=step_name,
-                task_id=task_id,
-                mode="r",
-                allow_not_done=True,
-            )
-        ]
-    else:
-        ds_list = list(datastore_set)  # get all tasks
-
-    if ds_list:
-
-        def echo_unicode(line, **kwargs):
-            click.secho(line.decode("UTF-8", errors="replace"), **kwargs)
-
-        # old style logs are non mflog-style logs
-        maybe_old_style = True
-        for ds in ds_list:
-            echo(
-                "Dumping logs of run_id=*{run_id}* "
-                "step=*{step}* task_id=*{task_id}*".format(
-                    run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id
-                ),
-                fg="magenta",
-            )
-
-            for stream in streams:
-                echo(stream, bold=True)
-                logs = ds.load_logs(LOG_SOURCES, stream)
-                if any(data for _, data in logs):
-                    # attempt to read new, mflog-style logs
-                    for line in mflog.merge_logs([blob for _, blob in logs]):
-                        if timestamps:
-                            ts = mflog.utc_to_local(line.utc_tstamp)
-                            tstamp = ts.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
-                            click.secho(tstamp + " ", fg=LOGGER_TIMESTAMP, nl=False)
-                        echo_unicode(line.msg)
-                    maybe_old_style = False
-                elif maybe_old_style:
-                    # if they are not available, we may be looking at
-                    # a legacy run (unless we have seen new-style data already
-                    # for another stream). This return an empty string if
-                    # nothing is found
-                    log = ds.load_log_legacy(stream)
-                    if log and timestamps:
-                        raise CommandException(
-                            "We can't show --timestamps for old runs. Sorry!"
-                        )
-                    echo_unicode(log, nl=False)
-    else:
-        raise CommandException(
-            "No Tasks found at the given path -- "
-            "either none exist or none have started yet"
-        )
-
-
-# TODO - move step and init under a separate 'internal' subcommand
-
-
-@cli.command(help="Internal command to execute a single task.", hidden=True)
-@click.argument("step-name")
-@click.option(
-    "--run-id",
-    default=None,
-    required=True,
-    help="ID for one execution of all steps in the flow.",
-)
-@click.option(
-    "--task-id",
-    default=None,
-    required=True,
-    show_default=True,
-    help="ID for this instance of the step.",
-)
-@click.option(
-    "--input-paths",
-    help="A comma-separated list of pathspecs specifying inputs for this step.",
-)
-@click.option(
-    "--input-paths-filename",
-    type=click.Path(exists=True, readable=True, dir_okay=False, resolve_path=True),
-    help="A filename containing the argument typically passed to `input-paths`",
-    hidden=True,
-)
-@click.option(
-    "--split-index",
-    type=int,
-    default=None,
-    show_default=True,
-    help="Index of this foreach split.",
-)
-@click.option(
-    "--tag",
-    "opt_tag",
-    multiple=True,
-    default=None,
-    help="Annotate this run with the given tag. You can specify "
-    "this option multiple times to attach multiple tags in "
-    "the task.",
-)
-@click.option(
-    "--namespace",
-    "opt_namespace",
-    default=None,
-    help="Change namespace from the default (your username) to the specified tag.",
-)
-@click.option(
-    "--retry-count",
-    default=0,
-    help="How many times we have attempted to run this task.",
-)
-@click.option(
-    "--max-user-code-retries",
-    default=0,
-    help="How many times we should attempt running the user code.",
-)
-@click.option(
-    "--clone-only",
-    default=None,
-    help="Pathspec of the origin task for this task to clone. Do "
-    "not execute anything.",
-)
-@click.option(
-    "--clone-run-id",
-    default=None,
-    help="Run id of the origin flow, if this task is part of a flow being resumed.",
-)
-@click.option(
-    "--with",
-    "decospecs",
-    multiple=True,
-    help="Add a decorator to this task. You can specify this "
-    "option multiple times to attach multiple decorators "
-    "to this task.",
-)
-@click.option(
-    "--ubf-context",
-    default="none",
-    type=click.Choice(["none", UBF_CONTROL, UBF_TASK]),
-    help="Provides additional context if this task is of type unbounded foreach.",
-)
-@click.option(
-    "--num-parallel",
-    default=0,
-    type=int,
-    help="Number of parallel instances of a step. Ignored in local mode (see parallel decorator code).",
-)
-@click.pass_context
-def step(
-    ctx,
-    step_name,
-    opt_tag=None,
-    run_id=None,
-    task_id=None,
-    input_paths=None,
-    input_paths_filename=None,
-    split_index=None,
-    opt_namespace=None,
-    retry_count=None,
-    max_user_code_retries=None,
-    clone_only=None,
-    clone_run_id=None,
-    decospecs=None,
-    ubf_context="none",
-    num_parallel=None,
-):
-    if ubf_context == "none":
-        ubf_context = None
-    if opt_namespace is not None:
-        namespace(opt_namespace or None)
-
-    func = None
-    try:
-        func = getattr(ctx.obj.flow, step_name)
-    except:
-        raise CommandException("Step *%s* doesn't exist." % step_name)
-    if not func.is_step:
-        raise CommandException("Function *%s* is not a step." % step_name)
-    echo("Executing a step, *%s*" % step_name, fg="magenta", bold=False)
-
-    if decospecs:
-        decorators._attach_decorators_to_step(func, decospecs)
-
-    step_kwargs = ctx.params
-    # Remove argument `step_name` from `step_kwargs`.
-    step_kwargs.pop("step_name", None)
-    # Remove `opt_*` prefix from (some) option keys.
-    step_kwargs = dict(
-        [(k[4:], v) if k.startswith("opt_") else (k, v) for k, v in step_kwargs.items()]
-    )
-    cli_args._set_step_kwargs(step_kwargs)
-
-    ctx.obj.metadata.add_sticky_tags(tags=opt_tag)
-    if not input_paths and input_paths_filename:
-        with open(input_paths_filename, mode="r", encoding="utf-8") as f:
-            input_paths = f.read().strip(" \n\"'")
-
-    paths = decompress_list(input_paths) if input_paths else []
-
-    task = MetaflowTask(
-        ctx.obj.flow,
-        ctx.obj.flow_datastore,
-        ctx.obj.metadata,
-        ctx.obj.environment,
-        ctx.obj.echo,
-        ctx.obj.event_logger,
-        ctx.obj.monitor,
-        ubf_context,
-    )
-    if clone_only:
-        task.clone_only(
-            step_name,
-            run_id,
-            task_id,
-            clone_only,
-            retry_count,
-        )
-    else:
-        task.run_step(
-            step_name,
-            run_id,
-            task_id,
-            clone_run_id,
-            paths,
-            split_index,
-            retry_count,
-            max_user_code_retries,
-        )
-
-    echo("Success", fg="green", bold=True, indent=True)
-
-
-@parameters.add_custom_parameters(deploy_mode=False)
-@cli.command(help="Internal command to initialize a run.", hidden=True)
-@click.option(
-    "--run-id",
-    default=None,
-    required=True,
-    help="ID for one execution of all steps in the flow.",
-)
-@click.option(
-    "--task-id", default=None, required=True, help="ID for this instance of the step."
-)
-@click.option(
-    "--tag",
-    "tags",
-    multiple=True,
-    default=None,
-    help="Tags for this instance of the step.",
-)
-@click.pass_obj
-def init(obj, run_id=None, task_id=None, tags=None, **kwargs):
-    # init is a separate command instead of an option in 'step'
-    # since we need to capture user-specified parameters with
-    # @add_custom_parameters. Adding custom parameters to 'step'
-    # is not desirable due to the possibility of name clashes between
-    # user-specified parameters and our internal options. Note that
-    # user-specified parameters are often defined as environment
-    # variables.
-
-    obj.metadata.add_sticky_tags(tags=tags)
-
-    runtime = NativeRuntime(
-        obj.flow,
-        obj.graph,
-        obj.flow_datastore,
-        obj.metadata,
-        obj.environment,
-        obj.package,
-        obj.logger,
-        obj.entrypoint,
-        obj.event_logger,
-        obj.monitor,
-        run_id=run_id,
-    )
-    obj.flow._set_constants(obj.graph, kwargs)
-    runtime.persist_constants(task_id=task_id)
-
-
-def common_run_options(func):
-    @click.option(
-        "--tag",
-        "tags",
-        multiple=True,
-        default=None,
-        help="Annotate this run with the given tag. You can specify "
-        "this option multiple times to attach multiple tags in "
-        "the run.",
-    )
-    @click.option(
-        "--max-workers",
-        default=16,
-        show_default=True,
-        help="Maximum number of parallel processes.",
-    )
-    @click.option(
-        "--max-num-splits",
-        default=100,
-        show_default=True,
-        help="Maximum number of splits allowed in a foreach. This "
-        "is a safety check preventing bugs from triggering "
-        "thousands of steps inadvertently.",
-    )
-    @click.option(
-        "--max-log-size",
-        default=10,
-        show_default=True,
-        help="Maximum size of stdout and stderr captured in "
-        "megabytes. If a step outputs more than this to "
-        "stdout/stderr, its output will be truncated.",
-    )
-    @click.option(
-        "--with",
-        "decospecs",
-        multiple=True,
-        help="Add a decorator to all steps. You can specify this "
-        "option multiple times to attach multiple decorators "
-        "in steps.",
-    )
-    @click.option(
-        "--run-id-file",
-        default=None,
-        show_default=True,
-        type=str,
-        help="Write the ID of this run to the file specified.",
-    )
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-@click.option(
-    "--origin-run-id",
-    default=None,
-    help="ID of the run that should be resumed. By default, the "
-    "last run executed locally.",
-)
-@click.option(
-    "--run-id",
-    default=None,
-    help="Run ID for the new run. By default, a new run-id will be generated",
-    hidden=True,
-)
-@click.option(
-    "--clone-only/--no-clone-only",
-    default=False,
-    show_default=True,
-    help="Only clone tasks without continuing execution",
-    hidden=True,
-)
-@click.option(
-    "--reentrant/--no-reentrant",
-    default=False,
-    show_default=True,
-    hidden=True,
-    help="If specified, allows this call to be called in parallel",
-)
-@click.option(
-    "--resume-identifier",
-    default=None,
-    show_default=True,
-    hidden=True,
-    help="If specified, it identifies the task that started this resume call. It is in the form of {step_name}-{task_id}",
-)
-@click.argument("step-to-rerun", required=False)
-@cli.command(help="Resume execution of a previous run of this flow.")
-@common_run_options
-@click.pass_obj
-def resume(
-    obj,
-    tags=None,
-    step_to_rerun=None,
-    origin_run_id=None,
-    run_id=None,
-    clone_only=False,
-    reentrant=False,
-    max_workers=None,
-    max_num_splits=None,
-    max_log_size=None,
-    decospecs=None,
-    run_id_file=None,
-    resume_identifier=None,
-):
-    before_run(obj, tags, decospecs + obj.environment.decospecs())
-
-    if origin_run_id is None:
-        origin_run_id = get_latest_run_id(obj.echo, obj.flow.name)
-        if origin_run_id is None:
-            raise CommandException(
-                "A previous run id was not found. Specify --origin-run-id."
-            )
-
-    if step_to_rerun is None:
-        clone_steps = set()
-    else:
-        # validate step name
-        if step_to_rerun not in obj.graph.nodes:
-            raise CommandException(
-                "invalid step name {0} specified, must be step present in "
-                "current form of execution graph. Valid step names include: {1}".format(
-                    step_to_rerun, ",".join(list(obj.graph.nodes.keys()))
-                )
-            )
-        clone_steps = {step_to_rerun}
-
-    if run_id:
-        # Run-ids that are provided by the metadata service are always integers.
-        # External providers or run-ids (like external schedulers) always need to
-        # be non-integers to avoid any clashes. This condition ensures this.
-        try:
-            int(run_id)
-        except:
-            pass
-        else:
-            raise CommandException("run-id %s cannot be an integer" % run_id)
-
-    runtime = NativeRuntime(
-        obj.flow,
-        obj.graph,
-        obj.flow_datastore,
-        obj.metadata,
-        obj.environment,
-        obj.package,
-        obj.logger,
-        obj.entrypoint,
-        obj.event_logger,
-        obj.monitor,
-        run_id=run_id,
-        clone_run_id=origin_run_id,
-        clone_only=clone_only,
-        reentrant=reentrant,
-        clone_steps=clone_steps,
-        max_workers=max_workers,
-        max_num_splits=max_num_splits,
-        max_log_size=max_log_size * 1024 * 1024,
-        resume_identifier=resume_identifier,
-    )
-    write_run_id(run_id_file, runtime.run_id)
-    runtime.print_workflow_info()
-    runtime.persist_constants()
-    if clone_only:
-        runtime.clone_original_run()
-    else:
-        runtime.execute()
-
-
-@tracing.cli_entrypoint("cli/run")
-@parameters.add_custom_parameters(deploy_mode=True)
-@cli.command(help="Run the workflow locally.")
-@common_run_options
-@click.option(
-    "--namespace",
-    "user_namespace",
-    default=None,
-    help="Change namespace from the default (your username) to "
-    "the specified tag. Note that this option does not alter "
-    "tags assigned to the objects produced by this run, just "
-    "what existing objects are visible in the client API. You "
-    "can enable the global namespace with an empty string."
-    "--namespace=",
-)
-@click.pass_obj
-def run(
-    obj,
-    tags=None,
-    max_workers=None,
-    max_num_splits=None,
-    max_log_size=None,
-    decospecs=None,
-    run_id_file=None,
-    user_namespace=None,
-    **kwargs
-):
-    if user_namespace is not None:
-        namespace(user_namespace or None)
-    before_run(obj, tags, decospecs + obj.environment.decospecs())
-
-    runtime = NativeRuntime(
-        obj.flow,
-        obj.graph,
-        obj.flow_datastore,
-        obj.metadata,
-        obj.environment,
-        obj.package,
-        obj.logger,
-        obj.entrypoint,
-        obj.event_logger,
-        obj.monitor,
-        max_workers=max_workers,
-        max_num_splits=max_num_splits,
-        max_log_size=max_log_size * 1024 * 1024,
-    )
-    write_latest_run_id(obj, runtime.run_id)
-    write_run_id(run_id_file, runtime.run_id)
-
-    obj.flow._set_constants(obj.graph, kwargs)
-    runtime.print_workflow_info()
-    runtime.persist_constants()
-    runtime.execute()
-
-
-def write_run_id(run_id_file, run_id):
-    if run_id_file is not None:
-        with open(run_id_file, "w") as f:
-            f.write(str(run_id))
-
-
-def before_run(obj, tags, decospecs):
-    validate_tags(tags)
-
-    # There's a --with option both at the top-level and for the run
-    # subcommand. Why?
-    #
-    # "run --with shoes" looks so much better than "--with shoes run".
-    # This is a very common use case of --with.
-    #
-    # A downside is that we need to have the following decorators handling
-    # in two places in this module and make sure _init_step_decorators
-    # doesn't get called twice.
-    if decospecs:
-        decorators._attach_decorators(obj.flow, decospecs)
-        obj.graph = FlowGraph(obj.flow.__class__)
-    obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)
-    # obj.environment.init_environment(obj.logger)
-
-    decorators._init_step_decorators(
-        obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
-    )
-
-    obj.metadata.add_sticky_tags(tags=tags)
-
-    # Package working directory only once per run.
-    # We explicitly avoid doing this in `start` since it is invoked for every
-    # step in the run.
-    obj.package = MetaflowPackage(
-        obj.flow, obj.environment, obj.echo, obj.package_suffixes
-    )
-
-
 @cli.command(help="Print the Metaflow version")
 @click.pass_obj
 def version(obj):
     echo_always(obj.version)
 
 
-@tracing.cli_entrypoint("cli/start")
+# NOTE: add_decorator_options should be TL because it checks to make sure
+# that no option conflict with the ones below
 @decorators.add_decorator_options
+@config_options
 @click.command(
-    cls=click.CommandCollection,
-    sources=[cli] + plugins.get_plugin_cli(),
+    cls=LazyPluginCommandCollection,
+    sources=[cli],
+    lazy_sources=plugins.get_plugin_cli_path(),
     invoke_without_command=True,
 )
+# Quiet is eager to make sure it is available when processing --config options since
+# we need it to construct a context to pass to any DeployTimeField for the default
+# value.
 @click.option(
     "--quiet/--not-quiet",
     show_default=True,
     default=False,
     help="Suppress unnecessary messages",
+    is_eager=True,
 )
 @click.option(
     "--metadata",
@@ -925,12 +262,22 @@ def version(obj):
     type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]),
     help="Execution environment type",
 )
+@click.option(
+    "--force-rebuild-environments/--no-force-rebuild-environments",
+    is_flag=True,
+    default=False,
+    hidden=True,
+    type=bool,
+    help="Explicitly rebuild the execution environments",
+)
+# See comment for --quiet
 @click.option(
     "--datastore",
     default=DEFAULT_DATASTORE,
     show_default=True,
     type=click.Choice([d.TYPE for d in DATASTORES]),
     help="Data backend type",
+    is_eager=True,
 )
 @click.option("--datastore-root", help="Root path for datastore")
 @click.option(
@@ -966,12 +313,29 @@ def version(obj):
     type=click.Choice(MONITOR_SIDECARS),
     help="Monitoring backend type",
 )
+@click.option(
+    "--local-config-file",
+    type=LocalFileInput(exists=True, readable=True, dir_okay=False, resolve_path=True),
+    required=False,
+    default=None,
+    help="A filename containing the dumped configuration values. Internal use only.",
+    hidden=True,
+    is_eager=True,
+)
+@click.option(
+    "--mode",
+    type=click.Choice(["spin"]),
+    default=None,
+    help="Execution mode for metaflow CLI commands. Use 'spin' to enable "
+    "spin metadata and spin datastore for executions",
+)
 @click.pass_context
 def start(
     ctx,
     quiet=False,
     metadata=None,
     environment=None,
+    force_rebuild_environments=False,
     datastore=None,
     datastore_root=None,
     decospecs=None,
@@ -979,9 +343,12 @@ def start(
     pylint=None,
     event_logger=None,
     monitor=None,
+    local_config_file=None,
+    config=None,
+    config_value=None,
+    mode=None,
     **deco_options
 ):
-    global echo
     if quiet:
         echo = echo_dev_null
     else:
@@ -992,40 +359,27 @@ def start(
     if use_r():
         version = metaflow_r_version()
 
+    from_start("MetaflowCLI: Starting")
     echo("Metaflow %s" % version, fg="magenta", bold=True, nl=False)
     echo(" executing *%s*" % ctx.obj.flow.name, fg="magenta", nl=False)
     echo(" for *%s*" % resolve_identity(), fg="magenta")
 
+    # Check if we need to setup the distribution finder (if running )
+    dist_info = MetaflowCodeContent.get_distribution_finder()
+    if dist_info:
+        sys.meta_path.append(dist_info)
+
+    # Setup the context
     cli_args._set_top_kwargs(ctx.params)
     ctx.obj.echo = echo
     ctx.obj.echo_always = echo_always
     ctx.obj.is_quiet = quiet
-    ctx.obj.graph = FlowGraph(ctx.obj.flow.__class__)
     ctx.obj.logger = logger
-    ctx.obj.check = _check
     ctx.obj.pylint = pylint
+    ctx.obj.check = functools.partial(_check, echo)
     ctx.obj.top_cli = cli
     ctx.obj.package_suffixes = package_suffixes.split(",")
-    ctx.obj.reconstruct_cli = _reconstruct_cli
-
-    ctx.obj.environment = [
-        e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
-    ][0](ctx.obj.flow)
-    ctx.obj.environment.validate_environment(echo, datastore)
-
-    ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](
-        flow=ctx.obj.flow, env=ctx.obj.environment
-    )
-    ctx.obj.event_logger.start()
-
-    ctx.obj.monitor = MONITOR_SIDECARS[monitor](
-        flow=ctx.obj.flow, env=ctx.obj.environment
-    )
-    ctx.obj.monitor.start()
-
-    ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
-        ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
-    )
+    ctx.obj.spin_mode = mode == "spin"
 
     ctx.obj.datastore_impl = [d for d in DATASTORES if d.TYPE == datastore][0]
 
@@ -1042,6 +396,106 @@ def start(
     ctx.obj.datastore_impl.datastore_root = datastore_root
 
     FlowDataStore.default_storage_impl = ctx.obj.datastore_impl
+
+    # At this point, we are able to resolve the user-configuration options so we can
+    # process all those decorators that the user added that will modify the flow based
+    # on those configurations. It is important to do this as early as possible since it
+    # actually modifies the flow itself
+
+    # When we process the options, the first one processed will return None and the
+    # second one processed will return the actual options. The order of processing
+    # depends on what (and in what order) the user specifies on the command line.
+    config_options = config or config_value
+
+    if (
+        hasattr(ctx, "saved_args")
+        and ctx.saved_args
+        and ctx.saved_args[0] == "resume"
+        and getattr(ctx.obj, "has_config_options", False)
+    ):
+        # In the case of resume, we actually need to load the configurations
+        # from the resumed run to process them. This can be slightly onerous so check
+        # if we need to in the first place
+        if getattr(ctx.obj, "has_cl_config_options", False):
+            raise click.UsageError(
+                "Cannot specify --config or --config-value with 'resume'"
+            )
+        # We now load the config artifacts from the original run id
+        run_id = None
+        try:
+            idx = ctx.saved_args.index("--origin-run-id")
+        except ValueError:
+            idx = -1
+        if idx >= 0:
+            run_id = ctx.saved_args[idx + 1]
+        else:
+            run_id = get_latest_run_id(ctx.obj.echo, ctx.obj.flow.name)
+        if run_id is None:
+            raise CommandException(
+                "A previous run id was not found. Specify --origin-run-id."
+            )
+        # We get the name of the parameters we need to load from the datastore -- these
+        # are accessed using the *variable* name and not necessarily the *parameter* name
+        config_var_names = []
+        config_param_names = []
+        for name, param in ctx.obj.flow._get_parameters():
+            if not param.IS_CONFIG_PARAMETER:
+                continue
+            config_var_names.append(name)
+            config_param_names.append(param.name)
+
+        # We just need a task datastore that will be thrown away -- we do this so
+        # we don't have to create the logger, monitor, etc.
+        debug.userconf_exec("Loading config parameters from run %s" % run_id)
+        for d in TaskDataStoreSet(
+            FlowDataStore(ctx.obj.flow.name),
+            run_id,
+            steps=["_parameters"],
+            prefetch_data_artifacts=config_var_names,
+        ):
+            param_ds = d
+
+        # We can now set the the CONFIGS value in the flow properly. This will overwrite
+        # anything that may have been passed in by default and we will use exactly what
+        # the original flow had. Note that these are accessed through the parameter name
+        ctx.obj.flow._flow_state[FlowStateItems.CONFIGS].clear()
+        d = ctx.obj.flow._flow_state[FlowStateItems.CONFIGS]
+        for param_name, var_name in zip(config_param_names, config_var_names):
+            val = param_ds[var_name]
+            debug.userconf_exec("Loaded config %s as: %s" % (param_name, val))
+            d[param_name] = val
+
+    elif getattr(ctx.obj, "delayed_config_exception", None):
+        # If we are not doing a resume, any exception we had parsing configs needs to
+        # be raised. For resume, since we ignore those options, we ignore the error.
+        raise ctx.obj.delayed_config_exception
+
+    # Init all values in the flow mutators and then process them
+    for decorator in ctx.obj.flow._flow_state[FlowStateItems.FLOW_MUTATORS]:
+        decorator.external_init()
+
+    new_cls = ctx.obj.flow._process_config_decorators(config_options)
+    if new_cls:
+        ctx.obj.flow = new_cls(use_cli=False)
+
+    ctx.obj.graph = ctx.obj.flow._graph
+
+    ctx.obj.environment = [
+        e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
+    ][0](ctx.obj.flow)
+    # set force rebuild flag for environments that support it.
+    ctx.obj.environment._force_rebuild = force_rebuild_environments
+    ctx.obj.environment.validate_environment(ctx.obj.logger, datastore)
+    ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](
+        flow=ctx.obj.flow, env=ctx.obj.environment
+    )
+    ctx.obj.monitor = MONITOR_SIDECARS[monitor](
+        flow=ctx.obj.flow, env=ctx.obj.environment
+    )
+    ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
+        ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
+    )
+
     ctx.obj.flow_datastore = FlowDataStore(
         ctx.obj.flow.name,
         ctx.obj.environment,
@@ -1050,6 +504,61 @@ def start(
         ctx.obj.monitor,
     )
 
+    ctx.obj.config_options = config_options
+    ctx.obj.is_spin = False
+    ctx.obj.skip_decorators = False
+
+    # Override values for spin steps, or if we are in spin mode
+    if (
+        hasattr(ctx, "saved_args")
+        and ctx.saved_args
+        and "spin" in ctx.saved_args[0]
+        or ctx.obj.spin_mode
+    ):
+        # To minimize side effects for spin, we will only use the following:
+        # - local metadata provider,
+        # - local datastore,
+        # - local environment,
+        # - null event logger,
+        # - null monitor
+        ctx.obj.is_spin = True
+        if "--skip-decorators" in ctx.saved_args:
+            ctx.obj.skip_decorators = True
+
+        ctx.obj.event_logger = LOGGING_SIDECARS["nullSidecarLogger"](
+            flow=ctx.obj.flow, env=ctx.obj.environment
+        )
+        ctx.obj.monitor = MONITOR_SIDECARS["nullSidecarMonitor"](
+            flow=ctx.obj.flow, env=ctx.obj.environment
+        )
+        # Use spin metadata, spin datastore, and spin datastore root
+        ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == "spin"][0](
+            ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
+        )
+        ctx.obj.datastore_impl = [d for d in DATASTORES if d.TYPE == "spin"][0]
+        datastore_root = ctx.obj.datastore_impl.get_datastore_root_from_config(
+            ctx.obj.echo, create_on_absent=True
+        )
+        ctx.obj.datastore_impl.datastore_root = datastore_root
+
+        ctx.obj.flow_datastore = FlowDataStore(
+            ctx.obj.flow.name,
+            ctx.obj.environment,  # Same environment as run/resume
+            ctx.obj.metadata,  # local metadata
+            ctx.obj.event_logger,  # null event logger
+            ctx.obj.monitor,  # null monitor
+            storage_impl=ctx.obj.datastore_impl,
+        )
+
+    # Start event logger and monitor
+    ctx.obj.event_logger.start()
+    _system_logger.init_system_logger(ctx.obj.flow.name, ctx.obj.event_logger)
+
+    ctx.obj.monitor.start()
+    _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
+
+    decorators._init(ctx.obj.flow)
+
     # It is important to initialize flow decorators early as some of the
     # things they provide may be used by some of the objects initialized after.
     decorators._init_flow_decorators(
@@ -1061,49 +570,83 @@ def start(
         ctx.obj.logger,
         echo,
         deco_options,
+        ctx.obj.is_spin,
+        ctx.obj.skip_decorators,
     )
 
-    if decospecs:
-        decorators._attach_decorators(ctx.obj.flow, decospecs)
+    # In the case of run/resume/spin, we will want to apply the TL decospecs
+    # *after* the run decospecs so that they don't take precedence. In other
+    # words, for the same decorator, we want `myflow.py run --with foo` to
+    # take precedence over any other `foo` decospec
+
+    # Note that top-level decospecs are used primarily with non run/resume
+    # options as well as with the airflow/argo/sfn integrations which pass
+    # all the decospecs (the ones from top-level but also the ones from the
+    # run/resume level) through the tl decospecs.
+    ctx.obj.tl_decospecs = list(decospecs or [])
 
     # initialize current and parameter context for deploy-time parameters
     current._set_env(flow=ctx.obj.flow, is_running=False)
     parameters.set_parameter_context(
-        ctx.obj.flow.name, ctx.obj.echo, ctx.obj.flow_datastore
+        ctx.obj.flow.name,
+        ctx.obj.echo,
+        ctx.obj.flow_datastore,
+        {
+            k: ConfigValue(v) if v is not None else None
+            for k, v in ctx.obj.flow.__class__._flow_state[
+                FlowStateItems.CONFIGS
+            ].items()
+        },
     )
 
-    if ctx.invoked_subcommand not in ("run", "resume"):
-        # run/resume are special cases because they can add more decorators with --with,
+    if (
+        hasattr(ctx, "saved_args")
+        and ctx.saved_args
+        and ctx.saved_args[0] not in ("run", "resume", "spin")
+    ):
+        # run/resume/spin are special cases because they can add more decorators with --with,
         # so they have to take care of themselves.
-        decorators._attach_decorators(ctx.obj.flow, ctx.obj.environment.decospecs())
+        all_decospecs = ctx.obj.tl_decospecs + list(
+            ctx.obj.environment.decospecs() or []
+        )
+
+        # We add the default decospecs for everything except init and step since in those
+        # cases, the decospecs will already have been handled by either a run/resume
+        # or a scheduler setting them up in their own way.
+        if ctx.saved_args[0] not in ("step", "init"):
+            all_decospecs += DEFAULT_DECOSPECS.split()
+        elif ctx.saved_args[0] == "spin-step":
+            # If we are in spin-args, we will not attach any decorators
+            all_decospecs = []
+        if all_decospecs:
+            decorators._attach_decorators(ctx.obj.flow, all_decospecs)
+            decorators._init(ctx.obj.flow)
+            # Regenerate graph if we attached more decorators
+            ctx.obj.flow.__class__._init_graph()
+            ctx.obj.graph = ctx.obj.flow._graph
+
         decorators._init_step_decorators(
             ctx.obj.flow,
             ctx.obj.graph,
             ctx.obj.environment,
             ctx.obj.flow_datastore,
             ctx.obj.logger,
+            # The last two arguments are only used for spin steps
+            ctx.obj.is_spin,
+            ctx.obj.skip_decorators,
         )
+
+        # Check the graph again (mutators may have changed it)
+        ctx.obj.graph = ctx.obj.flow._graph
+
         # TODO (savin): Enable lazy instantiation of package
         ctx.obj.package = None
+
     if ctx.invoked_subcommand is None:
         ctx.invoke(check)
 
 
-def _reconstruct_cli(params):
-    for k, v in params.items():
-        if v:
-            if k == "decospecs":
-                k = "with"
-            k = k.replace("_", "-")
-            if not isinstance(v, tuple):
-                v = [v]
-            for value in v:
-                yield "--%s" % k
-                if not isinstance(value, bool):
-                    yield str(value)
-
-
-def _check(graph, flow, environment, pylint=True, warnings=False, **kwargs):
+def _check(echo, graph, flow, environment, pylint=True, warnings=False, **kwargs):
     echo("Validating your flow...", fg="magenta", bold=False)
     linter = lint.linter
     # TODO set linter settings
@@ -1142,10 +685,13 @@ def _check(graph, flow, environment, pylint=True, warnings=False, **kwargs):
 
 def print_metaflow_exception(ex):
     echo_always(ex.headline, indent=True, nl=False, bold=True)
-    if ex.line_no is None:
-        echo_always(":")
-    else:
-        echo_always(" on line %d:" % ex.line_no, bold=True)
+    location = ""
+    if ex.source_file is not None:
+        location += " in file %s" % ex.source_file
+    if ex.line_no is not None:
+        location += " on line %d" % ex.line_no
+    location += ":"
+    echo_always(location, bold=True)
     echo_always(ex.message, indent=True, bold=False, padding_bottom=True)