metaflow 2.12.36__py2.py3-none-any.whl → 2.12.38__py2.py3-none-any.whl

metaflow/cli_components/step_cmd.py

@@ -0,0 +1,189 @@
+from metaflow._vendor import click
+
+from .. import decorators, namespace
+from ..cli import echo_always, echo_dev_null
+from ..cli_args import cli_args
+from ..exception import CommandException
+from ..task import MetaflowTask
+from ..unbounded_foreach import UBF_CONTROL, UBF_TASK
+from ..util import decompress_list
+
+
+@click.command(help="Internal command to execute a single task.", hidden=True)
+@click.argument("step-name")
+@click.option(
+    "--run-id",
+    default=None,
+    required=True,
+    help="ID for one execution of all steps in the flow.",
+)
+@click.option(
+    "--task-id",
+    default=None,
+    required=True,
+    show_default=True,
+    help="ID for this instance of the step.",
+)
+@click.option(
+    "--input-paths",
+    help="A comma-separated list of pathspecs specifying inputs for this step.",
+)
+@click.option(
+    "--input-paths-filename",
+    type=click.Path(exists=True, readable=True, dir_okay=False, resolve_path=True),
+    help="A filename containing the argument typically passed to `input-paths`",
+    hidden=True,
+)
+@click.option(
+    "--split-index",
+    type=int,
+    default=None,
+    show_default=True,
+    help="Index of this foreach split.",
+)
+@click.option(
+    "--tag",
+    "opt_tag",
+    multiple=True,
+    default=None,
+    help="Annotate this run with the given tag. You can specify "
+    "this option multiple times to attach multiple tags in "
+    "the task.",
+)
+@click.option(
+    "--namespace",
+    "opt_namespace",
+    default=None,
+    help="Change namespace from the default (your username) to the specified tag.",
+)
+@click.option(
+    "--retry-count",
+    default=0,
+    help="How many times we have attempted to run this task.",
+)
+@click.option(
+    "--max-user-code-retries",
+    default=0,
+    help="How many times we should attempt running the user code.",
+)
+@click.option(
+    "--clone-only",
+    default=None,
+    help="Pathspec of the origin task for this task to clone. Do "
+    "not execute anything.",
+)
+@click.option(
+    "--clone-run-id",
+    default=None,
+    help="Run id of the origin flow, if this task is part of a flow being resumed.",
+)
+@click.option(
+    "--with",
+    "decospecs",
+    multiple=True,
+    help="Add a decorator to this task. You can specify this "
+    "option multiple times to attach multiple decorators "
+    "to this task.",
+)
+@click.option(
+    "--ubf-context",
+    default="none",
+    type=click.Choice(["none", UBF_CONTROL, UBF_TASK]),
+    help="Provides additional context if this task is of type unbounded foreach.",
+)
+@click.option(
+    "--num-parallel",
+    default=0,
+    type=int,
+    help="Number of parallel instances of a step. Ignored in local mode (see parallel decorator code).",
+)
+@click.pass_context
+def step(
+    ctx,
+    step_name,
+    opt_tag=None,
+    run_id=None,
+    task_id=None,
+    input_paths=None,
+    input_paths_filename=None,
+    split_index=None,
+    opt_namespace=None,
+    retry_count=None,
+    max_user_code_retries=None,
+    clone_only=None,
+    clone_run_id=None,
+    decospecs=None,
+    ubf_context="none",
+    num_parallel=None,
+):
+
+    if ctx.obj.is_quiet:
+        echo = echo_dev_null
+    else:
+        echo = echo_always
+
+    if ubf_context == "none":
+        ubf_context = None
+    if opt_namespace is not None:
+        namespace(opt_namespace or None)
+
+    func = None
+    try:
+        func = getattr(ctx.obj.flow, step_name)
+    except:
+        raise CommandException("Step *%s* doesn't exist." % step_name)
+    if not func.is_step:
+        raise CommandException("Function *%s* is not a step." % step_name)
+    echo("Executing a step, *%s*" % step_name, fg="magenta", bold=False)
+
+    if decospecs:
+        decorators._attach_decorators_to_step(func, decospecs)
+        decorators._init(ctx.obj.flow)
+
+    step_kwargs = ctx.params
+    # Remove argument `step_name` from `step_kwargs`.
+    step_kwargs.pop("step_name", None)
+    # Remove `opt_*` prefix from (some) option keys.
+    step_kwargs = dict(
+        [(k[4:], v) if k.startswith("opt_") else (k, v) for k, v in step_kwargs.items()]
+    )
+    cli_args._set_step_kwargs(step_kwargs)
+
+    ctx.obj.metadata.add_sticky_tags(tags=opt_tag)
+    if not input_paths and input_paths_filename:
+        with open(input_paths_filename, mode="r", encoding="utf-8") as f:
+            input_paths = f.read().strip(" \n\"'")
+
+    paths = decompress_list(input_paths) if input_paths else []
+
+    task = MetaflowTask(
+        ctx.obj.flow,
+        ctx.obj.flow_datastore,
+        ctx.obj.metadata,
+        ctx.obj.environment,
+        ctx.obj.echo,
+        ctx.obj.event_logger,
+        ctx.obj.monitor,
+        ubf_context,
+    )
+    if clone_only:
+        task.clone_only(
+            step_name,
+            run_id,
+            task_id,
+            clone_only,
+            retry_count,
+        )
+    else:
+        task.run_step(
+            step_name,
+            run_id,
+            task_id,
+            clone_run_id,
+            paths,
+            split_index,
+            retry_count,
+            max_user_code_retries,
+        )
+
+    echo("Success", fg="green", bold=True, indent=True)

metaflow/cli_components/utils.py

@@ -0,0 +1,140 @@
+import importlib
+from metaflow._vendor import click
+from metaflow.extension_support.plugins import get_plugin
+
+
+class LazyPluginCommandCollection(click.CommandCollection):
+    # lazy_source should only point to things that are resolved as CLI plugins.
+    def __init__(self, *args, lazy_sources=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        # lazy_sources is a list of strings in the form
+        # "{plugin_name}" -> "{module-name}.{command-object-name}"
+        self.lazy_sources = lazy_sources or {}
+        self._lazy_loaded = {}
+
+    def invoke(self, ctx):
+        # NOTE: This is copied from MultiCommand.invoke. The change is that we
+        # behave like chain in the sense that we evaluate the subcommand *after*
+        # invoking the base command but we don't chain the commands like self.chain
+        # would otherwise indicate.
+        # The goal of this is to make sure that the first command is properly executed
+        # *first* prior to loading the other subcommands. It's more a lazy_subcommand_load
+        # than a chain.
+        # Look for CHANGE HERE in this code to see where the changes are made.
+        # If click is updated, this may also need to be updated. This version is for
+        # click 7.1.2.
+        def _process_result(value):
+            if self.result_callback is not None:
+                value = ctx.invoke(self.result_callback, value, **ctx.params)
+            return value
+
+        if not ctx.protected_args:
+            # If we are invoked without command the chain flag controls
+            # how this happens.  If we are not in chain mode, the return
+            # value here is the return value of the command.
+            # If however we are in chain mode, the return value is the
+            # return value of the result processor invoked with an empty
+            # list (which means that no subcommand actually was executed).
+            if self.invoke_without_command:
+                # CHANGE HERE: We behave like self.chain = False here
+
+                # if not self.chain:
+                return click.Command.invoke(self, ctx)
+                # with ctx:
+                #    click.Command.invoke(self, ctx)
+                #    return _process_result([])
+
+            ctx.fail("Missing command.")
+
+        # Fetch args back out
+        args = ctx.protected_args + ctx.args
+        ctx.args = []
+        ctx.protected_args = []
+        # CHANGE HERE: Add saved_args so we have access to it in the command to be
+        # able to infer what we are calling next
+        ctx.saved_args = args
+
+        # If we're not in chain mode, we only allow the invocation of a
+        # single command but we also inform the current context about the
+        # name of the command to invoke.
+        # CHANGE HERE: We change this block to do the invoke *before* the resolve_command
+        # Make sure the context is entered so we do not clean up
+        # resources until the result processor has worked.
+        with ctx:
+            ctx.invoked_subcommand = "*" if args else None
+            click.Command.invoke(self, ctx)
+            cmd_name, cmd, args = self.resolve_command(ctx, args)
+            sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)
+            with sub_ctx:
+                return _process_result(sub_ctx.command.invoke(sub_ctx))
+
+        # CHANGE HERE: Removed all the part of chain mode.
+
+    def list_commands(self, ctx):
+        base = super().list_commands(ctx)
+        for source_name, source in self.lazy_sources.items():
+            subgroup = self._lazy_load(source_name, source)
+            base.extend(subgroup.list_commands(ctx))
+        return base
+
+    def get_command(self, ctx, cmd_name):
+        base_cmd = super().get_command(ctx, cmd_name)
+        if base_cmd is not None:
+            return base_cmd
+        for source_name, source in self.lazy_sources.items():
+            subgroup = self._lazy_load(source_name, source)
+            cmd = subgroup.get_command(ctx, cmd_name)
+            if cmd is not None:
+                return cmd
+        return None
+
+    def _lazy_load(self, source_name, source_path):
+        if source_name in self._lazy_loaded:
+            return self._lazy_loaded[source_name]
+        cmd_object = get_plugin("cli", source_path, source_name)
+        if not isinstance(cmd_object, click.Group):
+            raise ValueError(
+                f"Lazy loading of {source_name} failed by returning "
+                "a non-group object"
+            )
+        self._lazy_loaded[source_name] = cmd_object
+        return cmd_object
+
+
+class LazyGroup(click.Group):
+    def __init__(self, *args, lazy_subcommands=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        # lazy_subcommands is a list of strings in the form
+        # "{command} -> "{module-name}.{command-object-name}"
+        self.lazy_subcommands = lazy_subcommands or {}
+        self._lazy_loaded = {}
+
+    def list_commands(self, ctx):
+        base = super().list_commands(ctx)
+        lazy = sorted(self.lazy_subcommands.keys())
+        return base + lazy
+
+    def get_command(self, ctx, cmd_name):
+        if cmd_name in self.lazy_subcommands:
+            return self._lazy_load(cmd_name)
+        return super().get_command(ctx, cmd_name)
+
+    def _lazy_load(self, cmd_name):
+        if cmd_name in self._lazy_loaded:
+            return self._lazy_loaded[cmd_name]
+
+        import_path = self.lazy_subcommands[cmd_name]
+        modname, cmd = import_path.rsplit(".", 1)
+        # do the import
+        mod = importlib.import_module(modname)
+        # get the Command object from that module
+        cmd_object = getattr(mod, cmd)
+        # check the result to make debugging easier. note that wrapped BaseCommand
+        # can be functions
+        if not isinstance(cmd_object, click.BaseCommand):
+            raise ValueError(
+                f"Lazy loading of {import_path} failed by returning "
+                f"a non-command object {type(cmd_object)}"
+            )
+        self._lazy_loaded[cmd_name] = cmd_object
+        return cmd_object

metaflow/cmd/develop/stub_generator.py

@@ -1238,25 +1238,32 @@ class StubGenerator:
                 buff.write(indentation + deco + "\n")
             buff.write(indentation + "def " + name + "(")
             kw_only_param = False
+            has_var_args = False
             for i, (par_name, parameter) in enumerate(my_sign.parameters.items()):
                 annotation = self._exploit_annotation(parameter.annotation)
-
                 default = exploit_default(parameter.default)
 
-                if kw_only_param and parameter.kind != inspect.Parameter.KEYWORD_ONLY:
+                if (
+                    kw_only_param
+                    and not has_var_args
+                    and parameter.kind != inspect.Parameter.KEYWORD_ONLY
+                ):
                     raise RuntimeError(
                         "In function '%s': cannot have a positional parameter after a "
                         "keyword only parameter" % name
                     )
+
                 if (
                     parameter.kind == inspect.Parameter.KEYWORD_ONLY
                     and not kw_only_param
+                    and not has_var_args
                 ):
                     kw_only_param = True
                     buff.write("*, ")
                 if parameter.kind == inspect.Parameter.VAR_KEYWORD:
                     par_name = "**%s" % par_name
                 elif parameter.kind == inspect.Parameter.VAR_POSITIONAL:
+                    has_var_args = True
                     par_name = "*%s" % par_name
 
                 if default:

metaflow/decorators.py

@@ -12,6 +12,12 @@ from .exception import (
 )
 
 from .parameters import current_flow
+from .user_configs.config_decorators import CustomStepDecorator
+from .user_configs.config_parameters import (
+    UNPACK_KEY,
+    resolve_delayed_evaluator,
+    unpack_delayed_evaluator,
+)
 
 from metaflow._vendor import click
 
@@ -21,6 +27,11 @@ except NameError:
     unicode = str
     basestring = str
 
+# Contains the decorators on which _init was called. We want to ensure it is called
+# only once on each decorator and, as the _init() function below can be called in
+# several places, we need to track which decorator had their init function called
+_inited_decorators = set()
+
 
 class BadStepDecoratorException(MetaflowException):
     headline = "Syntax error"
@@ -115,14 +126,36 @@ class Decorator(object):
     def __init__(self, attributes=None, statically_defined=False):
         self.attributes = self.defaults.copy()
         self.statically_defined = statically_defined
+        self._user_defined_attributes = set()
+        self._ran_init = False
 
         if attributes:
             for k, v in attributes.items():
-                if k in self.defaults:
+                if k in self.defaults or k.startswith(UNPACK_KEY):
                     self.attributes[k] = v
+                    if not k.startswith(UNPACK_KEY):
+                        self._user_defined_attributes.add(k)
                 else:
                     raise InvalidDecoratorAttribute(self.name, k, self.defaults)
 
+    def init(self):
+        """
+        Initializes the decorator. In general, any operation you would do in __init__
+        should be done here.
+        """
+
+        # In some cases (specifically when using remove_decorator), we may need to call
+        # init multiple times. Short-circuit re-evaluating.
+        if self._ran_init:
+            return
+
+        # Note that by design, later values override previous ones.
+        self.attributes = unpack_delayed_evaluator(self.attributes)
+        self._user_defined_attributes.update(self.attributes.keys())
+        self.attributes = resolve_delayed_evaluator(self.attributes)
+
+        self._ran_init = True
+
     @classmethod
     def _parse_decorator_spec(cls, deco_spec):
         if len(deco_spec) == 0:
@@ -203,10 +236,13 @@ class FlowDecorator(Decorator):
 
 # compare this to parameters.add_custom_parameters
 def add_decorator_options(cmd):
-    seen = {}
     flow_cls = getattr(current_flow, "flow_cls", None)
     if flow_cls is None:
         return cmd
+
+    seen = {}
+    existing_params = set(p.name.lower() for p in cmd.params)
+    # Add decorator options
     for deco in flow_decorators(flow_cls):
         for option, kwargs in deco.options.items():
             if option in seen:
@@ -217,7 +253,13 @@ def add_decorator_options(cmd):
                     % (deco.name, option, seen[option])
                 )
                 raise MetaflowInternalError(msg)
+            elif deco.name.lower() in existing_params:
+                raise MetaflowInternalError(
+                    "Flow decorator '%s' uses an option '%s' which is a reserved "
+                    "keyword. Please use a different option name." % (deco.name, option)
+                )
             else:
+                kwargs["envvar"] = "METAFLOW_FLOW_%s" % option.upper()
                 seen[option] = deco.name
                 cmd.params.insert(0, click.Option(("--" + option,), **kwargs))
     return cmd
@@ -425,10 +467,13 @@ def _base_step_decorator(decotype, *args, **kwargs):
     Decorator prototype for all step decorators. This function gets specialized
     and imported for all decorators types by _import_plugin_decorators().
     """
+
     if args:
         # No keyword arguments specified for the decorator, e.g. @foobar.
         # The first argument is the function to be decorated.
         func = args[0]
+        if isinstance(func, CustomStepDecorator):
+            func = func._my_step
         if not hasattr(func, "is_step"):
             raise BadStepDecoratorException(decotype.name, func)
 
@@ -510,6 +555,21 @@ def _attach_decorators_to_step(step, decospecs):
             step.decorators.append(deco)
 
 
+def _init(flow, only_non_static=False):
+    for decorators in flow._flow_decorators.values():
+        for deco in decorators:
+            if deco in _inited_decorators:
+                continue
+            deco.init()
+            _inited_decorators.add(deco)
+    for flowstep in flow:
+        for deco in flowstep.decorators:
+            if deco in _inited_decorators:
+                continue
+            deco.init()
+            _inited_decorators.add(deco)
+
+
 def _init_flow_decorators(
     flow, graph, environment, flow_datastore, metadata, logger, echo, deco_options
 ):
@@ -620,6 +680,7 @@ def step(
     """
     f.is_step = True
     f.decorators = []
+    f.config_decorators = []
     try:
         # python 3
         f.name = f.__name__

metaflow/extension_support/plugins.py

@@ -93,7 +93,32 @@ def merge_lists(base, overrides, attr):
     base[:] = l[:]
 
 
-def resolve_plugins(category):
+def get_plugin(category, class_path, name):
+    path, cls_name = class_path.rsplit(".", 1)
+    try:
+        plugin_module = importlib.import_module(path)
+    except ImportError as e:
+        raise ValueError(
+            "Cannot locate %s plugin '%s' at '%s'" % (category, name, path)
+        ) from e
+    cls = getattr(plugin_module, cls_name, None)
+    if cls is None:
+        raise ValueError(
+            "Cannot locate '%s' class for %s plugin at '%s'"
+            % (cls_name, category, path)
+        )
+    extracted_name = get_plugin_name(category, cls)
+    if extracted_name and extracted_name != name:
+        raise ValueError(
+            "Class '%s' at '%s' for %s plugin expected to be named '%s' but got '%s'"
+            % (cls_name, path, category, name, extracted_name)
+        )
+    globals()[cls_name] = cls
+    _ext_debug("        Added %s plugin '%s' from '%s'" % (category, name, class_path))
+    return cls
+
+
+def resolve_plugins(category, path_only=False):
     # Called to return a list of classes that are the available plugins for 'category'
 
     # The ENABLED_<category> variable is set in process_plugins
@@ -114,7 +139,7 @@ def resolve_plugins(category):
 
     available_plugins = globals()[_dict_for_category(category)]
     name_extractor = _plugin_categories[category]
-    if not name_extractor:
+    if path_only or not name_extractor:
         # If we have no name function, it means we just use the name in the dictionary
         # and we return a dictionary. This is for sidecars mostly as they do not have
         # a field that indicates their name
@@ -132,32 +157,14 @@ def resolve_plugins(category):
                 "Configuration requested %s plugin '%s' but no such plugin is available"
                 % (category, name)
             )
-        path, cls_name = class_path.rsplit(".", 1)
-        try:
-            plugin_module = importlib.import_module(path)
-        except ImportError:
-            raise ValueError(
-                "Cannot locate %s plugin '%s' at '%s'" % (category, name, path)
-            )
-        cls = getattr(plugin_module, cls_name, None)
-        if cls is None:
-            raise ValueError(
-                "Cannot locate '%s' class for %s plugin at '%s'"
-                % (cls_name, category, path)
-            )
-        if name_extractor and name_extractor(cls) != name:
-            raise ValueError(
-                "Class '%s' at '%s' for %s plugin expected to be named '%s' but got '%s'"
-                % (cls_name, path, category, name, name_extractor(cls))
-            )
-        globals()[cls_name] = cls
-        if name_extractor is not None:
-            to_return.append(cls)
+        if path_only:
+            to_return[name] = class_path
         else:
-            to_return[name] = cls
-        _ext_debug(
-            "        Added %s plugin '%s' from '%s'" % (category, name, class_path)
-        )
+            if name_extractor is not None:
+                to_return.append(get_plugin(category, class_path, name))
+            else:
+                to_return[name] = get_plugin(category, class_path, name)
+
     return to_return
 
 
@@ -193,6 +200,13 @@ _plugin_categories = {
 }
 
 
+def get_plugin_name(category, plugin):
+    extractor = _plugin_categories[category]
+    if extractor:
+        return extractor(plugin)
+    return None
+
+
 def _list_for_category(category):
     # Convenience function to name the variable containing List[Tuple[str, str]] where
     # each tuple contains: