ob-metaflow 2.12.39.1__py2.py3-none-any.whl → 2.13.0.1__py2.py3-none-any.whl

metaflow/lint.py

@@ -52,7 +52,7 @@ def check_reserved_words(graph):
     msg = "Step name *%s* is a reserved word. Choose another name for the " "step."
     for node in graph:
         if node.name in RESERVED:
-            raise LintWarn(msg % node.name)
+            raise LintWarn(msg % node.name, node.func_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -76,9 +76,9 @@ def check_that_end_is_end(graph):
     node = graph["end"]
 
     if node.has_tail_next or node.invalid_tail_next:
-        raise LintWarn(msg0, node.tail_next_lineno)
+        raise LintWarn(msg0, node.tail_next_lineno, node.source_file)
     if node.num_args > 1:
-        raise LintWarn(msg1, node.tail_next_lineno)
+        raise LintWarn(msg1, node.tail_next_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -90,7 +90,7 @@ def check_step_names(graph):
     )
     for node in graph:
         if re.search("[^a-z0-9_]", node.name) or node.name[0] == "_":
-            raise LintWarn(msg.format(node), node.func_lineno)
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -108,11 +108,11 @@ def check_num_args(graph):
     msg2 = "Step *{0.name}* is missing the 'self' argument."
     for node in graph:
         if node.num_args > 2:
-            raise LintWarn(msg0.format(node), node.func_lineno)
+            raise LintWarn(msg0.format(node), node.func_lineno, node.source_file)
         elif node.num_args == 2 and node.type != "join":
-            raise LintWarn(msg1.format(node), node.func_lineno)
+            raise LintWarn(msg1.format(node), node.func_lineno, node.source_file)
         elif node.num_args == 0:
-            raise LintWarn(msg2.format(node), node.func_lineno)
+            raise LintWarn(msg2.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -125,7 +125,7 @@ def check_static_transitions(graph):
     )
     for node in graph:
         if node.type != "end" and not node.has_tail_next:
-            raise LintWarn(msg.format(node), node.func_lineno)
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -138,7 +138,7 @@ def check_valid_transitions(graph):
     )
     for node in graph:
         if node.type != "end" and node.has_tail_next and node.invalid_tail_next:
-            raise LintWarn(msg.format(node), node.tail_next_lineno)
+            raise LintWarn(msg.format(node), node.tail_next_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -151,7 +151,11 @@ def check_unknown_transitions(graph):
     for node in graph:
         unknown = [n for n in node.out_funcs if n not in graph]
         if unknown:
-            raise LintWarn(msg.format(node, step=unknown[0]), node.tail_next_lineno)
+            raise LintWarn(
+                msg.format(node, step=unknown[0]),
+                node.tail_next_lineno,
+                node.source_file,
+            )
 
 
 @linter.ensure_acyclicity
@@ -167,7 +171,9 @@ def check_for_acyclicity(graph):
         for n in node.out_funcs:
             if n in seen:
                 path = "->".join(seen + [n])
-                raise LintWarn(msg.format(path), node.tail_next_lineno)
+                raise LintWarn(
+                    msg.format(path), node.tail_next_lineno, node.source_file
+                )
             else:
                 check_path(graph[n], seen + [n])
 
@@ -195,7 +201,7 @@ def check_for_orphans(graph):
     orphans = nodeset - seen
     if orphans:
         orphan = graph[list(orphans)[0]]
-        raise LintWarn(msg.format(orphan), orphan.func_lineno)
+        raise LintWarn(msg.format(orphan), orphan.func_lineno, orphan.source_file)
 
 
 @linter.ensure_static_graph
@@ -230,7 +236,9 @@ def check_split_join_balance(graph):
             if split_stack:
                 _, split_roots = split_stack.pop()
                 roots = ", ".join(split_roots)
-                raise LintWarn(msg0.format(roots=roots))
+                raise LintWarn(
+                    msg0.format(roots=roots), node.func_lineno, node.source_file
+                )
         elif node.type == "join":
             if split_stack:
                 _, split_roots = split_stack[-1]
@@ -243,9 +251,10 @@ def check_split_join_balance(graph):
                             node, paths=paths, num_roots=len(split_roots), roots=roots
                         ),
                         node.func_lineno,
+                        node.source_file,
                     )
             else:
-                raise LintWarn(msg2.format(node), node.func_lineno)
+                raise LintWarn(msg2.format(node), node.func_lineno, node.source_file)
 
             # check that incoming steps come from the same lineage
             # (no cross joins)
@@ -256,7 +265,7 @@ def check_split_join_balance(graph):
                     return tuple(graph[n].split_parents)
 
             if not all_equal(map(parents, node.in_funcs)):
-                raise LintWarn(msg3.format(node), node.func_lineno)
+                raise LintWarn(msg3.format(node), node.func_lineno, node.source_file)
 
         for n in node.out_funcs:
             traverse(graph[n], new_stack)
@@ -276,7 +285,9 @@ def check_empty_foreaches(graph):
         if node.type == "foreach":
             joins = [n for n in node.out_funcs if graph[n].type == "join"]
             if joins:
-                raise LintWarn(msg.format(node, join=joins[0]))
+                raise LintWarn(
+                    msg.format(node, join=joins[0]), node.func_lineno, node.source_file
+                )
 
 
 @linter.ensure_static_graph
@@ -290,7 +301,7 @@ def check_parallel_step_after_next(graph):
         if node.parallel_foreach and not all(
             graph[out_node].parallel_step for out_node in node.out_funcs
         ):
-            raise LintWarn(msg.format(node))
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -303,7 +314,9 @@ def check_join_followed_by_parallel_step(graph):
     )
     for node in graph:
         if node.parallel_step and not graph[node.out_funcs[0]].type == "join":
-            raise LintWarn(msg.format(node.out_funcs[0]))
+            raise LintWarn(
+                msg.format(node.out_funcs[0]), node.func_lineno, node.source_file
+            )
 
 
 @linter.ensure_static_graph
@@ -318,7 +331,9 @@ def check_parallel_foreach_calls_parallel_step(graph):
             for node2 in graph:
                 if node2.out_funcs and node.name in node2.out_funcs:
                     if not node2.parallel_foreach:
-                        raise LintWarn(msg.format(node, node2))
+                        raise LintWarn(
+                            msg.format(node, node2), node.func_lineno, node.source_file
+                        )
 
 
 @linter.ensure_non_nested_foreach
@@ -331,4 +346,4 @@ def check_nested_foreach(graph):
     for node in graph:
         if node.type == "foreach":
             if any(graph[p].type == "foreach" for p in node.split_parents):
-                raise LintWarn(msg.format(node))
+                raise LintWarn(msg.format(node), node.func_lineno, node.source_file)

metaflow/metaflow_config.py

@@ -511,6 +511,11 @@ MAX_CPU_PER_TASK = from_conf("MAX_CPU_PER_TASK")
 # lexicographic ordering of attempts. This won't work if MAX_ATTEMPTS > 99.
 MAX_ATTEMPTS = 6
 
+# Feature flag (experimental features that are *explicitly* unsupported)
+
+# Process configs even when using the click_api for Runner/Deployer
+CLICK_API_PROCESS_CONFIG = from_conf("CLICK_API_PROCESS_CONFIG", False)
+
 
 # PINNED_CONDA_LIBS are the libraries that metaflow depends on for execution
 # and are needed within a conda environment

metaflow/parameters.py

@@ -359,7 +359,7 @@ class Parameter(object):
             "show_default": show_default,
         }
 
-    def init(self):
+    def init(self, ignore_errors=False):
         # Prevent circular import
         from .user_configs.config_parameters import (
             resolve_delayed_evaluator,
@@ -367,14 +367,21 @@ class Parameter(object):
         )
 
         # Resolve any value from configurations
-        self.kwargs = unpack_delayed_evaluator(self.kwargs)
-        self.kwargs = resolve_delayed_evaluator(self.kwargs)
+        self.kwargs = unpack_delayed_evaluator(self.kwargs, ignore_errors=ignore_errors)
+        # Do it one item at a time so errors are ignored at that level (as opposed to
+        # at the entire kwargs leve)
+        self.kwargs = {
+            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
+            for k, v in self.kwargs.items()
+        }
 
         # This was the behavior before configs: values specified in args would override
         # stuff in kwargs which is what we implement here as well
         for key, value in self._override_kwargs.items():
             if value is not None:
-                self.kwargs[key] = value
+                self.kwargs[key] = resolve_delayed_evaluator(
+                    value, ignore_errors=ignore_errors
+                )
         # Set two default values if no-one specified them
         self.kwargs.setdefault("required", False)
         self.kwargs.setdefault("show_default", True)

metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py

@@ -50,24 +50,27 @@ class AwsSecretsManagerSecretsProvider(SecretsProvider):
         The secret payload from AWS is EITHER a string OR a binary blob.
 
         If the secret contains a string payload ("SecretString"):
-        - if the `parse_secret_string_as_json` option is True (default):
+        - if the `json` option is True (default):
             {SecretString} will be parsed as a JSON. If successfully parsed, AND the JSON contains a
             top-level object, each entry K/V in the object will also be converted to an entry in the result. V will
             always be casted to a string (if not already a string).
-        - If `parse_secret_string_as_json` option is False:
-            {SecretString} will be returned as a single entry in the result, with the key being the secret_id.
+        - If `json` option is False:
+            {SecretString} will be returned as a single entry in the result, where the key is either:
+                - the `secret_id`, OR
+                - the value set by `options={"env_var_name": custom_env_var_name}`.
 
-        Otherwise, the secret contains a binary blob payload ("SecretBinary"). In this case
-        - The result dic contains '{SecretName}': '{SecretBinary}', where {SecretBinary} is a base64-encoded string
+        Otherwise, if the secret contains a binary blob payload ("SecretBinary"):
+        - The result dict contains '{SecretName}': '{SecretBinary}', where {SecretBinary} is a base64-encoded string.
 
-        All keys in the result are sanitized to be more valid environment variable names. This is done on a best effort
+        All keys in the result are sanitized to be more valid environment variable names. This is done on a best-effort
         basis. Further validation is expected to be done by the invoking @secrets decorator itself.
 
-        :param secret_id: ARN or friendly name of the secret
-        :param options: unused
-        :param role: AWS IAM Role ARN to assume before reading the secret
-        :return: dict of environment variables. All keys and values are strings.
+        :param secret_id: ARN or friendly name of the secret.
+        :param options: Dictionary of additional options. E.g., `options={"env_var_name": custom_env_var_name}`.
+        :param role: AWS IAM Role ARN to assume before reading the secret.
+        :return: Dictionary of environment variables. All keys and values are strings.
         """
+
         import botocore
         from metaflow.plugins.aws.aws_client import get_aws_client
 

metaflow/plugins/cards/card_creator.py

@@ -122,6 +122,7 @@ class CardCreator:
             executable,
             sys.argv[0],
         ]
+
         cmd += self._top_level_options + [
             "card",
             "create",

metaflow/plugins/cards/card_decorator.py

@@ -1,13 +1,16 @@
+import json
+import os
+import re
+import tempfile
+
 from metaflow.decorators import StepDecorator
 from metaflow.metaflow_current import current
+from metaflow.user_configs.config_options import ConfigInput
+from metaflow.user_configs.config_parameters import dump_config_values
 from metaflow.util import to_unicode
+
 from .component_serializer import CardComponentCollector, get_card_class
 from .card_creator import CardCreator
-
-
-# from metaflow import get_metadata
-import re
-
 from .exception import CARD_ID_PATTERN, TYPE_CHECK_REGEX
 
 ASYNC_TIMEOUT = 30
@@ -111,6 +114,14 @@ class CardDecorator(StepDecorator):
         self._logger = logger
         self.card_options = None
 
+        # We check for configuration options. We do this here before they are
+        # converted to properties.
+        self._config_values = [
+            (config.name, ConfigInput.make_key_name(config.name))
+            for _, config in flow._get_parameters()
+            if config.IS_CONFIG_PARAMETER
+        ]
+
         self.card_options = self.attributes["options"]
 
         evt_name = "step-init"
@@ -146,6 +157,18 @@ class CardDecorator(StepDecorator):
         self._task_datastore = task_datastore
         self._metadata = metadata
 
+        # If we have configs, we need to dump them to a file so we can re-use them
+        # when calling the card creation subprocess.
+        if self._config_values:
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", delete=False
+            ) as config_file:
+                config_value = dump_config_values(flow)
+                json.dump(config_value, config_file)
+                self._config_file_name = config_file.name
+        else:
+            self._config_file_name = None
+
         card_type = self.attributes["type"]
         card_class = get_card_class(card_type)
 
@@ -179,7 +202,7 @@ class CardDecorator(StepDecorator):
         # we need to ensure that `current.card` has `CardComponentCollector` instantiated only once.
         if not self._is_event_registered("pre-step"):
             self._register_event("pre-step")
-            self._set_card_creator(CardCreator(self._create_top_level_args()))
+            self._set_card_creator(CardCreator(self._create_top_level_args(flow)))
 
             current._update_env(
                 {"card": CardComponentCollector(self._logger, self.card_creator)}
@@ -223,6 +246,13 @@ class CardDecorator(StepDecorator):
             self.card_creator.create(mode="render", final=True, **create_options)
             self.card_creator.create(mode="refresh", final=True, **create_options)
 
+        # Unlink the config file if it exists
+        if self._config_file_name:
+            try:
+                os.unlink(self._config_file_name)
+            except Exception as e:
+                pass
+
     @staticmethod
     def _options(mapping):
         for k, v in mapping.items():
@@ -232,9 +262,13 @@ class CardDecorator(StepDecorator):
                 for value in v:
                     yield "--%s" % k
                     if not isinstance(value, bool):
-                        yield to_unicode(value)
+                        if isinstance(value, tuple):
+                            for val in value:
+                                yield to_unicode(val)
+                        else:
+                            yield to_unicode(value)
 
-    def _create_top_level_args(self):
+    def _create_top_level_args(self, flow):
         top_level_options = {
             "quiet": True,
             "metadata": self._metadata.TYPE,
@@ -247,4 +281,8 @@ class CardDecorator(StepDecorator):
             # We don't provide --with as all execution is taking place in
             # the context of the main process
         }
+        if self._config_values:
+            top_level_options["config-value"] = self._config_values
+            top_level_options["local-config-file"] = self._config_file_name
+
         return list(self._options(top_level_options))

metaflow/runner/click_api.py

@@ -18,6 +18,7 @@ import json
 from collections import OrderedDict
 from typing import Any, Callable, Dict, List, Optional
 from typing import OrderedDict as TOrderedDict
+from typing import Tuple as TTuple
 from typing import Union
 
 from metaflow import FlowSpec, Parameter
@@ -38,8 +39,16 @@ from metaflow._vendor.typeguard import TypeCheckError, check_type
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
-from metaflow.user_configs.config_options import LocalFileInput
+from metaflow.user_configs.config_options import (
+    ConfigValue,
+    ConvertDictOrStr,
+    ConvertPath,
+    LocalFileInput,
+    MultipleTuple,
+    config_options_with_config_input,
+)
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -58,6 +67,7 @@ click_to_python_types = {
     JSONTypeClass: JSON,
     FilePathClass: str,
     LocalFileInput: str,
+    MultipleTuple: TTuple[str, Union[JSON, ConfigValue]],
 }
 
 
@@ -68,7 +78,7 @@ def _method_sanity_check(
     defaults: TOrderedDict[str, Any],
     **kwargs
 ) -> Dict[str, Any]:
-    method_params = {"args": {}, "options": {}}
+    method_params = {"args": {}, "options": {}, "defaults": defaults}
 
     possible_params = OrderedDict()
     possible_params.update(possible_arg_params)
@@ -90,10 +100,26 @@ def _method_sanity_check(
                 % (supplied_k, annotations[supplied_k], defaults[supplied_k])
             )
 
-        # because Click expects stringified JSON..
-        supplied_v = (
-            json.dumps(supplied_v) if annotations[supplied_k] == JSON else supplied_v
-        )
+        # Clean up values to make them into what click expects
+        if annotations[supplied_k] == JSON:
+            # JSON should be a string (json dumps)
+            supplied_v = json.dumps(supplied_v)
+        elif supplied_k == "config_value":
+            # Special handling of config value because we need to go look in the tuple
+            new_list = []
+            for cfg_name, cfg_value in supplied_v:
+                if isinstance(cfg_value, ConfigValue):
+                    # ConfigValue should be JSONified and converted to a string
+                    new_list.append((cfg_name, json.dumps(cfg_value.to_dict())))
+                elif isinstance(cfg_value, dict):
+                    # ConfigValue passed as a dictionary
+                    new_list.append((cfg_name, json.dumps(cfg_value)))
+                else:
+                    raise TypeError(
+                        "Invalid type for a config-value, expected a ConfigValue or "
+                        "dict but got '%s'" % type(cfg_value)
+                    )
+            supplied_v = new_list
 
         if supplied_k in possible_arg_params:
             cli_name = possible_arg_params[supplied_k].opts[0].strip("-")
@@ -188,35 +214,56 @@ loaded_modules = {}
 def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
     if not os.path.exists(flow_file):
         raise FileNotFoundError("Flow file not present at '%s'" % flow_file)
-    # Check if the module has already been loaded
-    if flow_file in loaded_modules:
-        module = loaded_modules[flow_file]
-    else:
-        # Load the module if it's not already loaded
-        spec = importlib.util.spec_from_file_location("module", flow_file)
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)
-        # Cache the loaded module
-        loaded_modules[flow_file] = module
-    classes = inspect.getmembers(module, inspect.isclass)
-
-    flow_cls = None
-    for _, kls in classes:
-        if kls != FlowSpec and issubclass(kls, FlowSpec):
-            if flow_cls is not None:
-                raise MetaflowException(
-                    "Multiple FlowSpec classes found in %s" % flow_file
-                )
-            flow_cls = kls
-
-    return flow_cls
+
+    flow_dir = os.path.dirname(os.path.abspath(flow_file))
+    path_was_added = False
+
+    # Only add to path if it's not already there
+    if flow_dir not in sys.path:
+        sys.path.insert(0, flow_dir)
+        path_was_added = True
+
+    try:
+        # Check if the module has already been loaded
+        if flow_file in loaded_modules:
+            module = loaded_modules[flow_file]
+        else:
+            # Load the module if it's not already loaded
+            spec = importlib.util.spec_from_file_location("module", flow_file)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            # Cache the loaded module
+            loaded_modules[flow_file] = module
+        classes = inspect.getmembers(module, inspect.isclass)
+
+        flow_cls = None
+        for _, kls in classes:
+            if kls != FlowSpec and issubclass(kls, FlowSpec):
+                if flow_cls is not None:
+                    raise MetaflowException(
+                        "Multiple FlowSpec classes found in %s" % flow_file
+                    )
+                flow_cls = kls
+
+        if flow_cls is None:
+            raise MetaflowException("No FlowSpec class found in %s" % flow_file)
+        return flow_cls
+    finally:
+        # Only remove from path if we added it
+        if path_was_added:
+            try:
+                sys.path.remove(flow_dir)
+            except ValueError:
+                # User's code might have removed it already
+                pass
 
 
 class MetaflowAPI(object):
-    def __init__(self, parent=None, flow_cls=None, **kwargs):
+    def __init__(self, parent=None, flow_cls=None, config_input=None, **kwargs):
         self._parent = parent
         self._chain = [{self._API_NAME: kwargs}]
         self._flow_cls = flow_cls
+        self._config_input = config_input
         self._cached_computed_parameters = None
 
     @property
@@ -238,11 +285,19 @@ class MetaflowAPI(object):
         flow_cls = extract_flow_class_from_file(flow_file)
 
         with flow_context(flow_cls) as _:
-            add_decorator_options(cli_collection)
+            cli_collection, config_input = config_options_with_config_input(
+                cli_collection
+            )
+            cli_collection = add_decorator_options(cli_collection)
 
         def getattr_wrapper(_self, name):
             # Functools.partial do not automatically bind self (no __get__)
-            return _self._internal_getattr(_self, name)
+            with flow_context(flow_cls) as _:
+                # We also wrap this in the proper flow context because since commands
+                # are loaded lazily, we need the proper flow context to compute things
+                # like parameters. If we do not do this, the outer flow's context will
+                # be used.
+                return _self._internal_getattr(_self, name)
 
         class_dict = {
             "__module__": "metaflow",
@@ -272,7 +327,12 @@ class MetaflowAPI(object):
                 defaults,
                 **kwargs,
             )
-            return to_return(parent=None, flow_cls=flow_cls, **method_params)
+            return to_return(
+                parent=None,
+                flow_cls=flow_cls,
+                config_input=config_input,
+                **method_params,
+            )
 
         m = _method
         m.__name__ = cli_collection.name
@@ -313,8 +373,12 @@ class MetaflowAPI(object):
                 for k, v in options.items():
                     if isinstance(v, list):
                         for i in v:
-                            components.append("--%s" % k)
-                            components.append(str(i))
+                            if isinstance(i, tuple):
+                                components.append("--%s" % k)
+                                components.extend(map(str, i))
+                            else:
+                                components.append("--%s" % k)
+                                components.append(str(i))
                     else:
                         components.append("--%s" % k)
                         if v != "flag":
@@ -323,21 +387,93 @@ class MetaflowAPI(object):
         return components
 
     def _compute_flow_parameters(self):
-        if self._flow_cls is None or self._parent is not None:
+        if (
+            self._flow_cls is None
+            or self._config_input is None
+            or self._parent is not None
+        ):
             raise RuntimeError(
                 "Computing flow-level parameters for a non start API. "
                 "Please report to the Metaflow team."
             )
-        # TODO: We need to actually compute the new parameters (based on configs) which
-        # would involve processing the options at least partially. We will do this
-        # before GA but for now making it work for regular parameters
+
         if self._cached_computed_parameters is not None:
             return self._cached_computed_parameters
         self._cached_computed_parameters = []
+
+        config_options = None
+        if CLICK_API_PROCESS_CONFIG:
+            with flow_context(self._flow_cls) as _:
+                # We are going to resolve the configs first and then get the parameters.
+                # Note that configs may update/add parameters so the order is important
+                # Since part of the processing of configs happens by click, we need to
+                # "fake" it.
+
+                # Extract any config options as well as datastore and quiet options
+                method_params = self._chain[0][self._API_NAME]
+                opts = method_params["options"]
+                defaults = method_params["defaults"]
+
+                ds = opts.get("datastore", defaults["datastore"])
+                quiet = opts.get("quiet", defaults["quiet"])
+                is_default = False
+                config_file = opts.get("config-file")
+                if config_file is None:
+                    is_default = True
+                    config_file = defaults.get("config_file")
+
+                if config_file:
+                    config_file = map(
+                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
+                        config_file,
+                    )
+
+                is_default = False
+                config_value = opts.get("config-value")
+                if config_value is None:
+                    is_default = True
+                    config_value = defaults.get("config_value")
+
+                if config_value:
+                    config_value = map(
+                        lambda x: (
+                            x[0],
+                            ConvertDictOrStr.convert_value(x[1], is_default),
+                        ),
+                        config_value,
+                    )
+
+                if (config_file is None) ^ (config_value is None):
+                    # If we have one, we should have the other
+                    raise MetaflowException(
+                        "Options were not properly set -- this is an internal error."
+                    )
+
+                if config_file:
+                    # Process both configurations; the second one will return all the merged
+                    # configuration options properly processed.
+                    self._config_input.process_configs(
+                        self._flow_cls.__name__, "config_file", config_file, quiet, ds
+                    )
+                    config_options = self._config_input.process_configs(
+                        self._flow_cls.__name__, "config_value", config_value, quiet, ds
+                    )
+
+        # At this point, we are like in start() in cli.py -- we obtained the
+        # properly processed config_options which we can now use to process
+        # the config decorators (including CustomStep/FlowDecorators)
+        # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
+        # it will init all parameters (config_options will be None)
+        # We ignore any errors if we don't check the configs in the click API.
+        new_cls = self._flow_cls._process_config_decorators(
+            config_options, ignore_errors=not CLICK_API_PROCESS_CONFIG
+        )
+        if new_cls:
+            self._flow_cls = new_cls
+
         for _, param in self._flow_cls._get_parameters():
             if param.IS_CONFIG_PARAMETER:
                 continue
-            param.init()
             self._cached_computed_parameters.append(param)
         return self._cached_computed_parameters