ob-metaflow 2.12.30.2__py2.py3-none-any.whl → 2.13.6.1__py2.py3-none-any.whl

metaflow/lint.py

@@ -52,7 +52,7 @@ def check_reserved_words(graph):
     msg = "Step name *%s* is a reserved word. Choose another name for the " "step."
     for node in graph:
         if node.name in RESERVED:
-            raise LintWarn(msg % node.name)
+            raise LintWarn(msg % node.name, node.func_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -76,9 +76,9 @@ def check_that_end_is_end(graph):
     node = graph["end"]
 
     if node.has_tail_next or node.invalid_tail_next:
-        raise LintWarn(msg0, node.tail_next_lineno)
+        raise LintWarn(msg0, node.tail_next_lineno, node.source_file)
     if node.num_args > 1:
-        raise LintWarn(msg1, node.tail_next_lineno)
+        raise LintWarn(msg1, node.tail_next_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -90,7 +90,7 @@ def check_step_names(graph):
     )
     for node in graph:
         if re.search("[^a-z0-9_]", node.name) or node.name[0] == "_":
-            raise LintWarn(msg.format(node), node.func_lineno)
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -108,11 +108,11 @@ def check_num_args(graph):
     msg2 = "Step *{0.name}* is missing the 'self' argument."
     for node in graph:
         if node.num_args > 2:
-            raise LintWarn(msg0.format(node), node.func_lineno)
+            raise LintWarn(msg0.format(node), node.func_lineno, node.source_file)
         elif node.num_args == 2 and node.type != "join":
-            raise LintWarn(msg1.format(node), node.func_lineno)
+            raise LintWarn(msg1.format(node), node.func_lineno, node.source_file)
         elif node.num_args == 0:
-            raise LintWarn(msg2.format(node), node.func_lineno)
+            raise LintWarn(msg2.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -125,7 +125,7 @@ def check_static_transitions(graph):
     )
     for node in graph:
         if node.type != "end" and not node.has_tail_next:
-            raise LintWarn(msg.format(node), node.func_lineno)
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -138,7 +138,7 @@ def check_valid_transitions(graph):
     )
     for node in graph:
         if node.type != "end" and node.has_tail_next and node.invalid_tail_next:
-            raise LintWarn(msg.format(node), node.tail_next_lineno)
+            raise LintWarn(msg.format(node), node.tail_next_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -151,7 +151,11 @@ def check_unknown_transitions(graph):
     for node in graph:
         unknown = [n for n in node.out_funcs if n not in graph]
         if unknown:
-            raise LintWarn(msg.format(node, step=unknown[0]), node.tail_next_lineno)
+            raise LintWarn(
+                msg.format(node, step=unknown[0]),
+                node.tail_next_lineno,
+                node.source_file,
+            )
 
 
 @linter.ensure_acyclicity
@@ -167,7 +171,9 @@ def check_for_acyclicity(graph):
         for n in node.out_funcs:
             if n in seen:
                 path = "->".join(seen + [n])
-                raise LintWarn(msg.format(path), node.tail_next_lineno)
+                raise LintWarn(
+                    msg.format(path), node.tail_next_lineno, node.source_file
+                )
             else:
                 check_path(graph[n], seen + [n])
 
@@ -195,7 +201,7 @@ def check_for_orphans(graph):
     orphans = nodeset - seen
     if orphans:
         orphan = graph[list(orphans)[0]]
-        raise LintWarn(msg.format(orphan), orphan.func_lineno)
+        raise LintWarn(msg.format(orphan), orphan.func_lineno, orphan.source_file)
 
 
 @linter.ensure_static_graph
@@ -230,7 +236,9 @@ def check_split_join_balance(graph):
             if split_stack:
                 _, split_roots = split_stack.pop()
                 roots = ", ".join(split_roots)
-                raise LintWarn(msg0.format(roots=roots))
+                raise LintWarn(
+                    msg0.format(roots=roots), node.func_lineno, node.source_file
+                )
         elif node.type == "join":
             if split_stack:
                 _, split_roots = split_stack[-1]
@@ -243,9 +251,10 @@ def check_split_join_balance(graph):
                             node, paths=paths, num_roots=len(split_roots), roots=roots
                         ),
                         node.func_lineno,
+                        node.source_file,
                     )
             else:
-                raise LintWarn(msg2.format(node), node.func_lineno)
+                raise LintWarn(msg2.format(node), node.func_lineno, node.source_file)
 
             # check that incoming steps come from the same lineage
             # (no cross joins)
@@ -256,7 +265,7 @@ def check_split_join_balance(graph):
                     return tuple(graph[n].split_parents)
 
             if not all_equal(map(parents, node.in_funcs)):
-                raise LintWarn(msg3.format(node), node.func_lineno)
+                raise LintWarn(msg3.format(node), node.func_lineno, node.source_file)
 
         for n in node.out_funcs:
             traverse(graph[n], new_stack)
@@ -276,7 +285,9 @@ def check_empty_foreaches(graph):
         if node.type == "foreach":
             joins = [n for n in node.out_funcs if graph[n].type == "join"]
             if joins:
-                raise LintWarn(msg.format(node, join=joins[0]))
+                raise LintWarn(
+                    msg.format(node, join=joins[0]), node.func_lineno, node.source_file
+                )
 
 
 @linter.ensure_static_graph
@@ -290,7 +301,7 @@ def check_parallel_step_after_next(graph):
         if node.parallel_foreach and not all(
             graph[out_node].parallel_step for out_node in node.out_funcs
         ):
-            raise LintWarn(msg.format(node))
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -303,7 +314,9 @@ def check_join_followed_by_parallel_step(graph):
     )
     for node in graph:
         if node.parallel_step and not graph[node.out_funcs[0]].type == "join":
-            raise LintWarn(msg.format(node.out_funcs[0]))
+            raise LintWarn(
+                msg.format(node.out_funcs[0]), node.func_lineno, node.source_file
+            )
 
 
 @linter.ensure_static_graph
@@ -318,7 +331,9 @@ def check_parallel_foreach_calls_parallel_step(graph):
             for node2 in graph:
                 if node2.out_funcs and node.name in node2.out_funcs:
                     if not node2.parallel_foreach:
-                        raise LintWarn(msg.format(node, node2))
+                        raise LintWarn(
+                            msg.format(node, node2), node.func_lineno, node.source_file
+                        )
 
 
 @linter.ensure_non_nested_foreach
@@ -331,4 +346,4 @@ def check_nested_foreach(graph):
     for node in graph:
         if node.type == "foreach":
             if any(graph[p].type == "foreach" for p in node.split_parents):
-                raise LintWarn(msg.format(node))
+                raise LintWarn(msg.format(node), node.func_lineno, node.source_file)

metaflow/metadata_provider/heartbeat.py

@@ -1,11 +1,12 @@
+import json
 import time
+from threading import Thread
+
 import requests
-import json
 
-from threading import Thread
-from metaflow.sidecar import MessageTypes, Message
-from metaflow.metaflow_config import SERVICE_HEADERS
 from metaflow.exception import MetaflowException
+from metaflow.metaflow_config import SERVICE_HEADERS
+from metaflow.sidecar import Message, MessageTypes
 
 HB_URL_KEY = "hb_url"
 
@@ -52,13 +53,27 @@ class MetadataHeartBeat(object):
                 retry_counter = 0
             except HeartBeatException as e:
                 retry_counter = retry_counter + 1
-                time.sleep(4**retry_counter)
+                time.sleep(1.5**retry_counter)
 
     def _heartbeat(self):
         if self.hb_url is not None:
-            response = requests.post(
-                url=self.hb_url, data="{}", headers=self.headers.copy()
-            )
+            try:
+                response = requests.post(
+                    url=self.hb_url, data="{}", headers=self.headers.copy()
+                )
+            except requests.exceptions.ConnectionError as e:
+                raise HeartBeatException(
+                    "HeartBeat request (%s) failed" " (ConnectionError)" % (self.hb_url)
+                )
+            except requests.exceptions.Timeout as e:
+                raise HeartBeatException(
+                    "HeartBeat request (%s) failed" " (Timeout)" % (self.hb_url)
+                )
+            except requests.exceptions.RequestException as e:
+                raise HeartBeatException(
+                    "HeartBeat request (%s) failed"
+                    " (RequestException) %s" % (self.hb_url, str(e))
+                )
             # Unfortunately, response.json() returns a string that we need
             # to cast to json; however when the request encounters an error
             # the return type is a json blob :/

metaflow/metaflow_config.py

@@ -356,6 +356,8 @@ KUBERNETES_PERSISTENT_VOLUME_CLAIMS = from_conf(
 KUBERNETES_SECRETS = from_conf("KUBERNETES_SECRETS", "")
 # Default labels for kubernetes pods
 KUBERNETES_LABELS = from_conf("KUBERNETES_LABELS", "")
+# Default annotations for kubernetes pods
+KUBERNETES_ANNOTATIONS = from_conf("KUBERNETES_ANNOTATIONS", "")
 # Default GPU vendor to use by K8S jobs created by Metaflow (supports nvidia, amd)
 KUBERNETES_GPU_VENDOR = from_conf("KUBERNETES_GPU_VENDOR", "nvidia")
 # Default container image for K8S
@@ -378,6 +380,8 @@ KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 KUBERNETES_CPU = from_conf("KUBERNETES_CPU", None)
 KUBERNETES_MEMORY = from_conf("KUBERNETES_MEMORY", None)
 KUBERNETES_DISK = from_conf("KUBERNETES_DISK", None)
+# Default kubernetes QoS class
+KUBERNETES_QOS = from_conf("KUBERNETES_QOS", "burstable")
 
 ARGO_WORKFLOWS_KUBERNETES_SECRETS = from_conf("ARGO_WORKFLOWS_KUBERNETES_SECRETS", "")
 ARGO_WORKFLOWS_ENV_VARS_TO_SKIP = from_conf("ARGO_WORKFLOWS_ENV_VARS_TO_SKIP", "")
@@ -439,7 +443,7 @@ ESCAPE_HATCH_WARNING = from_conf("ESCAPE_HATCH_WARNING", True)
 ###
 # Debug configuration
 ###
-DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen"]
+DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen", "userconf"]
 
 for typ in DEBUG_OPTIONS:
     vars()["DEBUG_%s" % typ.upper()] = from_conf("DEBUG_%s" % typ.upper(), False)
@@ -509,6 +513,11 @@ MAX_CPU_PER_TASK = from_conf("MAX_CPU_PER_TASK")
 # lexicographic ordering of attempts. This won't work if MAX_ATTEMPTS > 99.
 MAX_ATTEMPTS = 6
 
+# Feature flag (experimental features that are *explicitly* unsupported)
+
+# Process configs even when using the click_api for Runner/Deployer
+CLICK_API_PROCESS_CONFIG = from_conf("CLICK_API_PROCESS_CONFIG", False)
+
 
 # PINNED_CONDA_LIBS are the libraries that metaflow depends on for execution
 # and are needed within a conda environment

metaflow/multicore_utils.py

@@ -6,7 +6,18 @@ from tempfile import NamedTemporaryFile
 import time
 import metaflow.tracing as tracing
 
-from typing import Any, Callable, Iterable, Iterator, List, Optional
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    NoReturn,
+    Tuple,
+    TypeVar,
+    Union,
+)
 
 try:
     # Python 2
@@ -30,7 +41,13 @@ class MulticoreException(Exception):
     pass
 
 
-def _spawn(func, arg, dir):
+_A = TypeVar("_A")
+_R = TypeVar("_R")
+
+
+def _spawn(
+    func: Callable[[_A], _R], arg: _A, dir: Optional[str]
+) -> Union[Tuple[int, str], NoReturn]:
     with NamedTemporaryFile(prefix="parallel_map_", dir=dir, delete=False) as tmpfile:
         output_file = tmpfile.name
 
@@ -63,11 +80,11 @@ def _spawn(func, arg, dir):
 
 
 def parallel_imap_unordered(
-    func: Callable[[Any], Any],
-    iterable: Iterable[Any],
+    func: Callable[[_A], _R],
+    iterable: Iterable[_A],
     max_parallel: Optional[int] = None,
     dir: Optional[str] = None,
-) -> Iterator[Any]:
+) -> Iterator[_R]:
     """
     Parallelizes execution of a function using multiprocessing. The result
     order is not guaranteed.
@@ -79,9 +96,9 @@ def parallel_imap_unordered(
     iterable : Iterable[Any]
         Iterable over arguments to pass to fun
     max_parallel int, optional, default None
-        Maximum parallelism. If not specified, uses the number of CPUs
+        Maximum parallelism. If not specified, it uses the number of CPUs
     dir : str, optional, default None
-        If specified, directory where temporary files are created
+        If specified, it's the directory where temporary files are created
 
     Yields
     ------
@@ -121,14 +138,14 @@ def parallel_imap_unordered(
 
 
 def parallel_map(
-    func: Callable[[Any], Any],
-    iterable: Iterable[Any],
+    func: Callable[[_A], _R],
+    iterable: Iterable[_A],
     max_parallel: Optional[int] = None,
     dir: Optional[str] = None,
-) -> List[Any]:
+) -> List[_R]:
     """
     Parallelizes execution of a function using multiprocessing. The result
-    order is that of the arguments in `iterable`
+    order is that of the arguments in `iterable`.
 
     Parameters
     ----------
@@ -137,9 +154,9 @@ def parallel_map(
     iterable : Iterable[Any]
         Iterable over arguments to pass to fun
     max_parallel int, optional, default None
-        Maximum parallelism. If not specified, uses the number of CPUs
+        Maximum parallelism. If not specified, it uses the number of CPUs
     dir : str, optional, default None
-        If specified, directory where temporary files are created
+        If specified, it's the directory where temporary files are created
 
     Returns
     -------
@@ -155,4 +172,4 @@ def parallel_map(
     res = parallel_imap_unordered(
         wrapper, enumerate(iterable), max_parallel=max_parallel, dir=dir
     )
-    return [r for idx, r in sorted(res)]
+    return [r for _, r in sorted(res)]

metaflow/package.py

@@ -6,6 +6,7 @@ import time
 import json
 from io import BytesIO
 
+from .user_configs.config_parameters import CONFIG_FILE, dump_config_values
 from .extension_support import EXT_PKG, package_mfext_all
 from .metaflow_config import DEFAULT_PACKAGE_SUFFIXES
 from .exception import MetaflowException
@@ -151,11 +152,23 @@ class MetaflowPackage(object):
             for path_tuple in self._walk(flowdir, suffixes=self.suffixes):
                 yield path_tuple
 
+    def _add_configs(self, tar):
+        buf = BytesIO()
+        buf.write(json.dumps(dump_config_values(self._flow)).encode("utf-8"))
+        self._add_file(tar, os.path.basename(CONFIG_FILE), buf)
+
     def _add_info(self, tar):
-        info = tarfile.TarInfo(os.path.basename(INFO_FILE))
-        env = self.environment.get_environment_info(include_ext_info=True)
         buf = BytesIO()
-        buf.write(json.dumps(env).encode("utf-8"))
+        buf.write(
+            json.dumps(
+                self.environment.get_environment_info(include_ext_info=True)
+            ).encode("utf-8")
+        )
+        self._add_file(tar, os.path.basename(INFO_FILE), buf)
+
+    @staticmethod
+    def _add_file(tar, filename, buf):
+        info = tarfile.TarInfo(filename)
         buf.seek(0)
         info.size = len(buf.getvalue())
         # Setting this default to Dec 3, 2019
@@ -175,6 +188,7 @@ class MetaflowPackage(object):
             fileobj=buf, mode="w:gz", compresslevel=3, dereference=True
         ) as tar:
             self._add_info(tar)
+            self._add_configs(tar)
             for path, arcname in self.path_tuples():
                 tar.add(path, arcname=arcname, recursive=False, filter=no_mtime)
 

metaflow/parameters.py

@@ -3,7 +3,7 @@ import json
 from contextlib import contextmanager
 from threading import local
 
-from typing import Any, Callable, Dict, NamedTuple, Optional, Type, Union
+from typing import Any, Callable, Dict, NamedTuple, Optional, TYPE_CHECKING, Type, Union
 
 from metaflow._vendor import click
 
@@ -14,6 +14,9 @@ from .exception import (
     MetaflowException,
 )
 
+if TYPE_CHECKING:
+    from .user_configs.config_parameters import ConfigValue
+
 try:
     # Python2
     strtype = basestring
@@ -32,6 +35,7 @@ ParameterContext = NamedTuple(
         ("parameter_name", str),
         ("logger", Callable[..., None]),
         ("ds_type", str),
+        ("configs", Optional["ConfigValue"]),
     ],
 )
 
@@ -72,6 +76,16 @@ def flow_context(flow_cls):
 context_proto = None
 
 
+def replace_flow_context(flow_cls):
+    """
+    Replace the current flow context with a new flow class. This is used
+    when we change the current flow class after having run user configuration functions
+    """
+    current_flow.flow_cls_stack = current_flow.flow_cls_stack[1:]
+    current_flow.flow_cls_stack.insert(0, flow_cls)
+    current_flow.flow_cls = current_flow.flow_cls_stack[0]
+
+
 class JSONTypeClass(click.ParamType):
     name = "JSON"
 
@@ -151,6 +165,7 @@ class DeployTimeField(object):
             return self._check_type(val, deploy_time)
 
     def _check_type(self, val, deploy_time):
+
         # it is easy to introduce a deploy-time function that accidentally
         # returns a value whose type is not compatible with what is defined
         # in Parameter. Let's catch those mistakes early here, instead of
@@ -158,7 +173,7 @@ class DeployTimeField(object):
 
         # note: this doesn't work with long in Python2 or types defined as
         # click types, e.g. click.INT
-        TYPES = {bool: "bool", int: "int", float: "float", list: "list"}
+        TYPES = {bool: "bool", int: "int", float: "float", list: "list", dict: "dict"}
 
         msg = (
             "The value returned by the deploy-time function for "
@@ -166,7 +181,12 @@ class DeployTimeField(object):
             % (self.parameter_name, self.field)
         )
 
-        if self.parameter_type in TYPES:
+        if isinstance(self.parameter_type, list):
+            if not any(isinstance(val, x) for x in self.parameter_type):
+                msg += "Expected one of the following %s." % TYPES[self.parameter_type]
+                raise ParameterFieldTypeMismatch(msg)
+            return str(val) if self.return_str else val
+        elif self.parameter_type in TYPES:
             if type(val) != self.parameter_type:
                 msg += "Expected a %s." % TYPES[self.parameter_type]
                 raise ParameterFieldTypeMismatch(msg)
@@ -204,12 +224,18 @@ class DeployTimeField(object):
 def deploy_time_eval(value):
     if isinstance(value, DeployTimeField):
         return value(deploy_time=True)
+    elif isinstance(value, DelayedEvaluationParameter):
+        return value(return_str=True)
     else:
         return value
 
 
 # this is called by cli.main
-def set_parameter_context(flow_name, echo, datastore):
+def set_parameter_context(flow_name, echo, datastore, configs):
+    from .user_configs.config_parameters import (
+        ConfigValue,
+    )  # Prevent circular dependency
+
     global context_proto
     context_proto = ParameterContext(
         flow_name=flow_name,
@@ -217,6 +243,7 @@ def set_parameter_context(flow_name, echo, datastore):
         parameter_name=None,
         logger=echo,
         ds_type=datastore.TYPE,
+        configs=ConfigValue(dict(configs)),
     )
 
 
@@ -273,7 +300,11 @@ class Parameter(object):
     ----------
     name : str
         User-visible parameter name.
-    default : str or float or int or bool or `JSONType` or a function.
+    default : Union[str, float, int, bool, Dict[str, Any],
+                Callable[
+                    [ParameterContext], Union[str, float, int, bool, Dict[str, Any]]
+                ],
+            ], optional, default None
         Default value for the parameter. Use a special `JSONType` class to
         indicate that the value must be a valid JSON object. A function
         implies that the parameter corresponds to a *deploy-time parameter*.
@@ -282,15 +313,19 @@ class Parameter(object):
         If `default` is not specified, define the parameter type. Specify
         one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
         to the type of `default` or `str` if none specified.
-    help : str, optional
+    help : str, optional, default None
         Help text to show in `run --help`.
-    required : bool, default False
-        Require that the user specified a value for the parameter.
-        `required=True` implies that the `default` is not used.
-    show_default : bool, default True
-        If True, show the default value in the help text.
+    required : bool, optional, default None
+        Require that the user specified a value for the parameter. Note that if
+        a default is provide, the required flag is ignored.
+        A value of None is equivalent to False.
+    show_default : bool, optional, default None
+        If True, show the default value in the help text. A value of None is equivalent
+        to True.
     """
 
+    IS_CONFIG_PARAMETER = False
+
     def __init__(
         self,
         name: str,
@@ -301,31 +336,62 @@ class Parameter(object):
                 int,
                 bool,
                 Dict[str, Any],
-                Callable[[], Union[str, float, int, bool, Dict[str, Any]]],
+                Callable[
+                    [ParameterContext], Union[str, float, int, bool, Dict[str, Any]]
+                ],
             ]
         ] = None,
         type: Optional[
             Union[Type[str], Type[float], Type[int], Type[bool], JSONTypeClass]
         ] = None,
         help: Optional[str] = None,
-        required: bool = False,
-        show_default: bool = True,
+        required: Optional[bool] = None,
+        show_default: Optional[bool] = None,
         **kwargs: Dict[str, Any]
     ):
         self.name = name
         self.kwargs = kwargs
-        for k, v in {
+        self._override_kwargs = {
             "default": default,
             "type": type,
             "help": help,
             "required": required,
             "show_default": show_default,
-        }.items():
-            if v is not None:
-                self.kwargs[k] = v
+        }
+
+    def init(self, ignore_errors=False):
+        # Prevent circular import
+        from .user_configs.config_parameters import (
+            resolve_delayed_evaluator,
+            unpack_delayed_evaluator,
+        )
+
+        # Resolve any value from configurations
+        self.kwargs, _ = unpack_delayed_evaluator(
+            self.kwargs, ignore_errors=ignore_errors
+        )
+        # Do it one item at a time so errors are ignored at that level (as opposed to
+        # at the entire kwargs level)
+        self.kwargs = {
+            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
+            for k, v in self.kwargs.items()
+        }
+
+        # This was the behavior before configs: values specified in args would override
+        # stuff in kwargs which is what we implement here as well
+        for key, value in self._override_kwargs.items():
+            if value is not None:
+                self.kwargs[key] = resolve_delayed_evaluator(
+                    value, ignore_errors=ignore_errors
+                )
+        # Set two default values if no-one specified them
+        self.kwargs.setdefault("required", False)
+        self.kwargs.setdefault("show_default", True)
+
+        # Continue processing kwargs free of any configuration values :)
 
         # TODO: check that the type is one of the supported types
-        param_type = self.kwargs["type"] = self._get_type(kwargs)
+        param_type = self.kwargs["type"] = self._get_type(self.kwargs)
 
         reserved_params = [
             "params",
@@ -350,23 +416,27 @@ class Parameter(object):
             raise MetaflowException(
                 "Parameter name '%s' is a reserved "
                 "word. Please use a different "
-                "name for your parameter." % (name)
+                "name for your parameter." % (self.name)
             )
 
         # make sure the user is not trying to pass a function in one of the
         # fields that don't support function-values yet
         for field in ("show_default", "separator", "required"):
-            if callable(kwargs.get(field)):
+            if callable(self.kwargs.get(field)):
                 raise MetaflowException(
                     "Parameter *%s*: Field '%s' cannot "
-                    "have a function as its value" % (name, field)
+                    "have a function as its value" % (self.name, field)
                 )
 
         # default can be defined as a function
         default_field = self.kwargs.get("default")
         if callable(default_field) and not isinstance(default_field, DeployTimeField):
             self.kwargs["default"] = DeployTimeField(
-                name, param_type, "default", self.kwargs["default"], return_str=True
+                self.name,
+                param_type,
+                "default",
+                self.kwargs["default"],
+                return_str=True,
             )
 
         # note that separator doesn't work with DeployTimeFields unless you
@@ -375,7 +445,7 @@ class Parameter(object):
         if self.separator and not self.is_string_type:
             raise MetaflowException(
                 "Parameter *%s*: Separator is only allowed "
-                "for string parameters." % name
+                "for string parameters." % self.name
             )
 
     def __repr__(self):
@@ -431,7 +501,9 @@ def add_custom_parameters(deploy_mode=False):
         flow_cls = getattr(current_flow, "flow_cls", None)
         if flow_cls is None:
             return cmd
-        parameters = [p for _, p in flow_cls._get_parameters()]
+        parameters = [
+            p for _, p in flow_cls._get_parameters() if not p.IS_CONFIG_PARAMETER
+        ]
         for arg in parameters[::-1]:
             kwargs = arg.option_kwargs(deploy_mode)
             cmd.params.insert(0, click.Option(("--" + arg.name,), **kwargs))