metaflow 2.13.8__tar.gz → 2.13.10__tar.gz

{metaflow-2.13.8/metaflow.egg-info → metaflow-2.13.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.8
+Version: 2.13.10
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.8; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.10; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/cli.py

@@ -206,15 +206,15 @@ def output_raw(obj, json):
     else:
         _graph = str(obj.graph)
         _msg = "Internal representation of the flow:"
-    echo(_msg, fg="magenta", bold=False)
+    echo_always(_msg, fg="magenta", bold=False)
     echo_always(_graph, err=False)
 
 
 @cli.command(help="Visualize the flow with Graphviz.")
 @click.pass_obj
 def output_dot(obj):
-    echo("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False)
-    echo(
+    echo_always("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False)
+    echo_always(
         "Try piping the output to 'dot -Tpng -o graph.png' to produce "
         "an actual image.",
         indent=True,
@@ -330,7 +330,7 @@ def start(
     event_logger=None,
     monitor=None,
     local_config_file=None,
-    config_file=None,
+    config=None,
     config_value=None,
     **deco_options
 ):
@@ -383,7 +383,7 @@ def start(
     # When we process the options, the first one processed will return None and the
     # second one processed will return the actual options. The order of processing
     # depends on what (and in what order) the user specifies on the command line.
-    config_options = config_file or config_value
+    config_options = config or config_value
 
     if (
         hasattr(ctx, "saved_args")
@@ -396,7 +396,7 @@ def start(
         # if we need to in the first place
         if getattr(ctx.obj, "has_cl_config_options", False):
             raise click.UsageError(
-                "Cannot specify --config-file or --config-value with 'resume'"
+                "Cannot specify --config or --config-value with 'resume'"
             )
         # We now load the config artifacts from the original run id
         run_id = None

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/cli_args.py

@@ -72,10 +72,10 @@ class CLIArgs(object):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
-            if k in ("config_file", "config_value"):
+            if k in ("config", "config_value"):
                 # Special handling here since we gather them all in one option but actually
                 # need to send them one at a time using --config-value <name> kv.<name>.
-                # Note it can be either config_file or config_value depending
+                # Note it can be either config or config_value depending
                 # on click processing order.
                 for config_name in v.keys():
                     yield "--config-value"

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/cmd/develop/stub_generator.py

@@ -1133,8 +1133,14 @@ class StubGenerator:
             result = result[1:]
         # Add doc to first and last overloads. Jedi uses the last one and pycharm
         # the first one. Go figure.
-        result[0] = (result[0][0], docs["func_doc"])
-        result[-1] = (result[-1][0], docs["func_doc"])
+        result[0] = (
+            result[0][0],
+            docs["func_doc"] + "\nParameters\n----------\n" + docs["param_doc"],
+        )
+        result[-1] = (
+            result[-1][0],
+            docs["func_doc"] + "\nParameters\n----------\n" + docs["param_doc"],
+        )
         return result
 
     def _generate_function_stub(

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/datastore/task_datastore.py

@@ -253,7 +253,7 @@ class TaskDataStore(object):
 
     @only_if_not_done
     @require_mode("w")
-    def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0):
+    def save_artifacts(self, artifacts_iter, len_hint=0):
         """
         Saves Metaflow Artifacts (Python objects) to the datastore and stores
         any relevant metadata needed to retrieve them.
@@ -269,11 +269,6 @@ class TaskDataStore(object):
         artifacts : Iterator[(string, object)]
             Iterator over the human-readable name of the object to save
             and the object itself
-        force_v4 : boolean or Dict[string -> boolean]
-            Indicates whether the artifact should be pickled using the v4
-            version of pickle. If a single boolean, applies to all artifacts.
-            If a dictionary, applies to the object named only. Defaults to False
-            if not present or not specified
         len_hint: integer
             Estimated number of items in artifacts_iter
         """
@@ -281,40 +276,24 @@ class TaskDataStore(object):
 
         def pickle_iter():
             for name, obj in artifacts_iter:
-                do_v4 = (
-                    force_v4 and force_v4
-                    if isinstance(force_v4, bool)
-                    else force_v4.get(name, False)
-                )
-                if do_v4:
-                    encode_type = "gzip+pickle-v4"
-                    if encode_type not in self._encodings:
-                        raise DataException(
-                            "Artifact *%s* requires a serialization encoding that "
-                            "requires Python 3.4 or newer." % name
-                        )
+                encode_type = "gzip+pickle-v4"
+                if encode_type in self._encodings:
                     try:
                         blob = pickle.dumps(obj, protocol=4)
                     except TypeError as e:
-                        raise UnpicklableArtifactException(name)
+                        raise UnpicklableArtifactException(name) from e
                 else:
                     try:
                         blob = pickle.dumps(obj, protocol=2)
                         encode_type = "gzip+pickle-v2"
-                    except (SystemError, OverflowError):
-                        encode_type = "gzip+pickle-v4"
-                        if encode_type not in self._encodings:
-                            raise DataException(
-                                "Artifact *%s* is very large (over 2GB). "
-                                "You need to use Python 3.4 or newer if you want to "
-                                "serialize large objects." % name
-                            )
-                        try:
-                            blob = pickle.dumps(obj, protocol=4)
-                        except TypeError as e:
-                            raise UnpicklableArtifactException(name)
+                    except (SystemError, OverflowError) as e:
+                        raise DataException(
+                            "Artifact *%s* is very large (over 2GB). "
+                            "You need to use Python 3.4 or newer if you want to "
+                            "serialize large objects." % name
+                        ) from e
                     except TypeError as e:
-                        raise UnpicklableArtifactException(name)
+                        raise UnpicklableArtifactException(name) from e
 
                 self._info[name] = {
                     "size": len(blob),

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/flowspec.py

@@ -223,17 +223,18 @@ class FlowSpec(metaclass=FlowSpecMeta):
             seen.add(norm)
 
     @classmethod
-    def _process_config_decorators(cls, config_options, ignore_errors=False):
+    def _process_config_decorators(cls, config_options, process_configs=True):
 
         # Fast path for no user configurations
-        if not cls._flow_state.get(_FlowState.CONFIG_DECORATORS) and all(
-            len(step.config_decorators) == 0 for step in cls._steps
+        if not process_configs or (
+            not cls._flow_state.get(_FlowState.CONFIG_DECORATORS)
+            and all(len(step.config_decorators) == 0 for step in cls._steps)
         ):
             # Process parameters to allow them to also use config values easily
             for var, param in cls._get_parameters():
                 if param.IS_CONFIG_PARAMETER:
                     continue
-                param.init(ignore_errors)
+                param.init(not process_configs)
             return None
 
         debug.userconf_exec("Processing mutating step/flow decorators")
@@ -258,6 +259,11 @@ class FlowSpec(metaclass=FlowSpecMeta):
             debug.userconf_exec("Setting config %s to %s" % (var, str(val)))
             setattr(cls, var, val)
 
+        # Reset cached parameters since we have replaced configs already with ConfigValue
+        # so they are not parameters anymore to be re-evaluated when we do _get_parameters
+        if _FlowState.CACHED_PARAMETERS in cls._flow_state:
+            del cls._flow_state[_FlowState.CACHED_PARAMETERS]
+
         # Run all the decorators. Step decorators are directly in the step and
         # we will run those first and *then* we run all the flow level decorators
         for step in cls._steps:
@@ -277,7 +283,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
                 setattr(cls, step.name, step)
 
         mutable_flow = MutableFlow(cls)
-        for deco in cls._flow_state[_FlowState.CONFIG_DECORATORS]:
+        for deco in cls._flow_state.get(_FlowState.CONFIG_DECORATORS, []):
             if isinstance(deco, CustomFlowDecorator):
                 # Sanity check to make sure we are applying the decorator to the right
                 # class

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/metaflow_environment.py

@@ -125,7 +125,9 @@ class MetaflowEnvironment(object):
             )
 
     def _get_install_dependencies_cmd(self, datastore_type):
-        base_cmd = "{} -m pip install -qqq".format(self._python())
+        base_cmd = "{} -m pip install -qqq --no-compile --no-cache-dir --disable-pip-version-check".format(
+            self._python()
+        )
 
         datastore_packages = {
             "s3": ["boto3"],

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/plugins/argo/argo_workflows.py

@@ -111,6 +111,9 @@ class ArgoWorkflows(object):
         notify_on_success=False,
         notify_slack_webhook_url=None,
         notify_pager_duty_integration_key=None,
+        notify_incident_io_api_key=None,
+        incident_io_success_severity_id=None,
+        incident_io_error_severity_id=None,
         enable_heartbeat_daemon=True,
         enable_error_msg_capture=False,
     ):
@@ -160,6 +163,9 @@ class ArgoWorkflows(object):
         self.notify_on_success = notify_on_success
         self.notify_slack_webhook_url = notify_slack_webhook_url
         self.notify_pager_duty_integration_key = notify_pager_duty_integration_key
+        self.notify_incident_io_api_key = notify_incident_io_api_key
+        self.incident_io_success_severity_id = incident_io_success_severity_id
+        self.incident_io_error_severity_id = incident_io_error_severity_id
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
         self.parameters = self._process_parameters()
@@ -891,6 +897,17 @@ class ArgoWorkflows(object):
                             and self.notify_pager_duty_integration_key
                             else {}
                         ),
+                        **(
+                            {
+                                # workflow status maps to Completed
+                                "notify-incident-io-on-success": LifecycleHook()
+                                .expression("workflow.status == 'Succeeded'")
+                                .template("notify-incident-io-on-success"),
+                            }
+                            if self.notify_on_success
+                            and self.notify_incident_io_api_key
+                            else {}
+                        ),
                         **(
                             {
                                 # workflow status maps to Failed or Error
@@ -918,6 +935,19 @@ class ArgoWorkflows(object):
                             and self.notify_pager_duty_integration_key
                             else {}
                         ),
+                        **(
+                            {
+                                # workflow status maps to Failed or Error
+                                "notify-incident-io-on-failure": LifecycleHook()
+                                .expression("workflow.status == 'Failed'")
+                                .template("notify-incident-io-on-error"),
+                                "notify-incident-io-on-error": LifecycleHook()
+                                .expression("workflow.status == 'Error'")
+                                .template("notify-incident-io-on-error"),
+                            }
+                            if self.notify_on_error and self.notify_incident_io_api_key
+                            else {}
+                        ),
                         # Warning: terrible hack to workaround a bug in Argo Workflow
                         #          where the hooks listed above do not execute unless
                         #          there is an explicit exit hook. as and when this
@@ -2270,9 +2300,11 @@ class ArgoWorkflows(object):
         if self.notify_on_error:
             templates.append(self._slack_error_template())
             templates.append(self._pager_duty_alert_template())
+            templates.append(self._incident_io_alert_template())
         if self.notify_on_success:
             templates.append(self._slack_success_template())
             templates.append(self._pager_duty_change_template())
+            templates.append(self._incident_io_change_template())
         if self.notify_on_error or self.notify_on_success:
             # Warning: terrible hack to workaround a bug in Argo Workflow where the
             #          templates listed above do not execute unless there is an
@@ -2466,6 +2498,82 @@ class ArgoWorkflows(object):
             )
         )
 
+    def _incident_io_alert_template(self):
+        if self.notify_incident_io_api_key is None:
+            return None
+        if self.incident_io_error_severity_id is None:
+            raise MetaflowException(
+                "Creating incidents for errors requires a severity id."
+            )
+        return Template("notify-incident-io-on-error").http(
+            Http("POST")
+            .url("https://api.incident.io/v2/incidents")
+            .header("Content-Type", "application/json")
+            .header("Authorization", "Bearer %s" % self.notify_incident_io_api_key)
+            .body(
+                json.dumps(
+                    {
+                        "idempotency_key": "argo-{{workflow.name}}",  # use run id to deduplicate alerts.
+                        "visibility": "public",
+                        "severity_id": self.incident_io_error_severity_id,
+                        "name": "Flow %s has failed." % self.flow.name,
+                        "summary": "Metaflow run %s/argo-{{workflow.name}} failed! %s"
+                        % (self.flow.name, self._incident_io_ui_urls_for_run()),
+                        # TODO: Add support for custom field entries.
+                    }
+                )
+            )
+        )
+
+    def _incident_io_change_template(self):
+        if self.notify_incident_io_api_key is None:
+            return None
+        if self.incident_io_success_severity_id is None:
+            raise MetaflowException(
+                "Creating incidents for successes requires a severity id."
+            )
+        return Template("notify-incident-io-on-success").http(
+            Http("POST")
+            .url("https://api.incident.io/v2/incidents")
+            .header("Content-Type", "application/json")
+            .header("Authorization", "Bearer %s" % self.notify_incident_io_api_key)
+            .body(
+                json.dumps(
+                    {
+                        "idempotency_key": "argo-{{workflow.name}}",  # use run id to deduplicate alerts.
+                        "visibility": "public",
+                        "severity_id": self.incident_io_success_severity_id,
+                        # TODO: Do we need to make incident type configurable for successes? otherwise they are created as 'investigating'
+                        # "incident_type_id": ""
+                        "name": "Flow %s has succeeded." % self.flow.name,
+                        "summary": "Metaflow run %s/argo-{{workflow.name}} succeeded!%s"
+                        % (self.flow.name, self._incident_io_ui_urls_for_run()),
+                        # TODO: Add support for custom field entries.
+                    }
+                )
+            )
+        )
+
+    def _incident_io_ui_urls_for_run(self):
+        links = []
+        if UI_URL:
+            url = "[Metaflow UI](%s/%s/%s)" % (
+                UI_URL.rstrip("/"),
+                self.flow.name,
+                "argo-{{workflow.name}}",
+            )
+            links.append(url)
+        if ARGO_WORKFLOWS_UI_URL:
+            url = "[Argo UI](%s/workflows/%s/%s)" % (
+                ARGO_WORKFLOWS_UI_URL.rstrip("/"),
+                "{{workflow.namespace}}",
+                "{{workflow.name}}",
+            )
+            links.append(url)
+        if links:
+            links = ["See details for the run at: ", *links]
+        return "\n\n".join(links)
+
     def _pager_duty_change_template(self):
         # https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgy-send-a-change-event
         if self.notify_pager_duty_integration_key is None:

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -176,6 +176,21 @@ def argo_workflows(obj, name=None):
     default="",
     help="PagerDuty Events API V2 Integration key for workflow success/failure notifications.",
 )
+@click.option(
+    "--notify-incident-io-api-key",
+    default=None,
+    help="Incident.io API V2 key for workflow success/failure notifications.",
+)
+@click.option(
+    "--incident-io-success-severity-id",
+    default=None,
+    help="Incident.io severity id for success alerts.",
+)
+@click.option(
+    "--incident-io-error-severity-id",
+    default=None,
+    help="Incident.io severity id for error alerts.",
+)
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
     default=False,
@@ -213,6 +228,9 @@ def create(
     notify_on_success=False,
     notify_slack_webhook_url=None,
     notify_pager_duty_integration_key=None,
+    notify_incident_io_api_key=None,
+    incident_io_success_severity_id=None,
+    incident_io_error_severity_id=None,
     enable_heartbeat_daemon=True,
     deployer_attribute_file=None,
     enable_error_msg_capture=False,
@@ -268,6 +286,9 @@ def create(
         notify_on_success,
         notify_slack_webhook_url,
         notify_pager_duty_integration_key,
+        notify_incident_io_api_key,
+        incident_io_success_severity_id,
+        incident_io_error_severity_id,
         enable_heartbeat_daemon,
         enable_error_msg_capture,
     )
@@ -442,6 +463,9 @@ def make_flow(
     notify_on_success,
     notify_slack_webhook_url,
     notify_pager_duty_integration_key,
+    notify_incident_io_api_key,
+    incident_io_success_severity_id,
+    incident_io_error_severity_id,
     enable_heartbeat_daemon,
     enable_error_msg_capture,
 ):
@@ -453,17 +477,30 @@ def make_flow(
         )
 
     if (notify_on_error or notify_on_success) and not (
-        notify_slack_webhook_url or notify_pager_duty_integration_key
+        notify_slack_webhook_url
+        or notify_pager_duty_integration_key
+        or notify_incident_io_api_key
     ):
         raise MetaflowException(
-            "Notifications require specifying an incoming Slack webhook url via --notify-slack-webhook-url or "
-            "PagerDuty events v2 integration key via --notify-pager-duty-integration-key.\n If you would like to set up "
-            "notifications for your Slack workspace, follow the instructions at "
-            "https://api.slack.com/messaging/webhooks to generate a webhook url.\n For notifications through PagerDuty, "
-            "generate an integration key by following the instructions at "
-            "https://support.pagerduty.com/docs/services-and-integrations#create-a-generic-events-api-integration"
+            "Notifications require specifying an incoming Slack webhook url via --notify-slack-webhook-url, PagerDuty events v2 integration key via --notify-pager-duty-integration-key or\n"
+            "Incident.io integration API key via --notify-incident-io-api-key.\n"
+            " If you would like to set up notifications for your Slack workspace, follow the instructions at "
+            "https://api.slack.com/messaging/webhooks to generate a webhook url.\n"
+            " For notifications through PagerDuty, generate an integration key by following the instructions at "
+            "https://support.pagerduty.com/docs/services-and-integrations#create-a-generic-events-api-integration\n"
+            " For notifications through Incident.io, generate an API key with a permission to create incidents."
         )
 
+    if notify_incident_io_api_key:
+        if notify_on_error and incident_io_error_severity_id is None:
+            raise MetaflowException(
+                "Incident.io error notifications require a severity id. Please set one with --incident-io-error-severity-id"
+            )
+
+        if notify_on_success and incident_io_success_severity_id is None:
+            raise MetaflowException(
+                "Incident.io success notifications require a severity id. Please set one with --incident-io-success-severity-id"
+            )
     # Attach @kubernetes and @environment decorator to the flow to
     # ensure that the related decorator hooks are invoked.
     decorators._attach_decorators(
@@ -507,6 +544,9 @@ def make_flow(
         notify_on_success=notify_on_success,
         notify_slack_webhook_url=notify_slack_webhook_url,
         notify_pager_duty_integration_key=notify_pager_duty_integration_key,
+        notify_incident_io_api_key=notify_incident_io_api_key,
+        incident_io_success_severity_id=incident_io_success_severity_id,
+        incident_io_error_severity_id=incident_io_error_severity_id,
         enable_heartbeat_daemon=enable_heartbeat_daemon,
         enable_error_msg_capture=enable_error_msg_capture,
     )

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/plugins/project_decorator.py

@@ -26,6 +26,24 @@ class ProjectDecorator(FlowDecorator):
         projects that use the same production scheduler. The name may
         contain only lowercase alphanumeric characters and underscores.
 
+    branch : Optional[str], default None
+        The branch to use. If not specified, the branch is set to
+        `user.<username>` unless `production` is set to `True`. This can
+        also be set on the command line using `--branch` as a top-level option.
+        It is an error to specify `branch` in the decorator and on the command line.
+
+    production : bool, default False
+        Whether or not the branch is the production branch. This can also be set on the
+        command line using `--production` as a top-level option. It is an error to specify
+        `production` in the decorator and on the command line.
+        The project branch name will be:
+          - if `branch` is specified:
+            - if `production` is True: `prod.<branch>`
+            - if `production` is False: `test.<branch>`
+          - if `branch` is not specified:
+            - if `production` is True: `prod`
+            - if `production` is False: `user.<username>`
+
     MF Add To Current
     -----------------
     project_name -> str

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/plugins/pypi/micromamba.py

@@ -10,6 +10,7 @@ from metaflow.exception import MetaflowException
 from metaflow.util import which
 
 from .utils import MICROMAMBA_MIRROR_URL, MICROMAMBA_URL, conda_platform
+from threading import Lock
 
 
 class MicromambaException(MetaflowException):
@@ -54,22 +55,33 @@ class Micromamba(object):
             or which(os.path.join(self._path_to_hidden_micromamba, "bin/micromamba"))
         )
 
+        # We keep a mutex as environments are resolved in parallel,
+        # which causes a race condition in case micromamba needs to be installed first.
+        self.install_mutex = Lock()
+
     @property
-    def bin(self):
+    def bin(self) -> str:
         "Defer installing Micromamba until when the binary path is actually requested"
         if self._bin is not None:
             return self._bin
-        # Install Micromamba on the fly.
-        # TODO: Make this optional at some point.
-        _install_micromamba(self._path_to_hidden_micromamba)
-        self._bin = which(
-            os.path.join(self._path_to_hidden_micromamba, "bin/micromamba")
-        )
+        with self.install_mutex:
+            # another check as micromamba might have been installed when the mutex is released.
+            if self._bin is not None:
+                return self._bin
+
+            # Install Micromamba on the fly.
+            # TODO: Make this optional at some point.
+            _install_micromamba(self._path_to_hidden_micromamba)
+            self._bin = which(
+                os.path.join(self._path_to_hidden_micromamba, "bin/micromamba")
+            )
+
+            if self._bin is None:
+                msg = "No installation for *Micromamba* found.\n"
+                msg += "Visit https://mamba.readthedocs.io/en/latest/micromamba-installation.html for installation instructions."
+                raise MetaflowException(msg)
 
-        if self._bin is None:
-            msg = "No installation for *Micromamba* found.\n"
-            msg += "Visit https://mamba.readthedocs.io/en/latest/micromamba-installation.html for installation instructions."
-            raise MetaflowException(msg)
+        return self._bin
 
     def solve(self, id_, packages, python, platform):
         # Performance enhancements

{metaflow-2.13.8 → metaflow-2.13.10}/metaflow/runner/click_api.py

@@ -41,6 +41,7 @@ from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
 from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
+from metaflow.user_configs.config_decorators import CustomFlowDecorator
 from metaflow.user_configs.config_options import (
     ConfigValue,
     ConvertDictOrStr,
@@ -252,10 +253,14 @@ def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
             # Cache the loaded module
             loaded_modules[flow_file] = module
 
-        classes = inspect.getmembers(module, inspect.isclass)
+        classes = inspect.getmembers(
+            module, lambda x: inspect.isclass(x) or isinstance(x, CustomFlowDecorator)
+        )
         flow_cls = None
 
         for _, kls in classes:
+            if isinstance(kls, CustomFlowDecorator):
+                kls = kls._flow_cls
             if (
                 kls is not FlowSpec
                 and kls.__module__ == module_name
@@ -444,10 +449,10 @@ class MetaflowAPI(object):
                 ds = opts.get("datastore", defaults["datastore"])
                 quiet = opts.get("quiet", defaults["quiet"])
                 is_default = False
-                config_file = opts.get("config-file")
+                config_file = opts.get("config")
                 if config_file is None:
                     is_default = True
-                    config_file = defaults.get("config_file")
+                    config_file = defaults.get("config")
 
                 if config_file:
                     config_file = map(
@@ -480,7 +485,7 @@ class MetaflowAPI(object):
                     # Process both configurations; the second one will return all the merged
                     # configuration options properly processed.
                     self._config_input.process_configs(
-                        self._flow_cls.__name__, "config_file", config_file, quiet, ds
+                        self._flow_cls.__name__, "config", config_file, quiet, ds
                     )
                     config_options = self._config_input.process_configs(
                         self._flow_cls.__name__, "config_value", config_value, quiet, ds
@@ -493,7 +498,7 @@ class MetaflowAPI(object):
         # it will init all parameters (config_options will be None)
         # We ignore any errors if we don't check the configs in the click API.
         new_cls = self._flow_cls._process_config_decorators(
-            config_options, ignore_errors=not CLICK_API_PROCESS_CONFIG
+            config_options, process_configs=CLICK_API_PROCESS_CONFIG
         )
         if new_cls:
             self._flow_cls = new_cls
@@ -522,6 +527,12 @@ def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
             )
             arg_parameters[each_param.name] = each_param
         elif isinstance(each_param, click.Option):
+            if each_param.hidden:
+                # Skip hidden options because users should not be setting those.
+                # These are typically internal only options (used by the Runner in part
+                # for example to pass state files or configs to pass local-config-file).
+                continue
+
             opt_params_sigs[each_param.name], annotations[each_param.name] = (
                 get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
             )